From 95cf2c7f27c266f2b1d93cfeebca5ada4bb30e76 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 12 Nov 2026 14:41:43 +0000 Subject: [PATCH 1/2] lfs: record batch and proxied transfer metrics Service gains a MeterProvider field, defaulting to the global provider, mirroring TracerProvider. It records lfs.batch.objects by operation, and lfs.proxied, the bytes of content transferred through the server rather than directly with a presigning storage. Downloads are counted from the result of io.Copy in Serve rather than by wrapping the content reader, which would stop io.Copy from using sendfile. --- lfs/go.mod | 3 ++- lfs/go.sum | 2 ++ lfs/http.go | 7 ++++++- lfs/http_test.go | 42 ++++++++++++++++++++++++++++++++++++++++-- lfs/metrics.go | 38 ++++++++++++++++++++++++++++++++++++++ lfs/service.go | 5 +++++ lfs/tracing.go | 4 ++-- 7 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 lfs/metrics.go diff --git a/lfs/go.mod b/lfs/go.mod index 6fafc18..2253a43 100644 --- a/lfs/go.mod +++ b/lfs/go.mod @@ -10,7 +10,9 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/stretchr/testify v1.12.1 go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 ) @@ -39,7 +41,6 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect - go.opentelemetry.io/otel/metric v1.47.0 // indirect go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/sys v0.48.0 // indirect diff --git a/lfs/go.sum b/lfs/go.sum index 24aae61..5f27dd4 100644 --- a/lfs/go.sum +++ b/lfs/go.sum @@ -84,6 +84,8 @@ go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lq go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= diff --git a/lfs/http.go b/lfs/http.go index 2a7e674..be525ac 100644 --- a/lfs/http.go +++ b/lfs/http.go @@ -126,6 +126,7 @@ func (s *Service) serveUpload(logger *slog.Logger, w http.ResponseWriter, r *htt case result.Existed: w.WriteHeader(http.StatusOK) default: + s.addProxiedBytes(r.Context(), OperationUpload, r.ContentLength) w.WriteHeader(http.StatusNoContent) } } @@ -150,7 +151,11 @@ func (s *Service) serveDownload(logger *slog.Logger, w http.ResponseWriter, r *h w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) w.WriteHeader(http.StatusOK) - if _, err = io.Copy(w, result.Content); err != nil { + // Counted here rather than by wrapping Content, which would stop io.Copy + // from using sendfile for files. + n, err := io.Copy(w, result.Content) + s.addProxiedBytes(r.Context(), OperationDownload, n) + if err != nil { logger.ErrorContext(r.Context(), "Failed to stream object", "oid", oid, "error", err) } } diff --git a/lfs/http_test.go b/lfs/http_test.go index 22cbcea..58a835e 100644 --- a/lfs/http_test.go +++ b/lfs/http_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) func TestService_Serve(t *testing.T) { @@ -39,9 +41,33 @@ func TestService_Serve(t *testing.T) { return `{"operation":"` + operation + `","objects":[{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}]}` } verify := `{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}` + // sums returns the values of the counter, keyed by its lfs.operation. + sums := func(t *testing.T, reader *sdkmetric.ManualReader, name string) map[string]int64 { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + out := map[string]int64{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != name { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + operation, _ := dp.Attributes.Value("lfs.operation") + out[operation.AsString()] = dp.Value + } + } + } + return out + } t.Run("proxied transfer", func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + reader := sdkmetric.NewManualReader() + s := &Service{ + Store: newMemStore(), + Storage: newFilesystemBackend(t, "fs", "file://"), + MeterProvider: sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)), + } w := serve(s, readWrite, http.MethodPost, "/batch", batch(OperationUpload)) require.Equal(t, http.StatusOK, w.Code, w.Body.String()) @@ -74,10 +100,20 @@ func TestService_Serve(t *testing.T) { assert.Equal(t, "application/octet-stream", w.Header().Get("Content-Type")) assert.Equal(t, strconv.Itoa(size), w.Header().Get("Content-Length")) assert.Equal(t, content, w.Body.String()) + + assert.Equal(t, map[string]int64{OperationUpload: 1}, sums(t, reader, "lfs.batch.objects")) + // The rejected upload of forged content is not counted. + wantProxied := map[string]int64{OperationUpload: int64(size), OperationDownload: int64(size)} + assert.Equal(t, wantProxied, sums(t, reader, "lfs.proxied")) }) t.Run("presigned transfer redirects", func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t)} + reader := sdkmetric.NewManualReader() + s := &Service{ + Store: newMemStore(), + Storage: newS3PresignBackend(t), + MeterProvider: sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)), + } w := serve(s, readWrite, http.MethodPut, "/"+oid, "") require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) @@ -102,6 +138,8 @@ func TestService_Serve(t *testing.T) { require.NoError(t, resp.Body.Close()) require.NoError(t, err) assert.Equal(t, content, string(got)) + + assert.Empty(t, sums(t, reader, "lfs.proxied"), "content never went through the server") }) t.Run("checks permissions", func(t *testing.T) { diff --git a/lfs/metrics.go b/lfs/metrics.go new file mode 100644 index 0000000..cf402c6 --- /dev/null +++ b/lfs/metrics.go @@ -0,0 +1,38 @@ +package lfs + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// Instruments are created on use rather than once per Service: the SDK returns +// the same instrument for the same name, and Service has no constructor to +// create them in. +func (s *Service) meter() metric.Meter { + mp := s.MeterProvider + if mp == nil { + mp = otel.GetMeterProvider() + } + return mp.Meter(instrumentationName) +} + +func (s *Service) addBatchObjects(ctx context.Context, operation string, n int) { + counter, _ := s.meter().Int64Counter("lfs.batch.objects", + metric.WithUnit("{object}"), + metric.WithDescription("Objects requested in batch requests."), + ) + counter.Add(ctx, int64(n), metric.WithAttributes(attribute.String("lfs.operation", operation))) +} + +// addProxiedBytes records content that went through the server rather than +// directly between the client and a storage.Presigner. +func (s *Service) addProxiedBytes(ctx context.Context, operation string, n int64) { + counter, _ := s.meter().Int64Counter("lfs.proxied", + metric.WithUnit("By"), + metric.WithDescription("Object content transferred through the server."), + ) + counter.Add(ctx, n, metric.WithAttributes(attribute.String("lfs.operation", operation))) +} diff --git a/lfs/service.go b/lfs/service.go index 1a902ea..e298bcc 100644 --- a/lfs/service.go +++ b/lfs/service.go @@ -14,6 +14,7 @@ import ( "github.com/cockroachdb/errors" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/lfs/internal/iox" @@ -47,6 +48,9 @@ type Service struct { // storage backends. Nil uses the global provider of // go.opentelemetry.io/otel. TracerProvider trace.TracerProvider + // MeterProvider records the metrics of LFS operations. Nil uses the global + // provider of go.opentelemetry.io/otel. + MeterProvider metric.MeterProvider } // Endpoint describes how the client reaches the transfer endpoints of a @@ -94,6 +98,7 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e } oids[i] = o.OID } + s.addBatchObjects(ctx, req.Operation, len(req.Objects)) var existing map[string]Object switch req.Operation { diff --git a/lfs/tracing.go b/lfs/tracing.go index e670e10..4ca7187 100644 --- a/lfs/tracing.go +++ b/lfs/tracing.go @@ -12,14 +12,14 @@ import ( "unknwon.dev/git-lfs-server/lfs/storage" ) -const tracerName = "unknwon.dev/git-lfs-server/lfs" +const instrumentationName = "unknwon.dev/git-lfs-server/lfs" func (s *Service) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { tp := s.TracerProvider if tp == nil { tp = otel.GetTracerProvider() } - return tp.Tracer(tracerName).Start(ctx, name, trace.WithAttributes(attrs...)) + return tp.Tracer(instrumentationName).Start(ctx, name, trace.WithAttributes(attrs...)) } // startStorageSpan starts the span of a call to backend, so that every backend From 58c67c014a35c2e635b0e4913cf3e559fdcea739 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 12 Nov 2026 14:41:44 +0000 Subject: [PATCH 2/2] lfsd: expose Prometheus metrics on a separate listener Add a [metrics] section whose LISTEN_ADDRESS (127.0.0.1:3357 by default, empty to disable) serves /metrics apart from the LFS endpoints. The global meter provider exports to the default Prometheus registry, which also collects the Go runtime and process metrics. - HTTP: route handlers set Request.Pattern, which otelhttp reads to name the server span and to label the request metrics with http.route. - Forge API calls get client request metrics from otelhttp. - Database: the GORM plugin now reports the connection pool stats. - lfsd_auth_cache_lookups_total counts auth cache hits and misses. - lfsd_janitor_objects_total counts swept objects by pass and result. --- README.md | 24 ++++++++++++++++- cmd/lfsd/auth.go | 11 +++++++- cmd/lfsd/config.go | 8 ++++++ cmd/lfsd/main.go | 51 ++++++++++++++++++++++++++++------- cmd/lfsd/main_test.go | 29 +++++++++++++++++--- cmd/lfsd/metrics.go | 31 +++++++++++++++++++++ go.mod | 13 +++++++-- go.sum | 26 ++++++++++++++++-- internal/database/database.go | 6 ++--- internal/embed/config.ini | 6 +++++ internal/janitor/janitor.go | 26 +++++++++++++----- 11 files changed, 201 insertions(+), 30 deletions(-) create mode 100644 cmd/lfsd/metrics.go diff --git a/README.md b/README.md index 48b73b5..c1eb6bc 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ mux.HandleFunc("/{repo}/info/lfs/objects/", func(w http.ResponseWriter, r *http. The lower-level `Batch`, `Upload`, `Download`, and `Verify` methods are available for servers that need their own HTTP handling. -The module depends only on the OpenTelemetry API. `Service.TracerProvider` receives the spans of LFS operations (`lfs.Batch`, `lfs.Upload`, `lfs.Download`, `lfs.Verify`) and of every call to the storage backend (`storage.Put`, `storage.Head`, etc.), so custom backends are traced without extra work. When it is nil, the global provider from `go.opentelemetry.io/otel` is used. +The module depends only on the OpenTelemetry API. `Service.TracerProvider` receives the spans of LFS operations (`lfs.Batch`, `lfs.Upload`, `lfs.Download`, `lfs.Verify`) and of every call to the storage backend (`storage.Put`, `storage.Head`, etc.), so custom backends are traced without extra work. `Service.MeterProvider` receives the `lfs.batch.objects` and `lfs.proxied` metrics. Either provider falls back to the global one from `go.opentelemetry.io/otel` when it is nil. ## Tracing @@ -136,6 +136,28 @@ Each LFS request produces a span named after its route, e.g., `POST /{host}/{**} The janitor records a `janitor.Sweep` trace per sweep. `/healthz` is not traced. Log records written during a traced request carry its `traceID` and `spanID`, and error logs are attached to the span as events. +## Metrics + +lfsd serves Prometheus metrics at `/metrics` on a separate listener, so that they are not public along with the LFS endpoints: + +```ini +[metrics] +; Defaults to 127.0.0.1:3357. Use 0.0.0.0:3357 to scrape from another host or +; container, or leave empty to disable metrics. +LISTEN_ADDRESS = 0.0.0.0:3357 +``` + +| Metric | Labels | Description | +|--------|--------|-------------| +| `http_server_request_duration_seconds`, `http_server_request_body_size_bytes`, `http_server_response_body_size_bytes` | `http_route`, `http_request_method`, `http_response_status_code` | LFS requests. `/healthz` is excluded. | +| `http_client_request_duration_seconds` | `server_address`, `http_response_status_code` | Forge API calls. | +| `lfs_batch_objects_total` | `lfs_operation` | Objects requested in batch requests. | +| `lfs_proxied_bytes_total` | `lfs_operation` | Object content transferred through the server, i.e., with `filesystem` storage. Presigned transfers go directly between the client and the storage. | +| `lfsd_auth_cache_lookups_total` | `lfs_auth_cache_hit` | Lookups of cached authorization results. | +| `lfsd_janitor_objects_total` | `janitor_pass`, `janitor_result` | Objects swept by the janitor, by whether their content was deleted. | +| `go_sql_connections_*` | | Database connection pool. | +| `go_*`, `process_*` | | Go runtime and process. | + ## Local development This project uses [moon](https://moonrepo.dev/) as its task runner. Install it with Homebrew: diff --git a/cmd/lfsd/auth.go b/cmd/lfsd/auth.go index f278fdf..d8623a8 100644 --- a/cmd/lfsd/auth.go +++ b/cmd/lfsd/auth.go @@ -14,19 +14,26 @@ import ( "github.com/flamego/flamego" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/lfs" ) +const instrumentationName = "unknwon.dev/git-lfs-server/cmd/lfsd" + func authorize(forges map[string]forge.Provider) flamego.Handler { + cacheLookups, _ := otel.Meter(instrumentationName).Int64Counter("lfsd.auth.cache.lookups", + metric.WithUnit("{lookup}"), + metric.WithDescription("Lookups of cached authorization results, by whether they hit."), + ) return func(c flamego.Context, logger *logx.Logger, cache cache.Cache) { logger = logger.Scoped("authorize") host := hostFromContext(c) // The span never carries credentials: the forge decides access, and only // its outcome is recorded. - ctx, span := otel.Tracer("unknwon.dev/git-lfs-server/cmd/lfsd").Start(c.Request().Context(), "authorize") + ctx, span := otel.Tracer(instrumentationName).Start(c.Request().Context(), "authorize") defer span.End() provider, ok := forges[host] @@ -58,6 +65,7 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { if v, err := cache.Get(ctx, key); err == nil { if perm, ok := v.(forge.Permission); ok { span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", true), attribute.String("lfs.permission", string(perm))) + cacheLookups.Add(ctx, 1, metric.WithAttributes(attribute.Bool("lfs.auth.cache_hit", true))) c.Map(perm) return } @@ -67,6 +75,7 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { } span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", false)) + cacheLookups.Add(ctx, 1, metric.WithAttributes(attribute.Bool("lfs.auth.cache_hit", false))) perm, rawTTL, err := provider.Authorize(ctx, logger.Scoped("forge"), repo, username, token) if err != nil { if errors.Is(err, forge.ErrTokenInvalid) { diff --git a/cmd/lfsd/config.go b/cmd/lfsd/config.go index c7c6461..9972f3c 100644 --- a/cmd/lfsd/config.go +++ b/cmd/lfsd/config.go @@ -24,6 +24,7 @@ type Config struct { Storages map[Host]storage.Backend Log LogConfig Tracing tracing.Config + Metrics MetricsConfig } type ServerConfig struct { @@ -36,6 +37,10 @@ type LogConfig struct { Level string `ini:"LEVEL"` } +type MetricsConfig struct { + ListenAddress string `ini:"LISTEN_ADDRESS"` +} + const ( storageSectionPrefix = `storage "` forgeSectionPrefix = `forge "` @@ -67,6 +72,9 @@ func loadConfig(customPath string) (*Config, error) { if err := f.Section("tracing").MapTo(&c.Tracing); err != nil { return nil, errors.Wrap(err, `map [tracing] config`) } + if err := f.Section("metrics").MapTo(&c.Metrics); err != nil { + return nil, errors.Wrap(err, `map [metrics] config`) + } backendsByName := make(map[string]storage.Backend) for _, s := range f.Sections() { diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index 851dd32..7ae48e3 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -51,6 +51,20 @@ func main() { if config.Tracing.Enabled { logger.InfoContext(ctx, "Tracing enabled", "sampleRatio", config.Tracing.SampleRatio) } + // Metrics stay no-op without an address, like tracing when disabled. + var metricsSrv *http.Server + if config.Metrics.ListenAddress != "" { + metricsHandler, err := setupMetrics(buildCommit) + if err != nil { + logger.FatalContext(ctx, "Failed to set up metrics", "error", err) + } + mux := http.NewServeMux() + mux.Handle("GET /metrics", metricsHandler) + metricsSrv = &http.Server{ + Addr: config.Metrics.ListenAddress, + Handler: mux, + } + } for host, provider := range config.Forges { logger.InfoContext(ctx, "Forge initialized", "host", host, "type", provider.Type()) @@ -87,10 +101,10 @@ func main() { auth := authorize(config.Forges) const objectsRoute = "/{host}/{**}/info/lfs/objects" f.Group(objectsRoute, func() { - f.Post("/batch", traceRoute(objectsRoute+"/batch"), auth, serve) - f.Get("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) - f.Put("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) - f.Post("/verify", traceRoute(objectsRoute+"/verify"), auth, serve) + f.Post("/batch", instrumentRoute(objectsRoute+"/batch"), auth, serve) + f.Get("/{oid}", instrumentRoute(objectsRoute+"/{oid}"), auth, serve) + f.Put("/{oid}", instrumentRoute(objectsRoute+"/{oid}"), auth, serve) + f.Post("/verify", instrumentRoute(objectsRoute+"/verify"), auth, serve) }) ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) @@ -110,12 +124,26 @@ func main() { } stop() }) + if metricsSrv != nil { + routines.Go(func() { + // Metrics are not worth taking the LFS endpoints down for, so a + // failure here does not stop the server. + if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.ErrorContext(ctx, "Metrics server exited unexpectedly", "error", err) + } + }) + } routines.Go(func() { <-ctx.Done() logger.WarnContext(ctx, "Server shutdown requested") if err := srv.Shutdown(context.Background()); err != nil { logger.ErrorContext(ctx, "Failed to shut down HTTP server gracefully", "error", err) } + if metricsSrv != nil { + if err := metricsSrv.Shutdown(context.Background()); err != nil { + logger.ErrorContext(ctx, "Failed to shut down metrics server gracefully", "error", err) + } + } }) routines.Go(func() { janitor.New(db, config.Storages).Run(ctx, logger.Scoped("janitor")) @@ -124,6 +152,7 @@ func main() { logger.InfoContext(ctx, "Server ready", "buildDate", buildDate, "buildCommit", buildCommit, "listenAddr", config.Server.ListenAddress, + "metricsAddr", config.Metrics.ListenAddress, ) r := routines.WaitAndRecover() // Flush the spans of the requests that finished during shutdown. The @@ -136,13 +165,15 @@ func main() { } } -// traceRoute names the server span of the request after route, because the -// otelhttp handler wraps the router and does not know which route matched. -func traceRoute(route string) flamego.Handler { +// instrumentRoute records route as the pattern that matched the request, like +// http.ServeMux does. The otelhttp handler wraps the router and reads the +// pattern after the request to name the server span and to label the HTTP +// metrics with the route. It only sets the http.route span attribute when the +// span starts, before routing, so that is set here. +func instrumentRoute(route string) flamego.Handler { return func(c flamego.Context) { - span := trace.SpanFromContext(c.Request().Context()) - span.SetName(c.Request().Method + " " + route) - span.SetAttributes( + c.Request().Pattern = route + trace.SpanFromContext(c.Request().Context()).SetAttributes( attribute.String("http.route", route), attribute.String("lfs.forge.host", hostFromContext(c)), attribute.String("lfs.repo", repoNameFromContext(c)), diff --git a/cmd/lfsd/main_test.go b/cmd/lfsd/main_test.go index d00c575..1877532 100644 --- a/cmd/lfsd/main_test.go +++ b/cmd/lfsd/main_test.go @@ -10,20 +10,25 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" ) -func TestTraceRoute(t *testing.T) { +func TestInstrumentRoute(t *testing.T) { recorder := tracetest.NewSpanRecorder() - tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + reader := sdkmetric.NewManualReader() const route = "/{host}/{**}/info/lfs/objects/{oid}" f := flamego.New() - f.Get(route, traceRoute(route), func(c flamego.Context) { + f.Get(route, instrumentRoute(route), func(c flamego.Context) { c.ResponseWriter().WriteHeader(http.StatusOK) }) - handler := otelhttp.NewHandler(f, "lfsd", otelhttp.WithTracerProvider(tp)) + handler := otelhttp.NewHandler(f, "lfsd", + otelhttp.WithTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))), + otelhttp.WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))), + ) r := httptest.NewRequest(http.MethodGet, "/GitHub.com/Org/Repo/info/lfs/objects/abc", nil) handler.ServeHTTP(httptest.NewRecorder(), r) @@ -34,4 +39,20 @@ func TestTraceRoute(t *testing.T) { assert.Contains(t, spans[0].Attributes(), attribute.String("http.route", route)) assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.forge.host", "github.com")) assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.repo", "github.com/org/repo")) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + var routes []string + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != "http.server.request.duration" { + continue + } + for _, dp := range m.Data.(metricdata.Histogram[float64]).DataPoints { + route, _ := dp.Attributes.Value("http.route") + routes = append(routes, route.AsString()) + } + } + } + assert.Equal(t, []string{route}, routes, "the request duration must be labeled with the route") } diff --git a/cmd/lfsd/metrics.go b/cmd/lfsd/metrics.go new file mode 100644 index 0000000..e493c2f --- /dev/null +++ b/cmd/lfsd/metrics.go @@ -0,0 +1,31 @@ +package main + +import ( + "net/http" + + "github.com/cockroachdb/errors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" +) + +// setupMetrics installs the global meter provider and returns the handler of +// the Prometheus endpoint. Metrics are exported to the default Prometheus +// registry, which also collects the Go runtime and process metrics. +func setupMetrics(serviceVersion string) (http.Handler, error) { + exporter, err := otelprometheus.New() + if err != nil { + return nil, errors.Wrap(err, "create Prometheus exporter") + } + otel.SetMeterProvider(sdkmetric.NewMeterProvider( + sdkmetric.WithReader(exporter), + sdkmetric.WithResource(resource.NewSchemaless( + semconv.ServiceName("lfsd"), + semconv.ServiceVersion(serviceVersion), + )), + )) + return promhttp.Handler(), nil +} diff --git a/go.mod b/go.mod index d26f048..bb9eab1 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/flamego/cache v1.5.1 github.com/flamego/flamego v1.9.7 github.com/lib/pq v1.12.3 + github.com/prometheus/client_golang v1.25.0 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 github.com/stretchr/testify v1.12.1 @@ -16,7 +17,10 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/exporters/prometheus v0.69.0 + go.opentelemetry.io/otel/metric v1.47.0 go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 gopkg.in/ini.v1 v1.67.2 gorm.io/driver/postgres v1.6.0 @@ -45,6 +49,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect @@ -82,7 +87,7 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.20.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect @@ -90,9 +95,14 @@ require ( github.com/mattn/go-runewidth v0.0.23 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.72.0 // indirect + github.com/prometheus/otlptranslator v1.0.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -101,7 +111,6 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect - go.opentelemetry.io/otel/metric v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect diff --git a/go.sum b/go.sum index 3c49c1d..7384245 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -156,8 +158,8 @@ github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -165,6 +167,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= @@ -182,6 +186,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= @@ -195,6 +201,16 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= +github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= +github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= +github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= +github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -247,10 +263,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSd go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/exporters/prometheus v0.69.0 h1:LQVBTGnHvzrfGJpDoCSVTY7eaLZArvaTSq5l84Skxew= +go.opentelemetry.io/otel/exporters/prometheus v0.69.0/go.mod h1:moif0nlgj/M16NRT8lDZVt8MC52+EOAJfR8EKRJ8LyU= go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= @@ -263,6 +283,8 @@ go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0 go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/database/database.go b/internal/database/database.go index 18133d6..f96ddb2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -91,9 +91,9 @@ func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { } // Leave bound values out of spans so that they never record request data, - // e.g., object URIs. - if err = gormDB.Use(gormtracing.NewPlugin(gormtracing.WithoutQueryVariables(), gormtracing.WithoutMetrics())); err != nil { - return nil, errors.Wrap(err, "register tracing plugin") + // e.g., object URIs. The plugin also reports the connection pool stats. + if err = gormDB.Use(gormtracing.NewPlugin(gormtracing.WithoutQueryVariables())); err != nil { + return nil, errors.Wrap(err, "register OpenTelemetry plugin") } sqlDB, err := gormDB.DB() diff --git a/internal/embed/config.ini b/internal/embed/config.ini index 5dd4097..73de831 100644 --- a/internal/embed/config.ini +++ b/internal/embed/config.ini @@ -96,3 +96,9 @@ ENDPOINT = ; Fraction of new traces to sample, from 0 to 1. Requests that arrive with a ; sampled trace context are always sampled. SAMPLE_RATIO = 1 + +[metrics] +; Address for the Prometheus metrics endpoint (/metrics) to listen on. It is +; separate from LISTEN_ADDRESS so that metrics are not public along with the LFS +; endpoints. Leave empty to disable metrics. +LISTEN_ADDRESS = 127.0.0.1:3357 diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index 912517a..8db8f5c 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -8,6 +8,7 @@ import ( "github.com/cockroachdb/errors" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/database" @@ -16,6 +17,8 @@ import ( "unknwon.dev/git-lfs-server/lfs/storage" ) +const instrumentationName = "unknwon.dev/git-lfs-server/internal/janitor" + const ( sweepInterval = 30 * time.Minute orphanAge = 24 * time.Hour @@ -25,8 +28,9 @@ const ( // Janitor sweeps orphan (never verified) and unreferenced (repo_count = 0) // objects. type Janitor struct { - db *database.DB - storages []storage.Backend + db *database.DB + storages []storage.Backend + sweptObjects metric.Int64Counter } // New constructs a Janitor using the given list of storage backends. @@ -40,7 +44,11 @@ func New(db *database.DB, storages map[string]storage.Backend) *Janitor { seen[b.Scheme()] = struct{}{} uniq = append(uniq, b) } - return &Janitor{db: db, storages: uniq} + sweptObjects, _ := otel.Meter(instrumentationName).Int64Counter("lfsd.janitor.objects", + metric.WithUnit("{object}"), + metric.WithDescription("Objects the janitor swept, by pass and whether their content was deleted."), + ) + return &Janitor{db: db, storages: uniq, sweptObjects: sweptObjects} } // Run blocks until ctx is canceled, sweeping on the configured interval. The @@ -64,7 +72,7 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) { // Sweep runs both cleanup passes once. Errors are logged but not returned. // The next tick will retry. func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { - tracer := otel.Tracer("unknwon.dev/git-lfs-server/internal/janitor") + tracer := otel.Tracer(instrumentationName) ctx, span := tracer.Start(ctx, "janitor.Sweep") defer span.End() @@ -72,7 +80,7 @@ func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { if objects, err := j.db.SweepOrphanObjects(orphanCtx, orphanAge, sweepBatchSize); err != nil { logger.ErrorContext(orphanCtx, "Failed to sweep orphan objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(orphanCtx, logger.Scoped("orphan"), objects) + j.deleteObjects(orphanCtx, logger.Scoped("orphan"), "orphan", objects) } orphanSpan.End() @@ -80,13 +88,14 @@ func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { if objects, err := j.db.SweepUnreferencedObjects(unreferencedCtx, sweepBatchSize); err != nil { logger.ErrorContext(unreferencedCtx, "Failed to sweep unreferenced objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(unreferencedCtx, logger.Scoped("unreferenced"), objects) + j.deleteObjects(unreferencedCtx, logger.Scoped("unreferenced"), "unreferenced", objects) } unreferencedSpan.End() } // deleteObjects deletes the given list of objects from the storage backends. -func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, objects []database.Object) { +// The pass names the sweep that found them in metrics. +func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, pass string, objects []database.Object) { var deletedCount, failedCount int for _, o := range objects { uri := ptrx.Deref(o.ObjectURI, "") @@ -121,6 +130,9 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object attribute.Int("janitor.deleted_count", deletedCount), attribute.Int("janitor.failed_count", failedCount), ) + passAttr := attribute.String("janitor.pass", pass) + j.sweptObjects.Add(ctx, int64(deletedCount), metric.WithAttributes(passAttr, attribute.String("janitor.result", "deleted"))) + j.sweptObjects.Add(ctx, int64(failedCount), metric.WithAttributes(passAttr, attribute.String("janitor.result", "failed"))) logFn := logger.InfoContext if failedCount > 0 { logFn = logger.ErrorContext