diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 75fe764..91ed31d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,6 +7,8 @@ on: - '**.go' - 'go.mod' - 'go.sum' + - 'lfs/go.mod' + - 'lfs/go.sum' - '.golangci.yml' - '.github/workflows/go.yml' pull_request: @@ -14,6 +16,8 @@ on: - '**.go' - 'go.mod' - 'go.sum' + - 'lfs/go.mod' + - 'lfs/go.sum' - '.golangci.yml' - '.github/workflows/go.yml' env: @@ -38,6 +42,7 @@ jobs: shell: bash run: | go mod tidy + (cd lfs && go mod tidy) STATUS=$(git status --porcelain) if [ ! -z "$STATUS" ]; then echo "Unstaged files:" @@ -50,6 +55,12 @@ jobs: with: version: latest args: --timeout=30m + - name: Run golangci-lint (lfs module) + uses: golangci/golangci-lint-action@9fae48acfc02a90574d7c304a1758ef9895495fa # v7.0.1 + with: + version: latest + working-directory: lfs + args: --timeout=30m test: name: Test @@ -68,6 +79,7 @@ jobs: - name: Run tests with coverage run: | go test -shuffle=on -v -race -coverprofile=coverage -covermode=atomic -json ./... > test-report.json + (cd lfs && go test -shuffle=on -v -race -coverprofile=coverage -covermode=atomic -json ./...) >> test-report.json go install github.com/mfridman/tparse@latest tparse -all -file=test-report.json @@ -88,4 +100,7 @@ jobs: with: go-version: ${{ matrix.go-version }} - name: Run tests with coverage - run: go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... + shell: bash + run: | + go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... + cd lfs && go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3f8a221..4ff901a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,6 +9,7 @@ on: - 'integration-tests/**' - 'go.mod' - 'go.sum' + - 'lfs/**' - '.github/workflows/integration.yml' pull_request: paths: @@ -17,6 +18,7 @@ on: - 'integration-tests/**' - 'go.mod' - 'go.sum' + - 'lfs/**' - '.github/workflows/integration.yml' permissions: diff --git a/.moon/workspace.yml b/.moon/workspace.yml index e16843f..d78ebb2 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -2,3 +2,4 @@ $schema: 'https://moonrepo.dev/schemas/workspace.json' projects: - '.' + - 'lfs' diff --git a/Dockerfile b/Dockerfile index 1fb9a38..606f072 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.27.4-alpine3.24@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS builder WORKDIR /src COPY go.mod go.sum ./ +COPY lfs/go.mod lfs/go.sum ./lfs/ RUN go mod download COPY . . ARG BUILD_DATE diff --git a/README.md b/README.md index e833b39..48b73b5 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,48 @@ REPO_ALLOWLIST = myorg/**, otherorg/specific-repo The allowlist is enforced before the forge token check, so a request for a disallowed repo is rejected without a round-trip to the forge. +## Using as a library + +The `unknwon.dev/git-lfs-server/lfs` module implements the Batch API and the basic transfer adapter for other Git servers to embed. It is a separate Go module so that importers do not pull in the dependencies of lfsd (PostgreSQL, Flamego, forge clients). + +An `lfs.Service` combines an `lfs.Store`, which records objects and the repositories they are linked to, with a `storage.Backend`, which holds the content. A `storage.Proxier` backend (e.g., `storage.FilesystemBackend`) streams content through the server, and a `storage.Presigner` backend (e.g., `storage.S3PresignBackend`) redirects clients to the storage. `PreviousStorages` keeps objects readable after the backend for new uploads changes. + +`Service.Serve` handles the HTTP requests of the batch, upload, download, and verify endpoints. The caller routes the requests, authenticates them, and passes an `lfs.Access` with the repository, the base URL of its endpoints, and whether the actor can read and write. `Serve` works with any router that can run a `net/http` handler, e.g., Flamego, Macaron, or `http.ServeMux`: + +```go +mux.HandleFunc("/{repo}/info/lfs/objects/", func(w http.ResponseWriter, r *http.Request) { + access, ok := authorize(w, r) // Your authentication and authorization. + if !ok { + return + } + service.Serve(logger, w, r, access) +}) +``` + +The lower-level `Batch`, `Upload`, `Download`, and `Verify` methods are available for servers that need their own HTTP handling. + +The module depends only on the OpenTelemetry API. `Service.TracerProvider` receives the spans of LFS operations (`lfs.Batch`, `lfs.Upload`, `lfs.Download`, `lfs.Verify`) and of every call to the storage backend (`storage.Put`, `storage.Head`, etc.), so custom backends are traced without extra work. When it is nil, the global provider from `go.opentelemetry.io/otel` is used. + +## Tracing + +lfsd exports OpenTelemetry traces over OTLP/HTTP when tracing is enabled: + +```ini +[tracing] +ENABLED = true +ENDPOINT = http://otel-collector:4318/v1/traces +SAMPLE_RATIO = 0.1 +``` + +Leave `ENDPOINT` empty to configure the exporter with the standard `OTEL_EXPORTER_OTLP_*` environment variables, e.g., `OTEL_EXPORTER_OTLP_HEADERS` for an API key. `OTEL_SERVICE_NAME` and `OTEL_RESOURCE_ATTRIBUTES` override the default `service.name=lfsd` and `service.version` (the build commit). Requests that carry a sampled W3C `traceparent` are always sampled, so lfsd joins the traces of a reverse proxy or a client that propagates context. + +Each LFS request produces a span named after its route, e.g., `POST /{host}/{**}/info/lfs/objects/batch`, with these children: + +- `authorize`: whether the permission came from the cache, with the forge API call as a child. Credentials are never recorded, and the trace context is not sent to the forge. +- `lfs.Batch`, `lfs.Upload`, `lfs.Download`, or `lfs.Verify`, with the database queries and storage calls of the operation. + +The janitor records a `janitor.Sweep` trace per sweep. `/healthz` is not traced. Log records written during a traced request carry its `traceID` and `spanID`, and error logs are attached to the span as events. + ## Local development This project uses [moon](https://moonrepo.dev/) as its task runner. Install it with Homebrew: diff --git a/cmd/lfsd/auth.go b/cmd/lfsd/auth.go index 9d1dbeb..f278fdf 100644 --- a/cmd/lfsd/auth.go +++ b/cmd/lfsd/auth.go @@ -12,45 +12,52 @@ import ( "github.com/cockroachdb/errors" "github.com/flamego/cache" "github.com/flamego/flamego" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/logx" + "unknwon.dev/git-lfs-server/lfs" ) func authorize(forges map[string]forge.Provider) flamego.Handler { return func(c flamego.Context, logger *logx.Logger, cache cache.Cache) { logger = logger.Scoped("authorize") - ctx := c.Request().Context() - host := hostFromContext(c) + // The span never carries credentials: the forge decides access, and only + // its outcome is recorded. + ctx, span := otel.Tracer("unknwon.dev/git-lfs-server/cmd/lfsd").Start(c.Request().Context(), "authorize") + defer span.End() + provider, ok := forges[host] if !ok { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "unsupported forge host") + lfs.WriteError(c.ResponseWriter(), http.StatusBadRequest, "unsupported forge host") return } username, token, ok := c.Request().BasicAuth() if !ok || token == "" { c.ResponseWriter().Header().Set("WWW-Authenticate", `Basic realm="git-lfs"`) - writeLFSError(c.ResponseWriter(), http.StatusUnauthorized, "basic auth credentials are required") + lfs.WriteError(c.ResponseWriter(), http.StatusUnauthorized, "basic auth credentials are required") return } repo := strings.ToLower(c.Param("**")) if repo == "" { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "repository path is missing") + lfs.WriteError(c.ResponseWriter(), http.StatusBadRequest, "repository path is missing") return } if !provider.Allow(repo) { logger.InfoContext(ctx, "Repository rejected by allowlist", "host", host, "repo", repo) - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "repository is not in the allowlist") + lfs.WriteError(c.ResponseWriter(), http.StatusForbidden, "repository is not in the allowlist") return } key := authCacheKey(host, repo, username, token) if v, err := cache.Get(ctx, key); err == nil { if perm, ok := v.(forge.Permission); ok { + span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", true), attribute.String("lfs.permission", string(perm))) c.Map(perm) return } @@ -59,13 +66,15 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { logger.WarnContext(ctx, "Auth cache lookup failed", "error", err) } + span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", false)) perm, rawTTL, err := provider.Authorize(ctx, logger.Scoped("forge"), repo, username, token) if err != nil { if errors.Is(err, forge.ErrTokenInvalid) { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "token is invalid or lacks repository access") + span.SetAttributes(attribute.Bool("lfs.auth.token_invalid", true)) + lfs.WriteError(c.ResponseWriter(), http.StatusForbidden, "token is invalid or lacks repository access") } else { logger.ErrorContext(ctx, "Failed to verify repository permissions", "repo", repo, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + lfs.WriteError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } return } @@ -80,6 +89,7 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { } } + span.SetAttributes(attribute.String("lfs.permission", string(perm))) c.Map(perm) } } diff --git a/cmd/lfsd/config.go b/cmd/lfsd/config.go index 352ab2c..c7c6461 100644 --- a/cmd/lfsd/config.go +++ b/cmd/lfsd/config.go @@ -10,7 +10,8 @@ import ( "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/embed" "unknwon.dev/git-lfs-server/internal/forge" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/internal/tracing" + "unknwon.dev/git-lfs-server/lfs/storage" ) // Host is the host of a forge, e.g., "github.com". @@ -22,6 +23,7 @@ type Config struct { Forges map[Host]forge.Provider Storages map[Host]storage.Backend Log LogConfig + Tracing tracing.Config } type ServerConfig struct { @@ -62,6 +64,9 @@ func loadConfig(customPath string) (*Config, error) { if err := f.Section("log").MapTo(&c.Log); err != nil { return nil, errors.Wrap(err, `map [log] config`) } + if err := f.Section("tracing").MapTo(&c.Tracing); err != nil { + return nil, errors.Wrap(err, `map [tracing] config`) + } backendsByName := make(map[string]storage.Backend) for _, s := range f.Sections() { diff --git a/cmd/lfsd/lfs.go b/cmd/lfsd/lfs.go index f35aa73..5f2f8d3 100644 --- a/cmd/lfsd/lfs.go +++ b/cmd/lfsd/lfs.go @@ -1,547 +1,33 @@ package main import ( - "encoding/base64" - "encoding/hex" - "encoding/json" "fmt" - "io" - "maps" "net/http" - "regexp" - "slices" - "strconv" - "github.com/cockroachdb/errors" "github.com/flamego/flamego" - "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/forge" - "unknwon.dev/git-lfs-server/internal/iox" "unknwon.dev/git-lfs-server/internal/logx" - "unknwon.dev/git-lfs-server/internal/ptrx" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/lfs" ) -// oidPattern matches the SHA-256 hex digest format used as the LFS object ID: -// exactly 64 lowercase hex characters. -var oidPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) - -const ( - operationUpload = "upload" - operationDownload = "download" - transferBasic = "basic" - mediaTypeLFS = "application/vnd.git-lfs+json" - - // maxJSONRequestBytes caps JSON request bodies (batch, verify) to prevent - // authenticated callers from exhausting memory with oversized payloads. - maxJSONRequestBytes = 10 << 20 -) - -type batchRequest struct { - Operation string `json:"operation"` - Transfers []string `json:"transfers,omitempty"` - Objects []batchObject `json:"objects"` -} - -type batchObject struct { - OID string `json:"oid"` - Size int64 `json:"size"` -} - -type batchResponse struct { - Transfer string `json:"transfer"` - Objects []batchResponseObject `json:"objects"` -} - -type batchResponseObject struct { - OID string `json:"oid"` - Size int64 `json:"size"` - Authenticated bool `json:"authenticated,omitempty"` - Actions map[string]batchAction `json:"actions,omitempty"` - Error *batchObjectError `json:"error,omitempty"` -} - -type batchAction struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` -} - -type batchObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -type verifyRequest struct { - OID string `json:"oid"` - Size int64 `json:"size"` -} - -func serveBatch(db *database.DB, externalURL string, maxObjectSize int64) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("batch") - ctx := c.Request().Context() - - body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) - defer func() { _ = body.Close() }() - - var req batchRequest - if err := json.NewDecoder(body).Decode(&req); err != nil { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) - return - } - - switch req.Operation { - case operationUpload: - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") - return - } - case operationDownload: - if perm != forge.PermissionRead && perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") - return - } - default: - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, `operation must be "upload" or "download"`) - return - } - - if len(req.Transfers) > 0 && !slices.Contains(req.Transfers, transferBasic) { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, `only "basic" transfer is supported`) - return - } - - for _, o := range req.Objects { - if !oidPattern.MatchString(o.OID) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid oid %q", o.OID)) - return - } - } - - repoName := repoNameFromContext(c) - - var authHeader map[string]string - if authz := c.Request().Header.Get("Authorization"); authz != "" { - authHeader = map[string]string{"Authorization": authz} - } - - oids := make([]string, len(req.Objects)) - for i, o := range req.Objects { - oids[i] = o.OID - } - - var ( - existing map[string]database.Object - err error - ) - switch req.Operation { - case operationDownload: - existing, err = db.GetRepoObjectsByOIDs(ctx, repoName, oids) - case operationUpload: - existing, err = db.GetObjectsByOIDs(ctx, oids) - } - if err != nil { - logger.ErrorContext(ctx, "Failed to look up objects", "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, "failed to process batch") - return - } - - out := batchResponse{ - Transfer: transferBasic, - Objects: make([]batchResponseObject, 0, len(req.Objects)), - } - for _, in := range req.Objects { - obj, ok := existing[in.OID] - respObj := batchResponseObject{OID: in.OID, Size: in.Size, Authenticated: true} - - switch req.Operation { - case operationDownload: - switch { - case !ok: - respObj.Error = &batchObjectError{Code: http.StatusNotFound, Message: "object does not exist"} - case ptrx.Deref(obj.Size, -1) != in.Size: - respObj.Error = &batchObjectError{Code: http.StatusUnprocessableEntity, Message: "OID exists with different size"} - default: - respObj.Actions = map[string]batchAction{ - "download": {Href: objectHref(externalURL, repoName, in.OID), Header: authHeader}, - } - } - case operationUpload: - switch { - case maxObjectSize > 0 && in.Size > maxObjectSize: - respObj.Error = &batchObjectError{ - Code: http.StatusRequestEntityTooLarge, - Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, maxObjectSize), - } - case ok && ptrx.Deref(obj.Size, -1) == in.Size: - // Cross-repo dedup: object is verified globally, ensure - // the calling repo is linked so future downloads succeed. - // COALESCE in LinkObject's upsert preserves the existing - // object_uri, so the placeholder is never written. - if _, err := db.LinkObject(ctx, repoName, in.OID, in.Size, "ignored"); err != nil { - logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) - respObj.Error = &batchObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} - } - case ok && ptrx.Deref(obj.Size, -1) != in.Size: - respObj.Error = &batchObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} - default: - rawOID, err := hex.DecodeString(in.OID) - if err != nil { - respObj.Error = &batchObjectError{Code: http.StatusBadRequest, Message: fmt.Sprintf("invalid oid %q", in.OID)} - break - } - uploadHeader := make(map[string]string, len(authHeader)+1) - maps.Copy(uploadHeader, authHeader) - uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) - respObj.Actions = map[string]batchAction{ - "upload": {Href: objectHref(externalURL, repoName, in.OID), Header: uploadHeader}, - "verify": {Href: verifyHref(externalURL, repoName), Header: authHeader}, - } - } - } - out.Objects = append(out.Objects, respObj) - } - - c.ResponseWriter().Header().Set("Content-Type", mediaTypeLFS) - if err := json.NewEncoder(c.ResponseWriter()).Encode(out); err != nil { - logger.ErrorContext(ctx, "Failed to encode batch response", "error", err) - } - } -} - -func serveUpload(db *database.DB, storages map[string]storage.Backend, maxObjectSize int64) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("upload") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") - return - } - - oid := c.Param("oid") - if !oidPattern.MatchString(oid) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - size := c.Request().ContentLength - if size < 0 { - writeLFSError(c.ResponseWriter(), http.StatusLengthRequired, "Content-Length is required") - return - } - if maxObjectSize > 0 && size > maxObjectSize { - writeLFSError(c.ResponseWriter(), http.StatusRequestEntityTooLarge, fmt.Sprintf("object size %d exceeds limit %d", size, maxObjectSize)) - return - } - - backend, ok := storages[hostFromContext(c)] - if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - switch b := backend.(type) { - case storage.Proxier: - serveUploadProxy(c, logger, db, b, oid, size) - case storage.Presigner: - serveUploadPresign(c, logger, db, b, oid, size) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - } -} - -func serveUploadProxy(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Proxier, oid string, size int64) { - ctx := c.Request().Context() - - body := c.Request().Body().ReadCloser() - defer func() { _ = body.Close() }() - - reader := iox.NewSHA256Reader(iox.NewExactSizeReader(body, size), oid) - - uri, err := backend.Put(ctx, oid, reader) - if err != nil { - var sizeErr *iox.SizeMismatchError - var hashErr *iox.SHA256MismatchError - switch { - case errors.As(err, &sizeErr): - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, sizeErr.Error()) - case errors.As(err, &hashErr): - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, hashErr.Error()) - default: - logger.ErrorContext(ctx, "Failed to store object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - return - } - - repoName := repoNameFromContext(c) - storedURI, err := db.LinkObject(ctx, repoName, oid, size, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to link object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if storedURI != uri { - if delErr := backend.Delete(ctx, uri); delErr != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", delErr) - } - } - - c.ResponseWriter().WriteHeader(http.StatusNoContent) -} - -func serveUploadPresign(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Presigner, oid string, size int64) { - ctx := c.Request().Context() - - existing, err := db.GetObjectByOID(ctx, oid) - if err != nil && !errors.Is(err, database.ErrObjectNotFound) { - logger.ErrorContext(ctx, "Failed to look up object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if existing != nil && existing.VerifiedAt != nil { - c.ResponseWriter().WriteHeader(http.StatusOK) - return - } - - if err = db.InsertPendingObject(ctx, oid); err != nil { - logger.ErrorContext(ctx, "Failed to insert pending object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - url, headers, err := backend.PresignPut(ctx, oid, size) - if err != nil { - logger.ErrorContext(ctx, "Failed to presign upload", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - w := c.ResponseWriter() - for k, v := range headers { - w.Header().Set(k, v) - } - w.Header().Set("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) -} - -func serveDownload(db *database.DB, storages map[string]storage.Backend) flamego.Handler { +// serveLFS hands the request to the *lfs.Service of the forge host, with the +// permission resolved by the authorize middleware. +func serveLFS(externalURL string, services map[string]*lfs.Service) flamego.Handler { return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("download") - ctx := c.Request().Context() - - if perm != forge.PermissionRead && perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") - return - } - - oid := c.Param("oid") - if !oidPattern.MatchString(oid) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - backend, ok := storages[hostFromContext(c)] + service, ok := services[hostFromContext(c)] if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + logger.ErrorContext(c.Request().Context(), "No storage backend configured for host") + lfs.WriteError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) return } repoName := repoNameFromContext(c) - object, err := db.GetRepoObjectByOID(ctx, repoName, oid) - if err != nil { - if errors.Is(err, database.ErrObjectNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusNotFound, "object does not exist") - return - } - logger.ErrorContext(ctx, "Failed to look up object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - // GetRepoObjectByOID filters verified rows, so Size and ObjectURI are - // guaranteed non-nil here by the schema CHECK. - objectURI := ptrx.Deref(object.ObjectURI, "") - objectSize := ptrx.Deref(object.Size, -1) - if objectURI == "" || objectSize < 0 { - logger.ErrorContext(ctx, "Verified object has nil size or uri (schema violation)", "oid", oid) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - switch b := backend.(type) { - case storage.Proxier: - serveDownloadProxy(c, logger, b, oid, objectURI, objectSize) - case storage.Presigner: - serveDownloadPresign(c, logger, b, oid, objectURI) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - } -} - -func serveDownloadProxy(c flamego.Context, logger *logx.Logger, backend storage.Proxier, oid, uri string, size int64) { - ctx := c.Request().Context() - - rc, err := backend.Open(ctx, uri) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - logger.ErrorContext(ctx, "DB row exists but storage object is missing", "oid", oid, "uri", uri) - } else { - logger.ErrorContext(ctx, "Failed to open object", "oid", oid, "error", err) - } - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - defer func() { _ = rc.Close() }() - - w := c.ResponseWriter() - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - w.WriteHeader(http.StatusOK) - if _, err = io.Copy(w, rc); err != nil { - logger.ErrorContext(ctx, "Failed to stream object", "oid", oid, "error", err) - } -} - -func serveDownloadPresign(c flamego.Context, logger *logx.Logger, backend storage.Presigner, oid, uri string) { - ctx := c.Request().Context() - - url, err := backend.PresignGet(ctx, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to presign download", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - w := c.ResponseWriter() - w.Header().Set("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) -} - -func serveVerify(db *database.DB, storages map[string]storage.Backend) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("verify") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for verify") - return - } - - body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) - defer func() { _ = body.Close() }() - - var req verifyRequest - if err := json.NewDecoder(body).Decode(&req); err != nil { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) - return - } - if !oidPattern.MatchString(req.OID) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - backend, ok := storages[hostFromContext(c)] - if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - switch b := backend.(type) { - case storage.Proxier: - serveVerifyProxy(c, logger, db, req) - case storage.Presigner: - serveVerifyPresign(c, logger, db, b, req) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - } -} - -func serveVerifyProxy(c flamego.Context, logger *logx.Logger, db *database.DB, req verifyRequest) { - ctx := c.Request().Context() - - repoName := repoNameFromContext(c) - object, err := db.GetRepoObjectByOID(ctx, repoName, req.OID) - if err != nil { - if errors.Is(err, database.ErrObjectNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusNotFound, "object does not exist") - return - } - logger.ErrorContext(ctx, "Failed to look up object", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return + service.Serve(logger.Scoped("lfs").Logger, c.ResponseWriter(), c.Request().Request, lfs.Access{ + Repo: repoName, + Href: fmt.Sprintf("%s/%s/info/lfs/objects", externalURL, repoName), + CanRead: perm == forge.PermissionRead || perm == forge.PermissionWrite, + CanWrite: perm == forge.PermissionWrite, + }) } - if ptrx.Deref(object.Size, -1) != req.Size { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "size mismatch") - return - } - - c.ResponseWriter().WriteHeader(http.StatusNoContent) -} - -func serveVerifyPresign(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Presigner, req verifyRequest) { - ctx := c.Request().Context() - - uri := backend.URI(req.OID) - headSize, err := backend.Head(ctx, uri) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "object does not exist in storage") - return - } - logger.ErrorContext(ctx, "Failed to head object", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - // R2's signed Content-Length should make this impossible, but treat any - // mismatch as a hard error rather than trusting either side. - if headSize != req.Size { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "size mismatch") - return - } - - repoName := repoNameFromContext(c) - storedURI, err := db.VerifyObject(ctx, repoName, req.OID, req.Size, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to mark object verified", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if storedURI != uri { - if delErr := backend.Delete(ctx, uri); delErr != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", delErr) - } - } - - c.ResponseWriter().WriteHeader(http.StatusNoContent) -} - -func objectHref(externalURL, repoName, oid string) string { - return fmt.Sprintf("%s/%s/info/lfs/objects/%s", externalURL, repoName, oid) -} - -func verifyHref(externalURL, repoName string) string { - return fmt.Sprintf("%s/%s/info/lfs/objects/verify", externalURL, repoName) -} - -// writeLFSError writes an error response in the format required by the LFS -// Batch API spec: a JSON body of {"message": "..."} with the -// application/vnd.git-lfs+json content type. All LFS endpoints (batch, upload, -// download, verify, and the auth middleware that fronts them) must use this -// helper instead of http.Error so clients get a parseable body and the right -// content type on every error. -func writeLFSError(w http.ResponseWriter, code int, msg string) { - w.Header().Set("Content-Type", mediaTypeLFS) - w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) } diff --git a/cmd/lfsd/logging.go b/cmd/lfsd/logging.go index d9b8cc2..ab25805 100644 --- a/cmd/lfsd/logging.go +++ b/cmd/lfsd/logging.go @@ -44,5 +44,5 @@ func setupLogging(cfg LogConfig) *logx.Logger { Foreground(lipgloss.Color("226")) handler.SetStyles(styles) - return logx.New(handler) + return logx.New(logx.NewTraceHandler(handler)) } diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index 30e6048..851dd32 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -12,12 +12,17 @@ import ( "github.com/flamego/cache" "github.com/flamego/flamego" "github.com/sourcegraph/conc" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/janitor" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/internal/strx" + "unknwon.dev/git-lfs-server/internal/tracing" + "unknwon.dev/git-lfs-server/lfs" ) // Set via ldflags. @@ -38,6 +43,15 @@ func main() { ctx := context.Background() logger := setupLogging(config.Log) + // Set up before anything that creates spans, e.g., the database plugin. + shutdownTracing, err := tracing.Setup(ctx, config.Tracing, buildCommit) + if err != nil { + logger.FatalContext(ctx, "Failed to set up tracing", "error", err) + } + if config.Tracing.Enabled { + logger.InfoContext(ctx, "Tracing enabled", "sampleRatio", config.Tracing.SampleRatio) + } + for host, provider := range config.Forges { logger.InfoContext(ctx, "Forge initialized", "host", host, "type", provider.Type()) if provider.Type() == forge.TypeSkipAuth { @@ -60,19 +74,33 @@ func main() { f.Get("/healthz", serveHealthz(db)) - f.Group("/{host}/{**}/info/lfs/objects", func() { - f.Post("/batch", serveBatch(db, config.Server.ExternalURL, config.Server.MaxObjectSize)) - f.Get("/{oid}", serveDownload(db, config.Storages)) - f.Put("/{oid}", serveUpload(db, config.Storages, config.Server.MaxObjectSize)) - f.Post("/verify", serveVerify(db, config.Storages)) - }, authorize(config.Forges)) + services := make(map[string]*lfs.Service, len(config.Storages)) + for host, backend := range config.Storages { + services[host] = &lfs.Service{ + Store: db, + Storage: backend, + MaxObjectSize: config.Server.MaxObjectSize, + CrossRepoDedup: true, + } + } + serve := serveLFS(config.Server.ExternalURL, services) + auth := authorize(config.Forges) + const objectsRoute = "/{host}/{**}/info/lfs/objects" + f.Group(objectsRoute, func() { + f.Post("/batch", traceRoute(objectsRoute+"/batch"), auth, serve) + f.Get("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) + f.Put("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) + f.Post("/verify", traceRoute(objectsRoute+"/verify"), auth, serve) + }) ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() srv := &http.Server{ - Addr: config.Server.ListenAddress, - Handler: f, + Addr: config.Server.ListenAddress, + Handler: otelhttp.NewHandler(f, "lfsd", + otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }), + ), } var routines conc.WaitGroup @@ -97,11 +125,31 @@ func main() { "buildDate", buildDate, "buildCommit", buildCommit, "listenAddr", config.Server.ListenAddress, ) - if r := routines.WaitAndRecover(); r != nil { + r := routines.WaitAndRecover() + // Flush the spans of the requests that finished during shutdown. The + // signal context is already canceled, so the flush gets its own. + if err := shutdownTracing(context.Background()); err != nil { + logger.ErrorContext(ctx, "Failed to shut down tracing", "error", err) + } + if r != nil { logger.FatalContext(ctx, "Server panicked", "panic", r.Value, "stack", string(r.Stack)) } } +// traceRoute names the server span of the request after route, because the +// otelhttp handler wraps the router and does not know which route matched. +func traceRoute(route string) flamego.Handler { + return func(c flamego.Context) { + span := trace.SpanFromContext(c.Request().Context()) + span.SetName(c.Request().Method + " " + route) + span.SetAttributes( + attribute.String("http.route", route), + attribute.String("lfs.forge.host", hostFromContext(c)), + attribute.String("lfs.repo", repoNameFromContext(c)), + ) + } +} + func hostFromContext(c flamego.Context) string { return strings.ToLower(c.Param("host")) } diff --git a/cmd/lfsd/main_test.go b/cmd/lfsd/main_test.go new file mode 100644 index 0000000..d00c575 --- /dev/null +++ b/cmd/lfsd/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/flamego/flamego" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestTraceRoute(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + + const route = "/{host}/{**}/info/lfs/objects/{oid}" + f := flamego.New() + f.Get(route, traceRoute(route), func(c flamego.Context) { + c.ResponseWriter().WriteHeader(http.StatusOK) + }) + handler := otelhttp.NewHandler(f, "lfsd", otelhttp.WithTracerProvider(tp)) + + r := httptest.NewRequest(http.MethodGet, "/GitHub.com/Org/Repo/info/lfs/objects/abc", nil) + handler.ServeHTTP(httptest.NewRecorder(), r) + + spans := recorder.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "GET "+route, spans[0].Name()) + assert.Contains(t, spans[0].Attributes(), attribute.String("http.route", route)) + assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.forge.host", "github.com")) + assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.repo", "github.com/org/repo")) +} diff --git a/go.mod b/go.mod index 80c2e1f..d26f048 100644 --- a/go.mod +++ b/go.mod @@ -5,27 +5,36 @@ go 1.27 require ( charm.land/lipgloss/v2 v2.0.3 charm.land/log/v2 v2.0.0 - github.com/aws/aws-sdk-go-v2 v1.41.7 - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 - github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 github.com/flamego/cache v1.5.1 github.com/flamego/flamego v1.9.7 - github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/lib/pq v1.12.3 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 go.bobheadxi.dev/streamline v1.3.2 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 gopkg.in/ini.v1 v1.67.2 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 + gorm.io/plugin/opentelemetry v0.1.16 + unknwon.dev/git-lfs-server/lfs v0.0.0-00010101000000-000000000000 ) require ( bitbucket.org/creachadair/shell v0.0.7 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect github.com/alecthomas/participle/v2 v2.1.4 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect @@ -33,8 +42,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect github.com/charmbracelet/log v1.0.0 // indirect @@ -48,14 +60,20 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect github.com/cockroachdb/redact v1.1.8 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/djherbis/buffer v1.2.0 // indirect github.com/djherbis/nio/v3 v3.0.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/getsentry/sentry-go v0.46.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-logfmt/logfmt v0.6.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/itchyny/gojq v0.12.14 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -64,6 +82,7 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect @@ -71,19 +90,32 @@ require ( github.com/mattn/go-runewidth v0.0.23 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.opentelemetry.io/otel v1.11.0 // indirect - go.opentelemetry.io/otel/trace v1.11.0 // indirect - go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect - golang.org/x/tools v0.48.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect + gorm.io/driver/mysql v1.5.7 // indirect ) + +replace unknwon.dev/git-lfs-server/lfs => ./lfs diff --git a/go.sum b/go.sum index 0cbb3ab..3c49c1d 100644 --- a/go.sum +++ b/go.sum @@ -4,20 +4,26 @@ charm.land/lipgloss/v2 v2.0.3 h1:yM2zJ4Cf5Y51b7RHIwioil4ApI/aypFXXVHSwlM6RzU= charm.land/lipgloss/v2 v2.0.3/go.mod h1:7myLU9iG/3xluAWzpY/fSxYYHCgoKTie7laxk6ATwXA= charm.land/log/v2 v2.0.0 h1:SY3Cey7ipx86/MBXQHwsguOT6X1exT94mmJRdzTNs+s= charm.land/log/v2 v2.0.0/go.mod h1:c3cZSRqm20qUVVAR1WmS/7ab8bgha3C6G7DjPcaVZz0= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= @@ -38,8 +44,10 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= -github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= @@ -70,7 +78,6 @@ github.com/cockroachdb/redact v1.1.8 h1:8eVLLj6juKxiKrAEw2b8cJvNqWq++U8WOfQFuL7K github.com/cockroachdb/redact v1.1.8/go.mod h1:GceHHpJ0rMDpYARL5In88Alq/xMBUtVlz7Qxix6ZVkw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= @@ -79,6 +86,8 @@ github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4 github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flamego/cache v1.5.1 h1:2B4QhLFV7je0oUMCVKsAGAT+OyDHlXhozOoUffm+O3s= github.com/flamego/cache v1.5.1/go.mod h1:cTWYm/Ls35KKHo8vwcKgTlJUNXswEhzFWqVCTFzj24s= github.com/flamego/flamego v1.9.7 h1:x3gkGOALg+HkpqFngkxQ3ZMC2vIa3Kze/WIpYTU2L0k= @@ -89,18 +98,37 @@ github.com/getsentry/sentry-go v0.46.1 h1:mZyQFaQYkPxAdDG4HR8gDg6j4CnKYVWt4TF92N github.com/getsentry/sentry-go v0.46.1/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE= github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/autogold/v2 v2.2.1 h1:JPUXuZQGkcQMv7eeDXuNMovjfoRYaa0yVcm+F3voaGY= github.com/hexops/autogold/v2 v2.2.1/go.mod h1:IJwxtUfj1BGLm0YsR/k+dIxYi6xbeLjqGke2bzcOTMI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -127,8 +155,14 @@ github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RM github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= @@ -141,18 +175,25 @@ github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -161,91 +202,150 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA= -github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= -github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4= go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= -go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= -go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= -go.opentelemetry.io/otel/sdk v1.11.0/go.mod h1:REusa8RsyKaq0OlyangWXaw97t2VogoO4SSEeKkSTAk= -go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI= -go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c h1:ofk4SEqvpf6M5VWh88xRp42IYgdVOh7gffebnDj3OqU= +google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c/go.mod h1:OzYcNQzaMRKcOFRMtyLzG/cfW1SmS2eh+7UmOLLFblk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c h1:XXChExL+Tm7Ubp0x0VXUEncPLHrkXIgoVWqnxmKNbAA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA= +gorm.io/driver/clickhouse v0.7.0/go.mod h1:TmNo0wcVTsD4BBObiRnCahUgHJHjBIwuRejHwYt3JRs= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +gorm.io/plugin/opentelemetry v0.1.16 h1:Kypj2YYAliJqkIczDZDde6P6sFMhKSlG5IpngMFQGpc= +gorm.io/plugin/opentelemetry v0.1.16/go.mod h1:P3RmTeZXT+9n0F1ccUqR5uuTvEXDxF8k2UpO7mTIB2Y= mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= diff --git a/internal/database/database.go b/internal/database/database.go index 8b8eb16..18133d6 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -7,8 +7,10 @@ import ( "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" "gorm.io/driver/postgres" "gorm.io/gorm" + gormtracing "gorm.io/plugin/opentelemetry/tracing" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -57,6 +59,10 @@ func (d *DB) Ping(ctx context.Context) error { // New opens a PostgreSQL connection via GORM and configures the connection pool. func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { + // Groups the startup queries into one trace. + ctx, span := otel.Tracer("unknwon.dev/git-lfs-server/internal/database").Start(ctx, "database.New") + defer span.End() + dsn := fmt.Sprintf( "postgres://%s:%s@%s:%d/%s?sslmode=%s&application_name=lfsd", url.QueryEscape(cfg.User), @@ -84,6 +90,12 @@ func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { return nil, errors.Wrap(err, "open database connection") } + // Leave bound values out of spans so that they never record request data, + // e.g., object URIs. + if err = gormDB.Use(gormtracing.NewPlugin(gormtracing.WithoutQueryVariables(), gormtracing.WithoutMetrics())); err != nil { + return nil, errors.Wrap(err, "register tracing plugin") + } + sqlDB, err := gormDB.DB() if err != nil { return nil, errors.Wrap(err, "get underlying *sql.DB") @@ -109,14 +121,14 @@ func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { } logger.DebugContext(ctx, "Auto-migrating database tables") - if err = gormDB.AutoMigrate(tables...); err != nil { + if err = gormDB.WithContext(ctx).AutoMigrate(tables...); err != nil { return nil, errors.Wrap(err, "auto-migrate database tables") } // AutoMigrate doesn't emit CHECK constraints. Add ours idempotently: // Postgres has no "ADD CONSTRAINT IF NOT EXISTS" form for CHECK, so we // catch duplicate_object via SQLSTATE. - if err = gormDB.Exec(` + if err = gormDB.WithContext(ctx).Exec(` DO $$ BEGIN ALTER TABLE objects ADD CONSTRAINT check_verified_consistency CHECK ( (verified_at IS NULL AND size IS NULL AND object_uri IS NULL) diff --git a/internal/database/objects.go b/internal/database/objects.go index 84200b3..b376fb1 100644 --- a/internal/database/objects.go +++ b/internal/database/objects.go @@ -10,8 +10,11 @@ import ( "gorm.io/gorm" "unknwon.dev/git-lfs-server/internal/ptrx" + "unknwon.dev/git-lfs-server/lfs" ) +var _ lfs.Store = (*DB)(nil) + type Object struct { ID int64 `gorm:"column:id;primaryKey"` OID string `gorm:"column:oid;uniqueIndex;not null"` @@ -24,48 +27,6 @@ type Object struct { var ErrObjectNotFound = errors.New("database: object not found") -// GetRepoObjectByOID fetches the verified Object linked to repoName by oid. -// Pending (unverified) rows are excluded so unverified objects can never serve -// as download sources. Returns ErrObjectNotFound when no row matches. -func (d *DB) GetRepoObjectByOID(ctx context.Context, repoName, oid string) (*Object, error) { - var object Object - err := d.db.WithContext(ctx).Raw(` -SELECT objects.* -FROM objects -JOIN repo_objects ON repo_objects.object_id = objects.id -WHERE - repo_objects.repo_name = @repoName -AND objects.oid = @oid -AND objects.verified_at IS NOT NULL`, - map[string]any{"repoName": repoName, "oid": oid}, - ).Scan(&object).Error - if err != nil { - return nil, errors.Wrap(err, "select repo object by oid") - } - if object.ID == 0 { - return nil, errors.WithStack(ErrObjectNotFound) - } - return &object, nil -} - -// GetObjectByOID fetches an Object by oid (verified or pending) without any -// repo filtering. Used by the upload handler for the dedup short-circuit and -// to detect pending rows. Returns ErrObjectNotFound when no row matches. -func (d *DB) GetObjectByOID(ctx context.Context, oid string) (*Object, error) { - var object Object - err := d.db.WithContext(ctx).Raw( - `SELECT * FROM objects WHERE oid = @oid LIMIT 1`, - map[string]any{"oid": oid}, - ).Scan(&object).Error - if err != nil { - return nil, errors.Wrap(err, "select object by oid") - } - if object.ID == 0 { - return nil, errors.WithStack(ErrObjectNotFound) - } - return &object, nil -} - // LinkObject is the storage.Proxier verify-equivalent path: bytes are already // stored, write a fully-populated (verified) objects row and link it to // repoName in one transaction. Returns the canonical object_uri (which may @@ -195,11 +156,11 @@ WHERE id = @id`, return storedURI, nil } -// GetObjectsByOIDs fetches verified Object rows for the given OIDs. Results are -// keyed by OID, silently dropping missing OIDs. -func (d *DB) GetObjectsByOIDs(ctx context.Context, oids []string) (map[string]Object, error) { +// GetObjects fetches verified objects for the given OIDs. Results are keyed by +// OID, silently dropping missing OIDs. +func (d *DB) GetObjects(ctx context.Context, oids []string) (map[string]lfs.Object, error) { if len(oids) == 0 { - return map[string]Object{}, nil + return map[string]lfs.Object{}, nil } var rows []Object err := d.db.WithContext(ctx).Raw(` @@ -210,19 +171,14 @@ WHERE oid = ANY(@oids) AND verified_at IS NOT NULL`, if err != nil { return nil, errors.Wrap(err, "select objects by OIDs") } - out := make(map[string]Object, len(rows)) - for _, r := range rows { - out[r.OID] = r - } - return out, nil + return verifiedObjectsByOID(rows), nil } -// GetRepoObjectsByOIDs fetches verified Object rows for the given OIDs that are -// linked to the repository. Results are keyed by OID, silently dropping missing -// OIDs. -func (d *DB) GetRepoObjectsByOIDs(ctx context.Context, repoName string, oids []string) (map[string]Object, error) { +// GetRepoObjects fetches verified objects for the given OIDs that are linked to +// the repository. Results are keyed by OID, silently dropping missing OIDs. +func (d *DB) GetRepoObjects(ctx context.Context, repoName string, oids []string) (map[string]lfs.Object, error) { if len(oids) == 0 { - return map[string]Object{}, nil + return map[string]lfs.Object{}, nil } var rows []Object err := d.db.WithContext(ctx).Raw(` @@ -238,11 +194,21 @@ AND objects.verified_at IS NOT NULL`, if err != nil { return nil, errors.Wrap(err, "select repo objects by OIDs") } - out := make(map[string]Object, len(rows)) + return verifiedObjectsByOID(rows), nil +} + +// verifiedObjectsByOID converts verified rows, whose size and object_uri are +// guaranteed non-null by the check_verified_consistency constraint. +func verifiedObjectsByOID(rows []Object) map[string]lfs.Object { + out := make(map[string]lfs.Object, len(rows)) for _, r := range rows { - out[r.OID] = r + out[r.OID] = lfs.Object{ + OID: r.OID, + Size: ptrx.Deref(r.Size, -1), + URI: ptrx.Deref(r.ObjectURI, ""), + } } - return out, nil + return out } // SweepOrphanObjects deletes up to limit pending objects (verified_at IS NULL) diff --git a/internal/embed/config.ini b/internal/embed/config.ini index 366e8c4..5dd4097 100644 --- a/internal/embed/config.ini +++ b/internal/embed/config.ini @@ -85,3 +85,14 @@ TEMP_DIR = %(ROOT)s/.tmp [log] ; Default log level: debug, info, warn, error. LEVEL = debug + +[tracing] +; Set true to export OpenTelemetry traces over OTLP/HTTP. +ENABLED = false +; URL of the OTLP/HTTP traces endpoint, e.g., "http://localhost:4318/v1/traces". +; Leave empty to use the standard OTEL_EXPORTER_OTLP_* environment variables, +; which also configure headers, timeouts, and TLS. +ENDPOINT = +; Fraction of new traces to sample, from 0 to 1. Requests that arrive with a +; sampled trace context are always sampled. +SAMPLE_RATIO = 1 diff --git a/internal/forge/forge.go b/internal/forge/forge.go index f06067e..0b52b19 100644 --- a/internal/forge/forge.go +++ b/internal/forge/forge.go @@ -2,10 +2,13 @@ package forge import ( "context" + "net/http" "strings" "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/propagation" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -69,6 +72,18 @@ type Provider interface { Authorize(ctx context.Context, logger *logx.Logger, repo, username, token string) (Permission, time.Duration, error) } +// newHTTPClient returns the client for forge API calls. Its calls are traced as +// children of the caller's span, but the trace context is not sent to the +// forge, which is outside of our trust boundary. +func newHTTPClient() *http.Client { + return &http.Client{ + Timeout: 30 * time.Second, + Transport: otelhttp.NewTransport(http.DefaultTransport, + otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator()), + ), + } +} + // ErrTokenInvalid signals that the supplied token is invalid or lacks the // required repository access. var ErrTokenInvalid = errors.New("forge: token is invalid or lacks repository access") diff --git a/internal/forge/github.go b/internal/forge/github.go index 5e9ed5c..a6d36a7 100644 --- a/internal/forge/github.go +++ b/internal/forge/github.go @@ -29,7 +29,7 @@ func NewGitHubProvider(host string, allowlist *RepoAllowlist) *GitHubProvider { return &GitHubProvider{ host: host, apiBase: githubAPIBase(host), - client: &http.Client{Timeout: 30 * time.Second}, + client: newHTTPClient(), allowlist: allowlist, } } diff --git a/internal/forge/github_test.go b/internal/forge/github_test.go index e7ef76c..81d0675 100644 --- a/internal/forge/github_test.go +++ b/internal/forge/github_test.go @@ -2,10 +2,18 @@ package forge import ( "context" + "net/http" + "net/http/httptest" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -57,3 +65,43 @@ func TestGitHubTokenTTL(t *testing.T) { assert.Negative(t, githubTokenTTL(ctx, logger, expiry)) }) } + +func TestGitHubProvider_Tracing(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + // The forge client uses the global provider and must ignore the global + // propagator, so install both to prove that nothing is propagated. + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + const token = "ghp_secret" + var gotHeader http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeader = r.Header.Clone() + _, _ = w.Write([]byte(`{"permissions":{"pull":true,"push":true}}`)) + })) + t.Cleanup(server.Close) + + provider := NewGitHubProvider("github.example.com", nil) + provider.apiBase = server.URL + + ctx, parent := tp.Tracer("test").Start(t.Context(), "parent") + perm, _, err := provider.Authorize(ctx, logx.NewNoopLogger(), "org/repo", "x-access-token", token) + parent.End() + require.NoError(t, err) + assert.Equal(t, PermissionWrite, perm) + + assert.Empty(t, gotHeader.Get("traceparent"), "trace context must not be sent to the forge") + + var client sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + if span.SpanKind() == trace.SpanKindClient { + client = span + } + } + require.NotNil(t, client, "forge call must be traced") + assert.Equal(t, parent.SpanContext().SpanID(), client.Parent().SpanID()) + for _, a := range client.Attributes() { + assert.NotContains(t, a.Value.String(), token, "attribute %s", a.Key) + } +} diff --git a/internal/forge/gitlab.go b/internal/forge/gitlab.go index 0f77da3..67d223c 100644 --- a/internal/forge/gitlab.go +++ b/internal/forge/gitlab.go @@ -28,7 +28,7 @@ func (p *GitLabProvider) Allow(repo string) bool { return p.allowlist.Allow(repo func NewGitLabProvider(host string, allowlist *RepoAllowlist) *GitLabProvider { return &GitLabProvider{ baseURL: "https://" + host, - client: &http.Client{Timeout: 30 * time.Second}, + client: newHTTPClient(), allowlist: allowlist, } } diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index c8db2a0..912517a 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -6,11 +6,14 @@ import ( "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/internal/ptrx" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/lfs/storage" ) const ( @@ -61,17 +64,25 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) { // Sweep runs both cleanup passes once. Errors are logged but not returned. // The next tick will retry. func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { - if objects, err := j.db.SweepOrphanObjects(ctx, orphanAge, sweepBatchSize); err != nil { - logger.ErrorContext(ctx, "Failed to sweep orphan objects", "error", err) + tracer := otel.Tracer("unknwon.dev/git-lfs-server/internal/janitor") + ctx, span := tracer.Start(ctx, "janitor.Sweep") + defer span.End() + + orphanCtx, orphanSpan := tracer.Start(ctx, "janitor.SweepOrphans") + if objects, err := j.db.SweepOrphanObjects(orphanCtx, orphanAge, sweepBatchSize); err != nil { + logger.ErrorContext(orphanCtx, "Failed to sweep orphan objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(ctx, logger.Scoped("orphan"), objects) + j.deleteObjects(orphanCtx, logger.Scoped("orphan"), objects) } + orphanSpan.End() - if objects, err := j.db.SweepUnreferencedObjects(ctx, sweepBatchSize); err != nil { - logger.ErrorContext(ctx, "Failed to sweep unreferenced objects", "error", err) + unreferencedCtx, unreferencedSpan := tracer.Start(ctx, "janitor.SweepUnreferenced") + if objects, err := j.db.SweepUnreferencedObjects(unreferencedCtx, sweepBatchSize); err != nil { + logger.ErrorContext(unreferencedCtx, "Failed to sweep unreferenced objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(ctx, logger.Scoped("unreferenced"), objects) + j.deleteObjects(unreferencedCtx, logger.Scoped("unreferenced"), objects) } + unreferencedSpan.End() } // deleteObjects deletes the given list of objects from the storage backends. @@ -106,6 +117,10 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object deletedCount++ } } + trace.SpanFromContext(ctx).SetAttributes( + attribute.Int("janitor.deleted_count", deletedCount), + attribute.Int("janitor.failed_count", failedCount), + ) logFn := logger.InfoContext if failedCount > 0 { logFn = logger.ErrorContext diff --git a/internal/logx/trace_handler.go b/internal/logx/trace_handler.go new file mode 100644 index 0000000..30a11b6 --- /dev/null +++ b/internal/logx/trace_handler.go @@ -0,0 +1,61 @@ +package logx + +import ( + "context" + "log/slog" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// traceHandler wraps a slog.Handler to correlate records with the span in their +// context. Records get the trace and span IDs as attributes, and error records +// are added to the span as events and fail it, so a trace shows why it failed +// without searching the logs. +type traceHandler struct { + handler slog.Handler +} + +// NewTraceHandler returns a handler that correlates records with spans before +// passing them to handler. +func NewTraceHandler(handler slog.Handler) slog.Handler { + return &traceHandler{handler: handler} +} + +func (h *traceHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.handler.Enabled(ctx, level) +} + +func (h *traceHandler) Handle(ctx context.Context, r slog.Record) error { + span := trace.SpanFromContext(ctx) + spanContext := span.SpanContext() + if !spanContext.IsValid() { + return h.handler.Handle(ctx, r) + } + + if r.Level >= slog.LevelError && span.IsRecording() { + attrs := make([]attribute.KeyValue, 0, r.NumAttrs()) + r.Attrs(func(a slog.Attr) bool { + attrs = append(attrs, attribute.String(a.Key, a.Value.String())) + return true + }) + span.AddEvent(r.Message, trace.WithAttributes(attrs...)) + span.SetStatus(codes.Error, r.Message) + } + + r = r.Clone() + r.AddAttrs( + slog.String("traceID", spanContext.TraceID().String()), + slog.String("spanID", spanContext.SpanID().String()), + ) + return h.handler.Handle(ctx, r) +} + +func (h *traceHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &traceHandler{handler: h.handler.WithAttrs(attrs)} +} + +func (h *traceHandler) WithGroup(name string) slog.Handler { + return &traceHandler{handler: h.handler.WithGroup(name)} +} diff --git a/internal/logx/trace_handler_test.go b/internal/logx/trace_handler_test.go new file mode 100644 index 0000000..dedc5cb --- /dev/null +++ b/internal/logx/trace_handler_test.go @@ -0,0 +1,59 @@ +package logx + +import ( + "bytes" + "encoding/json" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestTraceHandler(t *testing.T) { + newLogger := func() (*Logger, *bytes.Buffer) { + var buf bytes.Buffer + return New(NewTraceHandler(slog.NewJSONHandler(&buf, nil))).Scoped("test"), &buf + } + decode := func(t *testing.T, buf *bytes.Buffer) map[string]any { + t.Helper() + var record map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &record)) + return record + } + + t.Run("records outside of a span are unchanged", func(t *testing.T) { + logger, buf := newLogger() + logger.InfoContext(t.Context(), "Hello") + record := decode(t, buf) + assert.NotContains(t, record, "traceID") + assert.NotContains(t, record, "spanID") + }) + + t.Run("records in a span carry its IDs and only errors fail it", func(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + ctx, span := tp.Tracer("test").Start(t.Context(), "span") + + logger, buf := newLogger() + logger.WarnContext(ctx, "Slow") + record := decode(t, buf) + assert.Equal(t, span.SpanContext().TraceID().String(), record["traceID"]) + assert.Equal(t, span.SpanContext().SpanID().String(), record["spanID"]) + + logger.ErrorContext(ctx, "Failed", "error", "boom") + span.End() + + ended := recorder.Ended() + require.Len(t, ended, 1) + assert.Equal(t, codes.Error, ended[0].Status().Code) + assert.Equal(t, "[test] Failed", ended[0].Status().Description) + events := ended[0].Events() + require.Len(t, events, 1, "only the error record becomes an event") + assert.Equal(t, "[test] Failed", events[0].Name) + assert.Equal(t, "boom", events[0].Attributes[0].Value.AsString()) + }) +} diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go new file mode 100644 index 0000000..6fa11b9 --- /dev/null +++ b/internal/tracing/tracing.go @@ -0,0 +1,69 @@ +// Package tracing sets up OpenTelemetry tracing for the process. +package tracing + +import ( + "context" + + "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" +) + +// Config holds the [tracing] section of the config. +type Config struct { + Enabled bool `ini:"ENABLED"` + // Endpoint is the URL of an OTLP/HTTP traces endpoint. Empty falls back to + // the OTEL_EXPORTER_OTLP_* environment variables. + Endpoint string `ini:"ENDPOINT"` + // SampleRatio is the fraction of new traces to sample. Requests that carry + // a sampled trace context are always sampled. + SampleRatio float64 `ini:"SAMPLE_RATIO"` +} + +// Setup installs the global tracer provider and propagator when tracing is +// enabled, and returns the function that flushes and stops them. Instrumented +// code uses the global provider, so nothing is recorded when tracing is +// disabled. +func Setup(ctx context.Context, cfg Config, serviceVersion string) (shutdown func(context.Context) error, err error) { + if !cfg.Enabled { + return func(context.Context) error { return nil }, nil + } + + var opts []otlptracehttp.Option + if cfg.Endpoint != "" { + opts = append(opts, otlptracehttp.WithEndpointURL(cfg.Endpoint)) + } + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, errors.Wrap(err, "create OTLP trace exporter") + } + + res, err := resource.New(ctx, + resource.WithTelemetrySDK(), + resource.WithAttributes( + semconv.ServiceName("lfsd"), + semconv.ServiceVersion(serviceVersion), + ), + // Applied last so that OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES + // override the defaults above. + resource.WithFromEnv(), + ) + if err != nil { + return nil, errors.Wrap(err, "create resource") + } + + provider := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))), + ) + otel.SetTracerProvider(provider) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + return func(ctx context.Context) error { + return errors.Wrap(provider.Shutdown(ctx), "shut down tracer provider") + }, nil +} diff --git a/lfs/go.mod b/lfs/go.mod new file mode 100644 index 0000000..6fafc18 --- /dev/null +++ b/lfs/go.mod @@ -0,0 +1,48 @@ +module unknwon.dev/git-lfs-server/lfs + +go 1.26.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 + github.com/cockroachdb/errors v1.13.0 + github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 + github.com/stretchr/testify v1.12.1 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/getsentry/sentry-go v0.46.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/tools v0.26.0 // indirect +) diff --git a/lfs/go.sum b/lfs/go.sum new file mode 100644 index 0000000..24aae61 --- /dev/null +++ b/lfs/go.sum @@ -0,0 +1,131 @@ +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= +github.com/cockroachdb/errors v1.13.0 h1:BoCcJeiP9hpBJDETkX19qi8Tb8So37srSsp3stTaDMQ= +github.com/cockroachdb/errors v1.13.0/go.mod h1:bjxt/4E5+OyuAnacpTIU9rn2mzPu1VlthvHP+xpROq0= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= +github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= diff --git a/lfs/http.go b/lfs/http.go new file mode 100644 index 0000000..2a7e674 --- /dev/null +++ b/lfs/http.go @@ -0,0 +1,209 @@ +package lfs + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "path" + "strconv" + + "github.com/cockroachdb/errors" +) + +const ( + // MediaType is the content type of LFS API requests and responses. + MediaType = "application/vnd.git-lfs+json" + + // maxJSONRequestBytes caps JSON request bodies (batch, verify) to prevent + // authenticated callers from exhausting memory with oversized payloads. + maxJSONRequestBytes = 10 << 20 +) + +// Access is what the caller resolved for a request after authenticating it. +type Access struct { + // Repo identifies the repository to the Store. + Repo string + // Href is the URL that the transfer endpoints of the repository are served + // under, without a trailing slash: objects at Href/{oid} and verifications + // at Href/verify. + Href string + // CanRead allows downloads. + CanRead bool + // CanWrite allows uploads and verifications. + CanWrite bool +} + +// Serve handles an LFS API request to the repository in access. It dispatches +// on the method and the last segment of the URL path, so it serves any URL +// layout whose endpoints end in "batch", "verify", and the OID: +// +// - POST .../batch +// - GET .../{oid} +// - PUT .../{oid} +// - POST .../verify +// +// The request's Authorization header is forwarded in the actions of batch +// responses, so the client authenticates transfers with the same credentials. +func (s *Service) Serve(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + name := path.Base(r.URL.Path) + switch { + case r.Method == http.MethodPost && name == "batch": + s.serveBatch(logger, w, r, access) + case r.Method == http.MethodPost && name == "verify": + s.serveVerify(logger, w, r, access) + case r.Method == http.MethodGet: + s.serveDownload(logger, w, r, access, name) + case r.Method == http.MethodPut: + s.serveUpload(logger, w, r, access, name) + default: + WriteError(w, http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed)) + } +} + +func (s *Service) serveBatch(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + ctx := r.Context() + + body := http.MaxBytesReader(w, r.Body, maxJSONRequestBytes) + defer func() { _ = body.Close() }() + + var req BatchRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + WriteError(w, http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) + return + } + + switch req.Operation { + case OperationUpload: + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for upload") + return + } + case OperationDownload: + if !access.CanRead { + WriteError(w, http.StatusForbidden, "read permission required for download") + return + } + default: + WriteError(w, http.StatusBadRequest, `operation must be "upload" or "download"`) + return + } + + endpoint := Endpoint{Href: access.Href} + if authz := r.Header.Get("Authorization"); authz != "" { + endpoint.Header = map[string]string{"Authorization": authz} + endpoint.Authenticated = true + } + resp, err := s.Batch(ctx, logger, access.Repo, endpoint, req) + if err != nil { + writeServiceError(logger, w, r, err, "failed to process batch") + return + } + + w.Header().Set("Content-Type", MediaType) + if err := json.NewEncoder(w).Encode(resp); err != nil { + logger.ErrorContext(ctx, "Failed to encode batch response", "error", err) + } +} + +func (s *Service) serveUpload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for upload") + return + } + + defer func() { _ = r.Body.Close() }() + result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.ContentLength, r.Body) + if err != nil { + writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + + switch { + case result.Redirect != nil: + writeRedirect(w, result.Redirect) + case result.Existed: + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNoContent) + } +} + +func (s *Service) serveDownload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { + if !access.CanRead { + WriteError(w, http.StatusForbidden, "read permission required for download") + return + } + + result, err := s.Download(r.Context(), access.Repo, oid) + if err != nil { + writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + if result.Redirect != nil { + writeRedirect(w, result.Redirect) + return + } + + defer func() { _ = result.Content.Close() }() + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) + w.WriteHeader(http.StatusOK) + if _, err = io.Copy(w, result.Content); err != nil { + logger.ErrorContext(r.Context(), "Failed to stream object", "oid", oid, "error", err) + } +} + +func (s *Service) serveVerify(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for verify") + return + } + + body := http.MaxBytesReader(w, r.Body, maxJSONRequestBytes) + defer func() { _ = body.Close() }() + + var req VerifyRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + WriteError(w, http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) + return + } + + if err := s.Verify(r.Context(), logger, access.Repo, req); err != nil { + writeServiceError(logger.With("oid", req.OID), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func writeRedirect(w http.ResponseWriter, redirect *Redirect) { + for k, v := range redirect.Header { + w.Header().Set(k, v) + } + w.Header().Set("Location", redirect.URL) + w.WriteHeader(http.StatusTemporaryRedirect) +} + +// writeServiceError responds with the client error carried by err as an +// *Error. Any other error is internal, so it is logged and the response only +// carries internalMessage. +func writeServiceError(logger *slog.Logger, w http.ResponseWriter, r *http.Request, err error, internalMessage string) { + var lfsErr *Error + if errors.As(err, &lfsErr) { + WriteError(w, lfsErr.Code, lfsErr.Message) + return + } + logger.ErrorContext(r.Context(), "Failed to serve LFS request", "path", r.URL.Path, "error", err) + WriteError(w, http.StatusInternalServerError, internalMessage) +} + +// WriteError writes an error response in the format required by the LFS API: a +// JSON body of {"message": "..."} with the MediaType content type. Callers use +// it for the errors of their own middleware, e.g., authentication, so clients +// get a parseable body on every error. +func WriteError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", MediaType) + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) +} diff --git a/lfs/http_test.go b/lfs/http_test.go new file mode 100644 index 0000000..22cbcea --- /dev/null +++ b/lfs/http_test.go @@ -0,0 +1,132 @@ +package lfs + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestService_Serve(t *testing.T) { + content := "served content" + oid := sha256Hex(content) + size := len(content) + const href = "https://lfs.example.com/repo/info/lfs/objects" + readWrite := Access{Repo: "repo", Href: href, CanRead: true, CanWrite: true} + + serve := func(s *Service, access Access, method, target, body string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, href+target, strings.NewReader(body)) + r.SetBasicAuth("user", "token") + w := httptest.NewRecorder() + s.Serve(discardLogger, w, r, access) + return w + } + requireError := func(t *testing.T, w *httptest.ResponseRecorder, wantCode int) { + t.Helper() + require.Equal(t, wantCode, w.Code, w.Body.String()) + assert.Equal(t, MediaType, w.Header().Get("Content-Type")) + var body struct{ Message string } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.NotEmpty(t, body.Message) + } + batch := func(operation string) string { + return `{"operation":"` + operation + `","objects":[{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}]}` + } + verify := `{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}` + + t.Run("proxied transfer", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + + w := serve(s, readWrite, http.MethodPost, "/batch", batch(OperationUpload)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + assert.Equal(t, MediaType, w.Header().Get("Content-Type")) + var resp BatchResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.Len(t, resp.Objects, 1) + authz := "Basic dXNlcjp0b2tlbg==" + assert.True(t, resp.Objects[0].Authenticated) + assert.Equal(t, Action{ + Href: href + "/" + oid, + Header: map[string]string{ + "Authorization": authz, + "Content-Type": "application/octet-stream", + "x-amz-checksum-sha256": "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", + }, + }, resp.Objects[0].Actions["upload"]) + assert.Equal(t, Action{Href: href + "/verify", Header: map[string]string{"Authorization": authz}}, resp.Objects[0].Actions["verify"]) + + requireError(t, serve(s, readWrite, http.MethodPut, "/"+oid, strings.Repeat("x", size)), http.StatusUnprocessableEntity) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify", verify), http.StatusNotFound) + + w = serve(s, readWrite, http.MethodPut, "/"+oid, content) + assert.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + w = serve(s, readWrite, http.MethodPost, "/verify", verify) + assert.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + + w = serve(s, readWrite, http.MethodGet, "/"+oid, "") + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + assert.Equal(t, "application/octet-stream", w.Header().Get("Content-Type")) + assert.Equal(t, strconv.Itoa(size), w.Header().Get("Content-Length")) + assert.Equal(t, content, w.Body.String()) + }) + + t.Run("presigned transfer redirects", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t)} + + w := serve(s, readWrite, http.MethodPut, "/"+oid, "") + require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) + assert.Equal(t, "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", w.Header().Get("x-amz-checksum-sha256")) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, w.Header().Get("Location"), strings.NewReader(content)) + require.NoError(t, err) + req.Header.Set("x-amz-checksum-sha256", w.Header().Get("x-amz-checksum-sha256")) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + + w = serve(s, readWrite, http.MethodPost, "/verify", verify) + require.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + + w = serve(s, readWrite, http.MethodGet, "/"+oid, "") + require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) + resp, err = http.Get(w.Header().Get("Location")) + require.NoError(t, err) + got, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + }) + + t.Run("checks permissions", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + readOnly := Access{Repo: "repo", Href: href, CanRead: true} + requireError(t, serve(s, readOnly, http.MethodPost, "/batch", batch(OperationUpload)), http.StatusForbidden) + requireError(t, serve(s, readOnly, http.MethodPut, "/"+oid, content), http.StatusForbidden) + requireError(t, serve(s, readOnly, http.MethodPost, "/verify", verify), http.StatusForbidden) + assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodPost, "/batch", batch(OperationDownload)).Code) + + writeOnly := Access{Repo: "repo", Href: href, CanWrite: true} + requireError(t, serve(s, writeOnly, http.MethodPost, "/batch", batch(OperationDownload)), http.StatusForbidden) + requireError(t, serve(s, writeOnly, http.MethodGet, "/"+oid, ""), http.StatusForbidden) + + _, err := s.Upload(t.Context(), discardLogger, "repo", oid, int64(size), strings.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodGet, "/"+oid, "").Code) + }) + + t.Run("rejects invalid requests", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + requireError(t, serve(s, readWrite, http.MethodPost, "/batch", "{"), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/batch", `{"operation":"delete"}`), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify", "{"), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodGet, "/not-an-oid", ""), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodDelete, "/"+oid, ""), http.StatusMethodNotAllowed) + }) +} diff --git a/internal/iox/exact_size_reader.go b/lfs/internal/iox/exact_size_reader.go similarity index 100% rename from internal/iox/exact_size_reader.go rename to lfs/internal/iox/exact_size_reader.go diff --git a/internal/iox/exact_size_reader_test.go b/lfs/internal/iox/exact_size_reader_test.go similarity index 100% rename from internal/iox/exact_size_reader_test.go rename to lfs/internal/iox/exact_size_reader_test.go diff --git a/internal/iox/sha256_reader.go b/lfs/internal/iox/sha256_reader.go similarity index 100% rename from internal/iox/sha256_reader.go rename to lfs/internal/iox/sha256_reader.go diff --git a/internal/iox/sha256_reader_test.go b/lfs/internal/iox/sha256_reader_test.go similarity index 100% rename from internal/iox/sha256_reader_test.go rename to lfs/internal/iox/sha256_reader_test.go diff --git a/lfs/lfs.go b/lfs/lfs.go new file mode 100644 index 0000000..548f32e --- /dev/null +++ b/lfs/lfs.go @@ -0,0 +1,123 @@ +// Package lfs implements the server side of the Git LFS Batch API and the +// basic transfer adapter on top of pluggable metadata and object storage. +// +// It leaves HTTP routing, authentication, authorization, and the encoding of +// responses to the caller, so that servers with different URL layouts, +// credentials, and databases can share the transfer logic. +// +// Spec: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md +package lfs + +import ( + "context" + "regexp" +) + +// oidPattern matches the SHA-256 hex digest format used as the LFS object ID: +// exactly 64 lowercase hex characters. +var oidPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// ValidOID reports whether oid is a valid LFS object ID. +func ValidOID(oid string) bool { + return oidPattern.MatchString(oid) +} + +const ( + OperationUpload = "upload" + OperationDownload = "download" + TransferBasic = "basic" +) + +// BatchRequest is the request payload of the batch endpoint. +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Objects []BatchObject `json:"objects"` +} + +// BatchObject is an object requested in a BatchRequest. +type BatchObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// BatchResponse is the response payload of the batch endpoint. +type BatchResponse struct { + Transfer string `json:"transfer"` + Objects []BatchResponseObject `json:"objects"` +} + +// BatchResponseObject is the result for one object of a BatchRequest. Either +// Actions or Error is set, unless the object needs no transfer. +type BatchResponseObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` + Authenticated bool `json:"authenticated,omitempty"` + Actions map[string]Action `json:"actions,omitempty"` + Error *ObjectError `json:"error,omitempty"` +} + +// Action tells the client where and how to transfer an object. +type Action struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` +} + +// ObjectError is the error of one object in a BatchResponse. +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// VerifyRequest is the request payload of the verify endpoint. +type VerifyRequest struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// Error is a client error that the caller should respond with, using Code as +// the HTTP status code. +type Error struct { + Code int + Message string +} + +func (e *Error) Error() string { + return e.Message +} + +// Object is a verified LFS object. +type Object struct { + OID string + Size int64 + // URI is the location of the object content, as returned by the storage + // backend that stores it. + URI string +} + +// Store is the metadata layer of LFS objects. An object is "pending" from the +// start of a presigned upload until it is verified, and only verified objects +// are served. A repository is identified by an opaque string chosen by the +// caller. +type Store interface { + // GetRepoObjects returns the verified objects among oids that are linked to + // repo, keyed by OID. Missing OIDs are omitted. + GetRepoObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) + // GetObjects returns the verified objects among oids regardless of the + // repositories they are linked to, keyed by OID. Missing OIDs are omitted. + // It is only called when Service.CrossRepoDedup is true. + GetObjects(ctx context.Context, oids []string) (map[string]Object, error) + // LinkObject records a verified object whose content is stored at uri and + // links it to repo. It returns the URI the object is recorded with, and when + // that differs from uri, the caller deletes the content at uri as no longer + // referenced. + LinkObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) + // InsertPendingObject records a pending object for oid, unless the object is + // already recorded, so that abandoned presigned uploads can be cleaned up. + InsertPendingObject(ctx context.Context, oid string) error + // VerifyObject marks the pending object for oid as verified with content + // stored at uri and links it to repo. It returns the URI the object is + // recorded with, like LinkObject. It fails when no object is recorded for + // oid. + VerifyObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) +} diff --git a/lfs/moon.yml b/lfs/moon.yml new file mode 100644 index 0000000..7605311 --- /dev/null +++ b/lfs/moon.yml @@ -0,0 +1,45 @@ +$schema: 'https://moonrepo.dev/schemas/project.json' + +language: 'go' +stack: 'backend' +id: 'lfs' + +fileGroups: + sources: + - '**/*.go' + tests: + - '**/*_test.go' + configs: + - 'go.mod' + - 'go.sum' + +tasks: + install: + command: 'go mod tidy' + inputs: + - '@group(configs)' + + lint: + command: 'golangci-lint run' + inputs: + - '@group(sources)' + - '@group(configs)' + deps: + - 'install' + + test: + command: 'go test ./...' + inputs: + - '@group(sources)' + - '@group(tests)' + - '@group(configs)' + deps: + - 'install' + + build: + command: 'go build -o /dev/null ./...' + inputs: + - '@group(sources)' + - '@group(configs)' + deps: + - 'install' diff --git a/lfs/service.go b/lfs/service.go new file mode 100644 index 0000000..1a902ea --- /dev/null +++ b/lfs/service.go @@ -0,0 +1,429 @@ +package lfs + +import ( + "context" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "log/slog" + "maps" + "net/http" + "slices" + "strings" + + "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "unknwon.dev/git-lfs-server/lfs/internal/iox" + "unknwon.dev/git-lfs-server/lfs/storage" +) + +// Service serves LFS operations for repositories. Callers authenticate and +// authorize each request, then either hand it to Serve, or call the operation +// methods and respond with the Code and Message of a returned *Error. Any other +// error returned by an operation method is an internal error. +type Service struct { + Store Store + // Storage receives uploads and decides the transfer mode: content is + // proxied through the caller for a storage.Proxier, and transferred + // directly by the client through redirects for a storage.Presigner. + Storage storage.Backend + // PreviousStorages are consulted for downloads of objects stored before + // Storage was changed. An object is served by the first backend, starting + // with Storage, whose scheme prefixes the object's URI. + PreviousStorages []storage.Backend + // MaxObjectSize is the maximum size in bytes of an uploaded object. Zero or + // negative means no limit. + MaxObjectSize int64 + // CrossRepoDedup allows an upload to be satisfied by an object that is + // already stored for another repository, without the client sending the + // content again. Without it, a repository only gains an object by uploading + // its full content, so knowing an OID and its size is not enough to read the + // object through a repository the client has write access to. + CrossRepoDedup bool + // TracerProvider creates the spans of LFS operations and of the calls to + // storage backends. Nil uses the global provider of + // go.opentelemetry.io/otel. + TracerProvider trace.TracerProvider +} + +// Endpoint describes how the client reaches the transfer endpoints of a +// repository: objects at Href/{oid} and verifications at Href/verify. +type Endpoint struct { + Href string + // Header is included in every action, e.g., to forward the credentials of + // the batch request. + Header map[string]string + // Authenticated tells the client that Header authenticates the actions, so + // it does not look up credentials for them. It is required when Storage is a + // storage.Presigner: otherwise the Git LFS client looks up credentials for + // the host of the presigned URL it is redirected to, and fails. + Authenticated bool +} + +// Redirect sends the client to another location to transfer the content. +type Redirect struct { + URL string + // Header must be sent by the client verbatim. + Header map[string]string +} + +// Batch answers a batch request for repo. Callers must check that the actor is +// allowed to perform req.Operation before calling it. +func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, endpoint Endpoint, req BatchRequest) (_ *BatchResponse, err error) { + ctx, span := s.startSpan(ctx, "lfs.Batch", + attribute.String("lfs.repo", repo), + attribute.String("lfs.operation", req.Operation), + attribute.Int("lfs.object_count", len(req.Objects)), + ) + defer func() { endSpan(span, err) }() + + if req.Operation != OperationUpload && req.Operation != OperationDownload { + return nil, &Error{Code: http.StatusBadRequest, Message: `operation must be "upload" or "download"`} + } + if len(req.Transfers) > 0 && !slices.Contains(req.Transfers, TransferBasic) { + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: `only "basic" transfer is supported`} + } + + oids := make([]string, len(req.Objects)) + for i, o := range req.Objects { + if !ValidOID(o.OID) { + return nil, &Error{Code: http.StatusBadRequest, Message: fmt.Sprintf("invalid oid %q", o.OID)} + } + oids[i] = o.OID + } + + var existing map[string]Object + switch req.Operation { + case OperationDownload: + existing, err = s.Store.GetRepoObjects(ctx, repo, oids) + case OperationUpload: + existing, err = s.uploadedObjects(ctx, repo, oids) + } + if err != nil { + return nil, errors.Wrap(err, "look up objects") + } + + objectHref := func(oid string) string { return endpoint.Href + "/" + oid } + resp := &BatchResponse{ + Transfer: TransferBasic, + Objects: make([]BatchResponseObject, 0, len(req.Objects)), + } + for _, in := range req.Objects { + obj, ok := existing[in.OID] + out := BatchResponseObject{OID: in.OID, Size: in.Size, Authenticated: endpoint.Authenticated} + + switch req.Operation { + case OperationDownload: + switch { + case !ok: + out.Error = &ObjectError{Code: http.StatusNotFound, Message: "object does not exist"} + case obj.Size != in.Size: + out.Error = &ObjectError{Code: http.StatusUnprocessableEntity, Message: "OID exists with different size"} + default: + out.Actions = map[string]Action{ + "download": {Href: objectHref(in.OID), Header: endpoint.Header}, + } + } + case OperationUpload: + switch { + case s.MaxObjectSize > 0 && in.Size > s.MaxObjectSize: + out.Error = &ObjectError{ + Code: http.StatusRequestEntityTooLarge, + Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, s.MaxObjectSize), + } + case ok && obj.Size == in.Size: + // No actions tell the client the object needs no upload. With + // cross-repo dedup, the object may only be linked to another + // repository so far. + if s.CrossRepoDedup { + if _, err := s.Store.LinkObject(ctx, repo, in.OID, in.Size, obj.URI); err != nil { + logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) + out.Error = &ObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} + } + } + case ok && obj.Size != in.Size: + out.Error = &ObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} + default: + rawOID, _ := hex.DecodeString(in.OID) // Validated above. + uploadHeader := make(map[string]string, len(endpoint.Header)+2) + maps.Copy(uploadHeader, endpoint.Header) + // The Git LFS client otherwise sends a Content-Type detected from + // the file content. + uploadHeader["Content-Type"] = "application/octet-stream" + // Presigned uploads require the client to send the checksum that + // the storage verifies the content against. + uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) + out.Actions = map[string]Action{ + "upload": {Href: objectHref(in.OID), Header: uploadHeader}, + "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + } + } + } + resp.Objects = append(resp.Objects, out) + } + return resp, nil +} + +// uploadedObjects returns the verified objects among oids that an upload to +// repo does not need to send again. +func (s *Service) uploadedObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) { + if s.CrossRepoDedup { + return s.Store.GetObjects(ctx, oids) + } + return s.Store.GetRepoObjects(ctx, repo, oids) +} + +// UploadResult is the outcome of a successful Service.Upload. +type UploadResult struct { + // Redirect is set when the client must upload the content to another + // location and then call the verify endpoint. + Redirect *Redirect + // Existed reports that the object was already stored and the content was not + // read. + Existed bool +} + +// Upload handles an upload of size bytes for oid to repo. For a +// storage.Proxier, it stores the content read from body after verifying its +// size and hash. For a storage.Presigner, it does not read body. +func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (_ *UploadResult, err error) { + ctx, span := s.startSpan(ctx, "lfs.Upload", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", oid), + attribute.Int64("lfs.size", size), + ) + defer func() { endSpan(span, err) }() + + if !ValidOID(oid) { + return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + if size < 0 { + return nil, &Error{Code: http.StatusLengthRequired, Message: "Content-Length is required"} + } + if s.MaxObjectSize > 0 && size > s.MaxObjectSize { + return nil, &Error{ + Code: http.StatusRequestEntityTooLarge, + Message: fmt.Sprintf("object size %d exceeds limit %d", size, s.MaxObjectSize), + } + } + + switch b := s.Storage.(type) { + case storage.Proxier: + return s.uploadProxy(ctx, logger, b, repo, oid, size, body) + case storage.Presigner: + return s.uploadPresign(ctx, b, repo, oid, size) + default: + return nil, errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) + } +} + +func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend storage.Proxier, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { + reader := iox.NewSHA256Reader(iox.NewExactSizeReader(body, size), oid) + putCtx, span := s.startStorageSpan(ctx, "Put", backend) + uri, err := backend.Put(putCtx, oid, reader) + endSpan(span, err) + if err == nil { + // Put skips reading when the content is already stored, but the client + // must still prove it has the content before the object is linked to + // repo. + _, err = io.Copy(io.Discard, reader) + } + if err != nil { + var sizeErr *iox.SizeMismatchError + var hashErr *iox.SHA256MismatchError + switch { + case errors.As(err, &sizeErr): + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: sizeErr.Error()} + case errors.As(err, &hashErr): + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: hashErr.Error()} + default: + return nil, errors.Wrap(err, "store object") + } + } + + storedURI, err := s.Store.LinkObject(ctx, repo, oid, size, uri) + if err != nil { + return nil, errors.Wrap(err, "link object") + } + if storedURI != uri { + s.deleteOrphan(ctx, logger, backend, uri) + } + return &UploadResult{}, nil +} + +func (s *Service) uploadPresign(ctx context.Context, backend storage.Presigner, repo, oid string, size int64) (*UploadResult, error) { + existing, err := s.uploadedObjects(ctx, repo, []string{oid}) + if err != nil { + return nil, errors.Wrap(err, "look up object") + } + if _, ok := existing[oid]; ok { + return &UploadResult{Existed: true}, nil + } + + if err = s.Store.InsertPendingObject(ctx, oid); err != nil { + return nil, errors.Wrap(err, "insert pending object") + } + presignCtx, span := s.startStorageSpan(ctx, "PresignPut", backend) + url, header, err := backend.PresignPut(presignCtx, oid, size) + endSpan(span, err) + if err != nil { + return nil, errors.Wrap(err, "presign upload") + } + return &UploadResult{Redirect: &Redirect{URL: url, Header: header}}, nil +} + +// DownloadResult is the outcome of a successful Service.Download. Exactly one +// of Redirect and Content is set. +type DownloadResult struct { + // Redirect is set when the client must download the content from another + // location. + Redirect *Redirect + // Content is the object content, which the caller must close. + Content io.ReadCloser + // Size is the size of Content in bytes. + Size int64 +} + +// Download handles a download of oid from repo. +func (s *Service) Download(ctx context.Context, repo, oid string) (_ *DownloadResult, err error) { + ctx, span := s.startSpan(ctx, "lfs.Download", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", oid), + ) + defer func() { endSpan(span, err) }() + + if !ValidOID(oid) { + return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + + objects, err := s.Store.GetRepoObjects(ctx, repo, []string{oid}) + if err != nil { + return nil, errors.Wrap(err, "look up object") + } + object, ok := objects[oid] + if !ok { + return nil, &Error{Code: http.StatusNotFound, Message: "object does not exist"} + } + + backend := s.backendFor(object.URI) + switch b := backend.(type) { + case storage.Proxier: + openCtx, span := s.startStorageSpan(ctx, "Open", b) + rc, err := b.Open(openCtx, object.URI) + endSpan(span, err) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, errors.Wrapf(err, "object %q is recorded but missing in storage", object.URI) + } + return nil, errors.Wrap(err, "open object") + } + return &DownloadResult{Content: rc, Size: object.Size}, nil + case storage.Presigner: + presignCtx, span := s.startStorageSpan(ctx, "PresignGet", b) + url, err := b.PresignGet(presignCtx, object.URI) + endSpan(span, err) + if err != nil { + return nil, errors.Wrap(err, "presign download") + } + return &DownloadResult{Redirect: &Redirect{URL: url}}, nil + case nil: + return nil, errors.Newf("no storage backend for uri %q", object.URI) + default: + return nil, errors.Newf("storage backend %T does not implement a known transfer mode", backend) + } +} + +// backendFor returns the backend that stores the object at uri, or nil when +// none does. +func (s *Service) backendFor(uri string) storage.Backend { + if strings.HasPrefix(uri, s.Storage.Scheme()) { + return s.Storage + } + for _, b := range s.PreviousStorages { + if strings.HasPrefix(uri, b.Scheme()) { + return b + } + } + return nil +} + +// Verify handles the verification of an upload to repo. A presigned upload +// becomes downloadable only after it is verified. +func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) (err error) { + ctx, span := s.startSpan(ctx, "lfs.Verify", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", req.OID), + attribute.Int64("lfs.size", req.Size), + ) + defer func() { endSpan(span, err) }() + + if !ValidOID(req.OID) { + return &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + + switch b := s.Storage.(type) { + case storage.Proxier: + return s.verifyProxy(ctx, repo, req) + case storage.Presigner: + return s.verifyPresign(ctx, logger, b, repo, req) + default: + return errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) + } +} + +func (s *Service) verifyProxy(ctx context.Context, repo string, req VerifyRequest) error { + objects, err := s.Store.GetRepoObjects(ctx, repo, []string{req.OID}) + if err != nil { + return errors.Wrap(err, "look up object") + } + object, ok := objects[req.OID] + if !ok { + return &Error{Code: http.StatusNotFound, Message: "object does not exist"} + } + if object.Size != req.Size { + return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} + } + return nil +} + +func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo string, req VerifyRequest) error { + uri := backend.URI(req.OID) + headCtx, span := s.startStorageSpan(ctx, "Head", backend) + headSize, err := backend.Head(headCtx, uri) + endSpan(span, err) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return &Error{Code: http.StatusUnprocessableEntity, Message: "object does not exist in storage"} + } + return errors.Wrap(err, "head object") + } + // The presigned upload signs Content-Length, which should make this + // impossible, but treat any mismatch as a hard error rather than trusting + // either side. + if headSize != req.Size { + return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} + } + + storedURI, err := s.Store.VerifyObject(ctx, repo, req.OID, req.Size, uri) + if err != nil { + return errors.Wrap(err, "mark object verified") + } + if storedURI != uri { + s.deleteOrphan(ctx, logger, backend, uri) + } + return nil +} + +// deleteOrphan deletes the content at uri that was stored again while the +// object was already recorded with another URI. +func (s *Service) deleteOrphan(ctx context.Context, logger *slog.Logger, backend storage.Backend, uri string) { + ctx, span := s.startStorageSpan(ctx, "Delete", backend) + err := backend.Delete(ctx, uri) + endSpan(span, err) + if err != nil { + logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) + } +} diff --git a/lfs/service_test.go b/lfs/service_test.go new file mode 100644 index 0000000..e173ec5 --- /dev/null +++ b/lfs/service_test.go @@ -0,0 +1,447 @@ +package lfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/cockroachdb/errors" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +// memStore is an in-memory Store. +type memStore struct { + mu sync.Mutex + objects map[string]*memObject // Keyed by OID. + links map[string]map[string]bool // Keyed by repo, then OID. +} + +type memObject struct { + size int64 + uri string + verified bool +} + +func newMemStore() *memStore { + return &memStore{objects: map[string]*memObject{}, links: map[string]map[string]bool{}} +} + +func (s *memStore) verifiedObjects(oids []string, keep func(oid string) bool) map[string]Object { + out := map[string]Object{} + for _, oid := range oids { + if o := s.objects[oid]; o != nil && o.verified && keep(oid) { + out[oid] = Object{OID: oid, Size: o.size, URI: o.uri} + } + } + return out +} + +func (s *memStore) GetRepoObjects(_ context.Context, repo string, oids []string) (map[string]Object, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.verifiedObjects(oids, func(oid string) bool { return s.links[repo][oid] }), nil +} + +func (s *memStore) GetObjects(_ context.Context, oids []string) (map[string]Object, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.verifiedObjects(oids, func(string) bool { return true }), nil +} + +func (s *memStore) link(repo, oid string) { + if s.links[repo] == nil { + s.links[repo] = map[string]bool{} + } + s.links[repo][oid] = true +} + +func (s *memStore) LinkObject(_ context.Context, repo, oid string, size int64, uri string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + o := s.objects[oid] + if o == nil { + o = &memObject{} + s.objects[oid] = o + } + if !o.verified { + o.size, o.uri, o.verified = size, uri, true + } + s.link(repo, oid) + return o.uri, nil +} + +func (s *memStore) InsertPendingObject(_ context.Context, oid string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.objects[oid] == nil { + s.objects[oid] = &memObject{} + } + return nil +} + +func (s *memStore) VerifyObject(_ context.Context, repo, oid string, size int64, uri string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + o := s.objects[oid] + if o == nil { + return "", errors.New("object not found") + } + if !o.verified { + o.size, o.uri, o.verified = size, uri, true + } + s.link(repo, oid) + return o.uri, nil +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func newFilesystemBackend(t *testing.T, name, scheme string) *storage.FilesystemBackend { + t.Helper() + root := t.TempDir() + b, err := storage.NewFilesystemBackend(name, scheme, root, filepath.Join(root, ".tmp")) + require.NoError(t, err) + return b +} + +func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { + t.Helper() + mem := s3mem.New() + require.NoError(t, mem.CreateBucket("lfs-objects")) + srv := httptest.NewServer(gofakes3.New(mem).Server()) + t.Cleanup(srv.Close) + b, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) + require.NoError(t, err) + return b +} + +var discardLogger = slog.New(slog.DiscardHandler) + +func requireLFSError(t *testing.T, err error, wantCode int) { + t.Helper() + var lfsErr *Error + require.True(t, errors.As(err, &lfsErr), "want *Error, got %v", err) + assert.Equal(t, wantCode, lfsErr.Code, lfsErr.Message) +} + +func TestService_Proxy(t *testing.T) { + ctx := t.Context() + content := "proxied content" + oid := sha256Hex(content) + size := int64(len(content)) + + t.Run("upload stores content and links it to the repository only", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, &UploadResult{}, result) + + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) + + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + got, err := io.ReadAll(download.Content) + require.NoError(t, download.Content.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + assert.Equal(t, size, download.Size) + + _, err = s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + }) + + t.Run("upload of stored content still verifies the body", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + forged := strings.Repeat("x", len(content)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(forged)) + requireLFSError(t, err, http.StatusUnprocessableEntity) + _, err = s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + + _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(content)) + require.NoError(t, err) + download, err := s.Download(ctx, "repo2", oid) + require.NoError(t, err) + require.NoError(t, download.Content.Close()) + }) + + t.Run("upload rejects invalid input", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), MaxObjectSize: size - 1} + _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", size, strings.NewReader(content)) + requireLFSError(t, err, http.StatusBadRequest) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, -1, strings.NewReader(content)) + requireLFSError(t, err, http.StatusLengthRequired) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + requireLFSError(t, err, http.StatusRequestEntityTooLarge) + + s.MaxObjectSize = 0 + _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content[1:])) + requireLFSError(t, err, http.StatusUnprocessableEntity) + }) + + t.Run("download reads objects from previous storages", func(t *testing.T) { + store := newMemStore() + previous := newFilesystemBackend(t, "old", "old-file://") + s := &Service{Store: store, Storage: previous} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + s = &Service{Store: store, Storage: newFilesystemBackend(t, "new", "file://"), PreviousStorages: []storage.Backend{previous}} + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + got, err := io.ReadAll(download.Content) + require.NoError(t, download.Content.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + + s.PreviousStorages = nil + _, err = s.Download(ctx, "repo1", oid) + require.Error(t, err) + assert.False(t, errors.As(err, new(*Error)), "a recorded object without a backend is an internal error") + }) +} + +func TestService_Batch(t *testing.T) { + ctx := t.Context() + content := "batched content" + oid := sha256Hex(content) + size := int64(len(content)) + endpoint := Endpoint{Href: "https://lfs.example.com/repo/info/lfs/objects"} + + newServiceWithObject := func(t *testing.T, crossRepoDedup bool) *Service { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), CrossRepoDedup: crossRepoDedup} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + return s + } + upload := BatchRequest{Operation: OperationUpload, Objects: []BatchObject{{OID: oid, Size: size}}} + download := BatchRequest{Operation: OperationDownload, Objects: []BatchObject{{OID: oid, Size: size}}} + + t.Run("upload of an object in the repository needs no transfer", func(t *testing.T) { + s := newServiceWithObject(t, false) + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, upload) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + }) + + t.Run("upload of an object in another repository", func(t *testing.T) { + t.Run("needs a transfer without cross-repo dedup", func(t *testing.T) { + s := newServiceWithObject(t, false) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + assert.Contains(t, resp.Objects[0].Actions, "upload") + + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + require.NotNil(t, resp.Objects[0].Error) + assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) + }) + + t.Run("links the object with cross-repo dedup", func(t *testing.T) { + s := newServiceWithObject(t, true) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + assert.Contains(t, resp.Objects[0].Actions, "download") + }) + }) + + t.Run("actions carry the endpoint", func(t *testing.T) { + s := newServiceWithObject(t, false) + endpoint := Endpoint{Href: endpoint.Href, Header: map[string]string{"Authorization": "Basic Zm9vOmJhcg=="}, Authenticated: true} + newContent := "new content" + newOID := sha256Hex(newContent) + + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, BatchRequest{ + Operation: OperationUpload, + Objects: []BatchObject{{OID: newOID, Size: int64(len(newContent))}}, + }) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{ + OID: newOID, + Size: int64(len(newContent)), + Authenticated: true, + Actions: map[string]Action{ + "upload": { + Href: endpoint.Href + "/" + newOID, + Header: map[string]string{ + "Authorization": "Basic Zm9vOmJhcg==", + "Content-Type": "application/octet-stream", + "x-amz-checksum-sha256": "/jJgjJ71ts9+P5RkgCU/928k9OwGePPQ8H+YRMv/lgE=", + }, + }, + "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + }, + }}, resp.Objects) + + resp, err = s.Batch(ctx, discardLogger, "repo1", endpoint, download) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{ + OID: oid, + Size: size, + Authenticated: true, + Actions: map[string]Action{"download": {Href: endpoint.Href + "/" + oid, Header: endpoint.Header}}, + }}, resp.Objects) + }) + + t.Run("rejects invalid requests", func(t *testing.T) { + s := newServiceWithObject(t, false) + for _, req := range []BatchRequest{ + {Operation: "delete"}, + {Operation: OperationDownload, Objects: []BatchObject{{OID: "not-an-oid"}}}, + } { + _, err := s.Batch(ctx, discardLogger, "repo1", endpoint, req) + requireLFSError(t, err, http.StatusBadRequest) + } + _, err := s.Batch(ctx, discardLogger, "repo1", endpoint, BatchRequest{Operation: OperationUpload, Transfers: []string{"tus"}}) + requireLFSError(t, err, http.StatusUnprocessableEntity) + }) +} + +func TestService_Presign(t *testing.T) { + ctx := t.Context() + content := "presigned content" + oid := sha256Hex(content) + size := int64(len(content)) + + // uploadToRedirect sends the content to the presigned URL like a client. + uploadToRedirect := func(t *testing.T, redirect *Redirect) { + t.Helper() + req, err := http.NewRequestWithContext(ctx, http.MethodPut, redirect.URL, strings.NewReader(content)) + require.NoError(t, err) + for k, v := range redirect.Header { + req.Header.Set(k, v) + } + req.ContentLength = size + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + + for _, crossRepoDedup := range []bool{false, true} { + s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t), CrossRepoDedup: crossRepoDedup} + + result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, nil) + require.NoError(t, err) + require.NotNil(t, result.Redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size}), http.StatusUnprocessableEntity) + _, err = s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) + + uploadToRedirect(t, result.Redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + require.NotNil(t, download.Redirect) + assert.Contains(t, download.Redirect.URL, oid) + + result, err = s.Upload(ctx, discardLogger, "repo1", oid, size, nil) + require.NoError(t, err) + assert.Equal(t, &UploadResult{Existed: true}, result) + + // Without cross-repo dedup, another repository must upload the content + // itself, which the storage checks against the OID. + result, err = s.Upload(ctx, discardLogger, "repo2", oid, size, nil) + require.NoError(t, err) + assert.Equal(t, crossRepoDedup, result.Redirect == nil, "crossRepoDedup=%v", crossRepoDedup) + } +} + +// failingStore fails every lookup. +type failingStore struct{ *memStore } + +func (failingStore) GetRepoObjects(context.Context, string, []string) (map[string]Object, error) { + return nil, errors.New("store is down") +} + +func TestService_Tracing(t *testing.T) { + ctx := t.Context() + content := "traced content" + oid := sha256Hex(content) + size := int64(len(content)) + + newService := func(store Store) (*Service, *tracetest.SpanRecorder) { + recorder := tracetest.NewSpanRecorder() + return &Service{ + Store: store, + Storage: newFilesystemBackend(t, "fs", "file://"), + TracerProvider: sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)), + }, recorder + } + spansByName := func(recorder *tracetest.SpanRecorder) map[string]sdktrace.ReadOnlySpan { + spans := map[string]sdktrace.ReadOnlySpan{} + for _, span := range recorder.Ended() { + spans[span.Name()] = span + } + return spans + } + + t.Run("storage calls are children of the operation", func(t *testing.T) { + s, recorder := newService(newMemStore()) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + spans := spansByName(recorder) + require.Contains(t, spans, "lfs.Upload") + require.Contains(t, spans, "storage.Put") + upload, put := spans["lfs.Upload"], spans["storage.Put"] + assert.Equal(t, upload.SpanContext().SpanID(), put.Parent().SpanID()) + assert.Contains(t, upload.Attributes(), attribute.String("lfs.repo", "repo1")) + assert.Contains(t, upload.Attributes(), attribute.String("lfs.oid", oid)) + assert.Contains(t, put.Attributes(), attribute.String("lfs.storage.name", "fs")) + assert.Contains(t, put.Attributes(), attribute.String("lfs.storage.type", "filesystem")) + }) + + t.Run("client errors do not fail the span", func(t *testing.T) { + s, recorder := newService(newMemStore()) + _, err := s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) + + download := spansByName(recorder)["lfs.Download"] + require.NotNil(t, download) + assert.Equal(t, codes.Unset, download.Status().Code) + assert.Contains(t, download.Attributes(), attribute.Int("lfs.error.code", http.StatusNotFound)) + }) + + t.Run("internal errors fail the span", func(t *testing.T) { + s, recorder := newService(failingStore{newMemStore()}) + _, err := s.Download(ctx, "repo1", oid) + require.Error(t, err) + + download := spansByName(recorder)["lfs.Download"] + require.NotNil(t, download) + assert.Equal(t, codes.Error, download.Status().Code) + assert.Contains(t, download.Status().Description, "store is down") + }) +} diff --git a/internal/storage/filesystem.go b/lfs/storage/filesystem.go similarity index 96% rename from internal/storage/filesystem.go rename to lfs/storage/filesystem.go index 430efc8..a86e1b6 100644 --- a/internal/storage/filesystem.go +++ b/lfs/storage/filesystem.go @@ -48,6 +48,11 @@ func (b *FilesystemBackend) storagePath(oid string) string { return filepath.Join(b.root, oid[0:2], oid[2:4], oid) } +// URI returns the file URI of the object under the storage root. +func (b *FilesystemBackend) URI(oid string) string { + return b.uriFromPath(b.storagePath(oid)) +} + func (b *FilesystemBackend) Put(ctx context.Context, oid string, r io.Reader) (string, error) { if len(oid) < 4 { return "", errors.Newf("invalid oid %q", oid) diff --git a/internal/storage/filesystem_test.go b/lfs/storage/filesystem_test.go similarity index 100% rename from internal/storage/filesystem_test.go rename to lfs/storage/filesystem_test.go diff --git a/internal/storage/s3_presign.go b/lfs/storage/s3_presign.go similarity index 100% rename from internal/storage/s3_presign.go rename to lfs/storage/s3_presign.go diff --git a/internal/storage/s3_presign_test.go b/lfs/storage/s3_presign_test.go similarity index 100% rename from internal/storage/s3_presign_test.go rename to lfs/storage/s3_presign_test.go diff --git a/internal/storage/storage.go b/lfs/storage/storage.go similarity index 97% rename from internal/storage/storage.go rename to lfs/storage/storage.go index bdc8bf3..6bb95ae 100644 --- a/internal/storage/storage.go +++ b/lfs/storage/storage.go @@ -20,6 +20,10 @@ type Backend interface { Type() Type // Scheme returns the URI scheme prefix (including "://") that this backend owns. Scheme() string + // URI returns the canonical, non-expiring location of the object. Pure + // function of the OID and the backend's static config (e.g. bucket name), + // safe to compute without I/O. + URI(oid string) string // Delete removes the object at uri. Returns nil if already absent. Delete(ctx context.Context, uri string) error } @@ -40,6 +44,7 @@ type Proxier interface { // Put stores the bytes from the reader under the given OID and returns a URI // that uniquely identifies the stored object, including the backend scheme. + // It may return without reading r when the object is already stored. Put(ctx context.Context, oid string, r io.Reader) (uri string, err error) // Open returns a reader for the object at URI. It returns ErrNotFound when no // matching object exists. The caller MUST close the reader. @@ -51,10 +56,6 @@ type Proxier interface { type Presigner interface { Backend - // URI returns the canonical, non-expiring location of the object. Pure - // function of the OID and the backend's static config (e.g. bucket name), - // safe to compute without I/O. - URI(oid string) string // PresignPut returns a short-lived URL the client PUTs to, along with the // headers the client MUST send verbatim. PresignPut(ctx context.Context, oid string, size int64) (url string, headers map[string]string, err error) diff --git a/lfs/tracing.go b/lfs/tracing.go new file mode 100644 index 0000000..e670e10 --- /dev/null +++ b/lfs/tracing.go @@ -0,0 +1,48 @@ +package lfs + +import ( + "context" + + "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +const tracerName = "unknwon.dev/git-lfs-server/lfs" + +func (s *Service) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + tp := s.TracerProvider + if tp == nil { + tp = otel.GetTracerProvider() + } + return tp.Tracer(tracerName).Start(ctx, name, trace.WithAttributes(attrs...)) +} + +// startStorageSpan starts the span of a call to backend, so that every backend +// plugged into the Service is traced the same way. +func (s *Service) startStorageSpan(ctx context.Context, method string, backend storage.Backend) (context.Context, trace.Span) { + return s.startSpan(ctx, "storage."+method, + attribute.String("lfs.storage.name", backend.Name()), + attribute.String("lfs.storage.type", string(backend.Type())), + ) +} + +// endSpan ends span with the outcome of err. A client error is the expected +// answer to a bad request rather than a failure of the operation, so it is +// recorded as an attribute and leaves the span status unset. +func endSpan(span trace.Span, err error) { + if err != nil { + var lfsErr *Error + if errors.As(err, &lfsErr) { + span.SetAttributes(attribute.Int("lfs.error.code", lfsErr.Code)) + } else { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + } + span.End() +}