From 3f0bed48f587bb29f2325cc2d658d0b26db3aeeb Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Wed, 11 Nov 2026 14:41:04 +0000 Subject: [PATCH 1/5] lfs: extract the Batch API into an importable module Move the transfer logic of lfsd into a separate Go module, unknwon.dev/git-lfs-server/lfs, so that other Git servers can serve LFS with the same storage backends. The module has an lfs.Service that answers batch, upload, download, and verify requests on top of an lfs.Store for metadata and a storage.Backend for content, and leaves routing, authentication, and response encoding to the caller. lfsd now builds one lfs.Service per forge host and keeps its HTTP responses, except that a proxied upload of an already stored object must send matching content before the object is linked to another repository. Previously, any body was accepted in that case, so a writer of one repository could gain read access to any object of another repository by its OID and size. --- .github/workflows/go.yml | 17 +- .github/workflows/integration.yml | 2 + .moon/workspace.yml | 1 + Dockerfile | 1 + README.md | 6 + cmd/lfsd/config.go | 2 +- cmd/lfsd/lfs.go | 505 +++--------------- cmd/lfsd/main.go | 20 +- go.mod | 13 +- go.sum | 10 - internal/database/objects.go | 84 +-- internal/janitor/janitor.go | 2 +- lfs/go.mod | 40 ++ lfs/go.sum | 114 ++++ .../internal}/iox/exact_size_reader.go | 0 .../internal}/iox/exact_size_reader_test.go | 0 .../internal}/iox/sha256_reader.go | 0 .../internal}/iox/sha256_reader_test.go | 0 lfs/lfs.go | 123 +++++ lfs/moon.yml | 45 ++ lfs/service.go | 379 +++++++++++++ lfs/service_test.go | 373 +++++++++++++ {internal => lfs}/storage/filesystem.go | 5 + {internal => lfs}/storage/filesystem_test.go | 0 {internal => lfs}/storage/s3_presign.go | 0 {internal => lfs}/storage/s3_presign_test.go | 0 {internal => lfs}/storage/storage.go | 9 +- 27 files changed, 1241 insertions(+), 510 deletions(-) create mode 100644 lfs/go.mod create mode 100644 lfs/go.sum rename {internal => lfs/internal}/iox/exact_size_reader.go (100%) rename {internal => lfs/internal}/iox/exact_size_reader_test.go (100%) rename {internal => lfs/internal}/iox/sha256_reader.go (100%) rename {internal => lfs/internal}/iox/sha256_reader_test.go (100%) create mode 100644 lfs/lfs.go create mode 100644 lfs/moon.yml create mode 100644 lfs/service.go create mode 100644 lfs/service_test.go rename {internal => lfs}/storage/filesystem.go (96%) rename {internal => lfs}/storage/filesystem_test.go (100%) rename {internal => lfs}/storage/s3_presign.go (100%) rename {internal => lfs}/storage/s3_presign_test.go (100%) rename {internal => lfs}/storage/storage.go (97%) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 75fe764..91ed31d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,6 +7,8 @@ on: - '**.go' - 'go.mod' - 'go.sum' + - 'lfs/go.mod' + - 'lfs/go.sum' - '.golangci.yml' - '.github/workflows/go.yml' pull_request: @@ -14,6 +16,8 @@ on: - '**.go' - 'go.mod' - 'go.sum' + - 'lfs/go.mod' + - 'lfs/go.sum' - '.golangci.yml' - '.github/workflows/go.yml' env: @@ -38,6 +42,7 @@ jobs: shell: bash run: | go mod tidy + (cd lfs && go mod tidy) STATUS=$(git status --porcelain) if [ ! -z "$STATUS" ]; then echo "Unstaged files:" @@ -50,6 +55,12 @@ jobs: with: version: latest args: --timeout=30m + - name: Run golangci-lint (lfs module) + uses: golangci/golangci-lint-action@9fae48acfc02a90574d7c304a1758ef9895495fa # v7.0.1 + with: + version: latest + working-directory: lfs + args: --timeout=30m test: name: Test @@ -68,6 +79,7 @@ jobs: - name: Run tests with coverage run: | go test -shuffle=on -v -race -coverprofile=coverage -covermode=atomic -json ./... > test-report.json + (cd lfs && go test -shuffle=on -v -race -coverprofile=coverage -covermode=atomic -json ./...) >> test-report.json go install github.com/mfridman/tparse@latest tparse -all -file=test-report.json @@ -88,4 +100,7 @@ jobs: with: go-version: ${{ matrix.go-version }} - name: Run tests with coverage - run: go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... + shell: bash + run: | + go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... + cd lfs && go test -shuffle=on -v -coverprofile=coverage -covermode=atomic ./... diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3f8a221..4ff901a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,6 +9,7 @@ on: - 'integration-tests/**' - 'go.mod' - 'go.sum' + - 'lfs/**' - '.github/workflows/integration.yml' pull_request: paths: @@ -17,6 +18,7 @@ on: - 'integration-tests/**' - 'go.mod' - 'go.sum' + - 'lfs/**' - '.github/workflows/integration.yml' permissions: diff --git a/.moon/workspace.yml b/.moon/workspace.yml index e16843f..d78ebb2 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -2,3 +2,4 @@ $schema: 'https://moonrepo.dev/schemas/workspace.json' projects: - '.' + - 'lfs' diff --git a/Dockerfile b/Dockerfile index 1fb9a38..606f072 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.27.4-alpine3.24@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS builder WORKDIR /src COPY go.mod go.sum ./ +COPY lfs/go.mod lfs/go.sum ./lfs/ RUN go mod download COPY . . ARG BUILD_DATE diff --git a/README.md b/README.md index e833b39..ac1d589 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,12 @@ REPO_ALLOWLIST = myorg/**, otherorg/specific-repo The allowlist is enforced before the forge token check, so a request for a disallowed repo is rejected without a round-trip to the forge. +## Using as a library + +The `unknwon.dev/git-lfs-server/lfs` module implements the Batch API and the basic transfer adapter for other Git servers to embed. It is a separate Go module so that importers do not pull in the dependencies of lfsd (PostgreSQL, Flamego, forge clients). + +An `lfs.Service` combines an `lfs.Store`, which records objects and the repositories they are linked to, with a `storage.Backend`, which holds the content. The caller keeps its own routing, authentication, authorization, and response encoding, identifies repositories with any string it likes, and calls `Batch`, `Upload`, `Download`, and `Verify` from its handlers. A `storage.Proxier` backend (e.g., `storage.FilesystemBackend`) streams content through the caller, and a `storage.Presigner` backend (e.g., `storage.S3PresignBackend`) redirects clients to the storage. `PreviousStorages` keeps objects readable after the backend for new uploads changes. + ## Local development This project uses [moon](https://moonrepo.dev/) as its task runner. Install it with Homebrew: diff --git a/cmd/lfsd/config.go b/cmd/lfsd/config.go index 352ab2c..9a22d35 100644 --- a/cmd/lfsd/config.go +++ b/cmd/lfsd/config.go @@ -10,7 +10,7 @@ import ( "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/embed" "unknwon.dev/git-lfs-server/internal/forge" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/lfs/storage" ) // Host is the host of a forge, e.g., "github.com". diff --git a/cmd/lfsd/lfs.go b/cmd/lfsd/lfs.go index f35aa73..c93842a 100644 --- a/cmd/lfsd/lfs.go +++ b/cmd/lfsd/lfs.go @@ -1,103 +1,63 @@ package main import ( - "encoding/base64" - "encoding/hex" + "context" "encoding/json" "fmt" "io" - "maps" "net/http" - "regexp" - "slices" "strconv" "github.com/cockroachdb/errors" "github.com/flamego/flamego" - "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/forge" - "unknwon.dev/git-lfs-server/internal/iox" "unknwon.dev/git-lfs-server/internal/logx" - "unknwon.dev/git-lfs-server/internal/ptrx" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/lfs" ) -// oidPattern matches the SHA-256 hex digest format used as the LFS object ID: -// exactly 64 lowercase hex characters. -var oidPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) - const ( - operationUpload = "upload" - operationDownload = "download" - transferBasic = "basic" - mediaTypeLFS = "application/vnd.git-lfs+json" + mediaTypeLFS = "application/vnd.git-lfs+json" // maxJSONRequestBytes caps JSON request bodies (batch, verify) to prevent // authenticated callers from exhausting memory with oversized payloads. maxJSONRequestBytes = 10 << 20 ) -type batchRequest struct { - Operation string `json:"operation"` - Transfers []string `json:"transfers,omitempty"` - Objects []batchObject `json:"objects"` -} - -type batchObject struct { - OID string `json:"oid"` - Size int64 `json:"size"` -} - -type batchResponse struct { - Transfer string `json:"transfer"` - Objects []batchResponseObject `json:"objects"` -} - -type batchResponseObject struct { - OID string `json:"oid"` - Size int64 `json:"size"` - Authenticated bool `json:"authenticated,omitempty"` - Actions map[string]batchAction `json:"actions,omitempty"` - Error *batchObjectError `json:"error,omitempty"` -} - -type batchAction struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` -} - -type batchObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -type verifyRequest struct { - OID string `json:"oid"` - Size int64 `json:"size"` +// mapService maps the *lfs.Service of the forge host to the request context. +func mapService(services map[string]*lfs.Service) flamego.Handler { + return func(c flamego.Context, logger *logx.Logger) { + service, ok := services[hostFromContext(c)] + if !ok { + logger.ErrorContext(c.Request().Context(), "No storage backend configured for host") + writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + return + } + c.Map(service) + } } -func serveBatch(db *database.DB, externalURL string, maxObjectSize int64) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { +func serveBatch(externalURL string) flamego.Handler { + return func(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { logger = logger.Scoped("batch") ctx := c.Request().Context() body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) defer func() { _ = body.Close() }() - var req batchRequest + var req lfs.BatchRequest if err := json.NewDecoder(body).Decode(&req); err != nil { writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) return } switch req.Operation { - case operationUpload: + case lfs.OperationUpload: if perm != forge.PermissionWrite { writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") return } - case operationDownload: + case lfs.OperationDownload: if perm != forge.PermissionRead && perm != forge.PermissionWrite { writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") return @@ -107,431 +67,132 @@ func serveBatch(db *database.DB, externalURL string, maxObjectSize int64) flameg return } - if len(req.Transfers) > 0 && !slices.Contains(req.Transfers, transferBasic) { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, `only "basic" transfer is supported`) - return - } - - for _, o := range req.Objects { - if !oidPattern.MatchString(o.OID) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid oid %q", o.OID)) - return - } - } - repoName := repoNameFromContext(c) - - var authHeader map[string]string - if authz := c.Request().Header.Get("Authorization"); authz != "" { - authHeader = map[string]string{"Authorization": authz} + endpoint := lfs.Endpoint{ + Href: fmt.Sprintf("%s/%s/info/lfs/objects", externalURL, repoName), + // Every request passes the authorize middleware with Basic Auth, so + // forwarding the header authenticates the actions. + Authenticated: true, } - - oids := make([]string, len(req.Objects)) - for i, o := range req.Objects { - oids[i] = o.OID + if authz := c.Request().Header.Get("Authorization"); authz != "" { + endpoint.Header = map[string]string{"Authorization": authz} } - var ( - existing map[string]database.Object - err error - ) - switch req.Operation { - case operationDownload: - existing, err = db.GetRepoObjectsByOIDs(ctx, repoName, oids) - case operationUpload: - existing, err = db.GetObjectsByOIDs(ctx, oids) - } + resp, err := service.Batch(ctx, logger.Logger, repoName, endpoint, req) if err != nil { - logger.ErrorContext(ctx, "Failed to look up objects", "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, "failed to process batch") + writeServiceError(ctx, logger, c.ResponseWriter(), err, "Failed to process batch", "failed to process batch") return } - out := batchResponse{ - Transfer: transferBasic, - Objects: make([]batchResponseObject, 0, len(req.Objects)), - } - for _, in := range req.Objects { - obj, ok := existing[in.OID] - respObj := batchResponseObject{OID: in.OID, Size: in.Size, Authenticated: true} - - switch req.Operation { - case operationDownload: - switch { - case !ok: - respObj.Error = &batchObjectError{Code: http.StatusNotFound, Message: "object does not exist"} - case ptrx.Deref(obj.Size, -1) != in.Size: - respObj.Error = &batchObjectError{Code: http.StatusUnprocessableEntity, Message: "OID exists with different size"} - default: - respObj.Actions = map[string]batchAction{ - "download": {Href: objectHref(externalURL, repoName, in.OID), Header: authHeader}, - } - } - case operationUpload: - switch { - case maxObjectSize > 0 && in.Size > maxObjectSize: - respObj.Error = &batchObjectError{ - Code: http.StatusRequestEntityTooLarge, - Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, maxObjectSize), - } - case ok && ptrx.Deref(obj.Size, -1) == in.Size: - // Cross-repo dedup: object is verified globally, ensure - // the calling repo is linked so future downloads succeed. - // COALESCE in LinkObject's upsert preserves the existing - // object_uri, so the placeholder is never written. - if _, err := db.LinkObject(ctx, repoName, in.OID, in.Size, "ignored"); err != nil { - logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) - respObj.Error = &batchObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} - } - case ok && ptrx.Deref(obj.Size, -1) != in.Size: - respObj.Error = &batchObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} - default: - rawOID, err := hex.DecodeString(in.OID) - if err != nil { - respObj.Error = &batchObjectError{Code: http.StatusBadRequest, Message: fmt.Sprintf("invalid oid %q", in.OID)} - break - } - uploadHeader := make(map[string]string, len(authHeader)+1) - maps.Copy(uploadHeader, authHeader) - uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) - respObj.Actions = map[string]batchAction{ - "upload": {Href: objectHref(externalURL, repoName, in.OID), Header: uploadHeader}, - "verify": {Href: verifyHref(externalURL, repoName), Header: authHeader}, - } - } - } - out.Objects = append(out.Objects, respObj) - } - c.ResponseWriter().Header().Set("Content-Type", mediaTypeLFS) - if err := json.NewEncoder(c.ResponseWriter()).Encode(out); err != nil { + if err := json.NewEncoder(c.ResponseWriter()).Encode(resp); err != nil { logger.ErrorContext(ctx, "Failed to encode batch response", "error", err) } } } -func serveUpload(db *database.DB, storages map[string]storage.Backend, maxObjectSize int64) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("upload") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") - return - } - - oid := c.Param("oid") - if !oidPattern.MatchString(oid) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - size := c.Request().ContentLength - if size < 0 { - writeLFSError(c.ResponseWriter(), http.StatusLengthRequired, "Content-Length is required") - return - } - if maxObjectSize > 0 && size > maxObjectSize { - writeLFSError(c.ResponseWriter(), http.StatusRequestEntityTooLarge, fmt.Sprintf("object size %d exceeds limit %d", size, maxObjectSize)) - return - } - - backend, ok := storages[hostFromContext(c)] - if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } +func serveUpload(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { + logger = logger.Scoped("upload") + ctx := c.Request().Context() - switch b := backend.(type) { - case storage.Proxier: - serveUploadProxy(c, logger, db, b, oid, size) - case storage.Presigner: - serveUploadPresign(c, logger, db, b, oid, size) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } + if perm != forge.PermissionWrite { + writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") + return } -} - -func serveUploadProxy(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Proxier, oid string, size int64) { - ctx := c.Request().Context() body := c.Request().Body().ReadCloser() defer func() { _ = body.Close() }() - reader := iox.NewSHA256Reader(iox.NewExactSizeReader(body, size), oid) - - uri, err := backend.Put(ctx, oid, reader) + oid := c.Param("oid") + result, err := service.Upload(ctx, logger.Logger, repoNameFromContext(c), oid, c.Request().ContentLength, body) if err != nil { - var sizeErr *iox.SizeMismatchError - var hashErr *iox.SHA256MismatchError - switch { - case errors.As(err, &sizeErr): - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, sizeErr.Error()) - case errors.As(err, &hashErr): - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, hashErr.Error()) - default: - logger.ErrorContext(ctx, "Failed to store object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } + writeServiceError(ctx, logger.With("oid", oid), c.ResponseWriter(), err, "Failed to upload object", http.StatusText(http.StatusInternalServerError)) return } - repoName := repoNameFromContext(c) - storedURI, err := db.LinkObject(ctx, repoName, oid, size, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to link object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if storedURI != uri { - if delErr := backend.Delete(ctx, uri); delErr != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", delErr) + w := c.ResponseWriter() + switch { + case result.Redirect != nil: + for k, v := range result.Redirect.Header { + w.Header().Set(k, v) } + w.Header().Set("Location", result.Redirect.URL) + w.WriteHeader(http.StatusTemporaryRedirect) + case result.Existed: + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNoContent) } - - c.ResponseWriter().WriteHeader(http.StatusNoContent) } -func serveUploadPresign(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Presigner, oid string, size int64) { +func serveDownload(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { + logger = logger.Scoped("download") ctx := c.Request().Context() - existing, err := db.GetObjectByOID(ctx, oid) - if err != nil && !errors.Is(err, database.ErrObjectNotFound) { - logger.ErrorContext(ctx, "Failed to look up object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if existing != nil && existing.VerifiedAt != nil { - c.ResponseWriter().WriteHeader(http.StatusOK) + if perm != forge.PermissionRead && perm != forge.PermissionWrite { + writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") return } - if err = db.InsertPendingObject(ctx, oid); err != nil { - logger.ErrorContext(ctx, "Failed to insert pending object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - url, headers, err := backend.PresignPut(ctx, oid, size) + oid := c.Param("oid") + result, err := service.Download(ctx, repoNameFromContext(c), oid) if err != nil { - logger.ErrorContext(ctx, "Failed to presign upload", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + writeServiceError(ctx, logger.With("oid", oid), c.ResponseWriter(), err, "Failed to download object", http.StatusText(http.StatusInternalServerError)) return } w := c.ResponseWriter() - for k, v := range headers { - w.Header().Set(k, v) - } - w.Header().Set("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) -} - -func serveDownload(db *database.DB, storages map[string]storage.Backend) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("download") - ctx := c.Request().Context() - - if perm != forge.PermissionRead && perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") - return - } - - oid := c.Param("oid") - if !oidPattern.MatchString(oid) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - backend, ok := storages[hostFromContext(c)] - if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - repoName := repoNameFromContext(c) - object, err := db.GetRepoObjectByOID(ctx, repoName, oid) - if err != nil { - if errors.Is(err, database.ErrObjectNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusNotFound, "object does not exist") - return - } - logger.ErrorContext(ctx, "Failed to look up object", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - // GetRepoObjectByOID filters verified rows, so Size and ObjectURI are - // guaranteed non-nil here by the schema CHECK. - objectURI := ptrx.Deref(object.ObjectURI, "") - objectSize := ptrx.Deref(object.Size, -1) - if objectURI == "" || objectSize < 0 { - logger.ErrorContext(ctx, "Verified object has nil size or uri (schema violation)", "oid", oid) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - switch b := backend.(type) { - case storage.Proxier: - serveDownloadProxy(c, logger, b, oid, objectURI, objectSize) - case storage.Presigner: - serveDownloadPresign(c, logger, b, oid, objectURI) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - } -} - -func serveDownloadProxy(c flamego.Context, logger *logx.Logger, backend storage.Proxier, oid, uri string, size int64) { - ctx := c.Request().Context() - - rc, err := backend.Open(ctx, uri) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - logger.ErrorContext(ctx, "DB row exists but storage object is missing", "oid", oid, "uri", uri) - } else { - logger.ErrorContext(ctx, "Failed to open object", "oid", oid, "error", err) - } - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + if result.Redirect != nil { + w.Header().Set("Location", result.Redirect.URL) + w.WriteHeader(http.StatusTemporaryRedirect) return } - defer func() { _ = rc.Close() }() - w := c.ResponseWriter() + defer func() { _ = result.Content.Close() }() w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) w.WriteHeader(http.StatusOK) - if _, err = io.Copy(w, rc); err != nil { + if _, err = io.Copy(w, result.Content); err != nil { logger.ErrorContext(ctx, "Failed to stream object", "oid", oid, "error", err) } } -func serveDownloadPresign(c flamego.Context, logger *logx.Logger, backend storage.Presigner, oid, uri string) { +func serveVerify(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { + logger = logger.Scoped("verify") ctx := c.Request().Context() - url, err := backend.PresignGet(ctx, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to presign download", "oid", oid, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + if perm != forge.PermissionWrite { + writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for verify") return } - w := c.ResponseWriter() - w.Header().Set("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) -} - -func serveVerify(db *database.DB, storages map[string]storage.Backend) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { - logger = logger.Scoped("verify") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for verify") - return - } - - body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) - defer func() { _ = body.Close() }() - - var req verifyRequest - if err := json.NewDecoder(body).Decode(&req); err != nil { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) - return - } - if !oidPattern.MatchString(req.OID) { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "invalid oid") - return - } - - backend, ok := storages[hostFromContext(c)] - if !ok { - logger.ErrorContext(ctx, "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - - switch b := backend.(type) { - case storage.Proxier: - serveVerifyProxy(c, logger, db, req) - case storage.Presigner: - serveVerifyPresign(c, logger, db, b, req) - default: - logger.ErrorContext(ctx, "Storage backend does not implement a known transfer mode", "type", fmt.Sprintf("%T", backend)) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - } -} - -func serveVerifyProxy(c flamego.Context, logger *logx.Logger, db *database.DB, req verifyRequest) { - ctx := c.Request().Context() + body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) + defer func() { _ = body.Close() }() - repoName := repoNameFromContext(c) - object, err := db.GetRepoObjectByOID(ctx, repoName, req.OID) - if err != nil { - if errors.Is(err, database.ErrObjectNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusNotFound, "object does not exist") - return - } - logger.ErrorContext(ctx, "Failed to look up object", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + var req lfs.VerifyRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) return } - if ptrx.Deref(object.Size, -1) != req.Size { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "size mismatch") + + if err := service.Verify(ctx, logger.Logger, repoNameFromContext(c), req); err != nil { + writeServiceError(ctx, logger.With("oid", req.OID), c.ResponseWriter(), err, "Failed to verify object", http.StatusText(http.StatusInternalServerError)) return } - c.ResponseWriter().WriteHeader(http.StatusNoContent) } -func serveVerifyPresign(c flamego.Context, logger *logx.Logger, db *database.DB, backend storage.Presigner, req verifyRequest) { - ctx := c.Request().Context() - - uri := backend.URI(req.OID) - headSize, err := backend.Head(ctx, uri) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "object does not exist in storage") - return - } - logger.ErrorContext(ctx, "Failed to head object", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - // R2's signed Content-Length should make this impossible, but treat any - // mismatch as a hard error rather than trusting either side. - if headSize != req.Size { - writeLFSError(c.ResponseWriter(), http.StatusUnprocessableEntity, "size mismatch") +// writeServiceError responds with the client error carried by err as an +// *lfs.Error. Any other error is internal, so it is logged with logMessage and +// the response only carries internalMessage. +func writeServiceError(ctx context.Context, logger *logx.Logger, w http.ResponseWriter, err error, logMessage, internalMessage string) { + var lfsErr *lfs.Error + if errors.As(err, &lfsErr) { + writeLFSError(w, lfsErr.Code, lfsErr.Message) return } - - repoName := repoNameFromContext(c) - storedURI, err := db.VerifyObject(ctx, repoName, req.OID, req.Size, uri) - if err != nil { - logger.ErrorContext(ctx, "Failed to mark object verified", "oid", req.OID, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - if storedURI != uri { - if delErr := backend.Delete(ctx, uri); delErr != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", delErr) - } - } - - c.ResponseWriter().WriteHeader(http.StatusNoContent) -} - -func objectHref(externalURL, repoName, oid string) string { - return fmt.Sprintf("%s/%s/info/lfs/objects/%s", externalURL, repoName, oid) -} - -func verifyHref(externalURL, repoName string) string { - return fmt.Sprintf("%s/%s/info/lfs/objects/verify", externalURL, repoName) + logger.ErrorContext(ctx, logMessage, "error", err) + writeLFSError(w, http.StatusInternalServerError, internalMessage) } // writeLFSError writes an error response in the format required by the LFS diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index 30e6048..b071bd9 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -18,6 +18,7 @@ import ( "unknwon.dev/git-lfs-server/internal/janitor" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/internal/strx" + "unknwon.dev/git-lfs-server/lfs" ) // Set via ldflags. @@ -60,12 +61,21 @@ func main() { f.Get("/healthz", serveHealthz(db)) + services := make(map[string]*lfs.Service, len(config.Storages)) + for host, backend := range config.Storages { + services[host] = &lfs.Service{ + Store: db, + Storage: backend, + MaxObjectSize: config.Server.MaxObjectSize, + CrossRepoDedup: true, + } + } f.Group("/{host}/{**}/info/lfs/objects", func() { - f.Post("/batch", serveBatch(db, config.Server.ExternalURL, config.Server.MaxObjectSize)) - f.Get("/{oid}", serveDownload(db, config.Storages)) - f.Put("/{oid}", serveUpload(db, config.Storages, config.Server.MaxObjectSize)) - f.Post("/verify", serveVerify(db, config.Storages)) - }, authorize(config.Forges)) + f.Post("/batch", serveBatch(config.Server.ExternalURL)) + f.Get("/{oid}", serveDownload) + f.Put("/{oid}", serveUpload) + f.Post("/verify", serveVerify) + }, authorize(config.Forges), mapService(services)) ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() diff --git a/go.mod b/go.mod index 80c2e1f..bbb2b84 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,9 @@ go 1.27 require ( charm.land/lipgloss/v2 v2.0.3 charm.land/log/v2 v2.0.0 - github.com/aws/aws-sdk-go-v2 v1.41.7 - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 - github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 github.com/flamego/cache v1.5.1 github.com/flamego/flamego v1.9.7 - github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/lib/pq v1.12.3 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 @@ -20,12 +16,15 @@ require ( gopkg.in/ini.v1 v1.67.2 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 + unknwon.dev/git-lfs-server/lfs v0.0.0-00010101000000-000000000000 ) require ( bitbucket.org/creachadair/shell v0.0.7 // indirect github.com/alecthomas/participle/v2 v2.1.4 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect @@ -33,6 +32,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect @@ -75,15 +75,14 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/otel v1.11.0 // indirect go.opentelemetry.io/otel/trace v1.11.0 // indirect - go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect - golang.org/x/tools v0.48.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace unknwon.dev/git-lfs-server/lfs => ./lfs diff --git a/go.sum b/go.sum index 0cbb3ab..62a7ab2 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,6 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= @@ -38,8 +36,6 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= -github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= @@ -165,8 +161,6 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA= -github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= -github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -184,8 +178,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4= go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= @@ -238,8 +230,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/database/objects.go b/internal/database/objects.go index 84200b3..b376fb1 100644 --- a/internal/database/objects.go +++ b/internal/database/objects.go @@ -10,8 +10,11 @@ import ( "gorm.io/gorm" "unknwon.dev/git-lfs-server/internal/ptrx" + "unknwon.dev/git-lfs-server/lfs" ) +var _ lfs.Store = (*DB)(nil) + type Object struct { ID int64 `gorm:"column:id;primaryKey"` OID string `gorm:"column:oid;uniqueIndex;not null"` @@ -24,48 +27,6 @@ type Object struct { var ErrObjectNotFound = errors.New("database: object not found") -// GetRepoObjectByOID fetches the verified Object linked to repoName by oid. -// Pending (unverified) rows are excluded so unverified objects can never serve -// as download sources. Returns ErrObjectNotFound when no row matches. -func (d *DB) GetRepoObjectByOID(ctx context.Context, repoName, oid string) (*Object, error) { - var object Object - err := d.db.WithContext(ctx).Raw(` -SELECT objects.* -FROM objects -JOIN repo_objects ON repo_objects.object_id = objects.id -WHERE - repo_objects.repo_name = @repoName -AND objects.oid = @oid -AND objects.verified_at IS NOT NULL`, - map[string]any{"repoName": repoName, "oid": oid}, - ).Scan(&object).Error - if err != nil { - return nil, errors.Wrap(err, "select repo object by oid") - } - if object.ID == 0 { - return nil, errors.WithStack(ErrObjectNotFound) - } - return &object, nil -} - -// GetObjectByOID fetches an Object by oid (verified or pending) without any -// repo filtering. Used by the upload handler for the dedup short-circuit and -// to detect pending rows. Returns ErrObjectNotFound when no row matches. -func (d *DB) GetObjectByOID(ctx context.Context, oid string) (*Object, error) { - var object Object - err := d.db.WithContext(ctx).Raw( - `SELECT * FROM objects WHERE oid = @oid LIMIT 1`, - map[string]any{"oid": oid}, - ).Scan(&object).Error - if err != nil { - return nil, errors.Wrap(err, "select object by oid") - } - if object.ID == 0 { - return nil, errors.WithStack(ErrObjectNotFound) - } - return &object, nil -} - // LinkObject is the storage.Proxier verify-equivalent path: bytes are already // stored, write a fully-populated (verified) objects row and link it to // repoName in one transaction. Returns the canonical object_uri (which may @@ -195,11 +156,11 @@ WHERE id = @id`, return storedURI, nil } -// GetObjectsByOIDs fetches verified Object rows for the given OIDs. Results are -// keyed by OID, silently dropping missing OIDs. -func (d *DB) GetObjectsByOIDs(ctx context.Context, oids []string) (map[string]Object, error) { +// GetObjects fetches verified objects for the given OIDs. Results are keyed by +// OID, silently dropping missing OIDs. +func (d *DB) GetObjects(ctx context.Context, oids []string) (map[string]lfs.Object, error) { if len(oids) == 0 { - return map[string]Object{}, nil + return map[string]lfs.Object{}, nil } var rows []Object err := d.db.WithContext(ctx).Raw(` @@ -210,19 +171,14 @@ WHERE oid = ANY(@oids) AND verified_at IS NOT NULL`, if err != nil { return nil, errors.Wrap(err, "select objects by OIDs") } - out := make(map[string]Object, len(rows)) - for _, r := range rows { - out[r.OID] = r - } - return out, nil + return verifiedObjectsByOID(rows), nil } -// GetRepoObjectsByOIDs fetches verified Object rows for the given OIDs that are -// linked to the repository. Results are keyed by OID, silently dropping missing -// OIDs. -func (d *DB) GetRepoObjectsByOIDs(ctx context.Context, repoName string, oids []string) (map[string]Object, error) { +// GetRepoObjects fetches verified objects for the given OIDs that are linked to +// the repository. Results are keyed by OID, silently dropping missing OIDs. +func (d *DB) GetRepoObjects(ctx context.Context, repoName string, oids []string) (map[string]lfs.Object, error) { if len(oids) == 0 { - return map[string]Object{}, nil + return map[string]lfs.Object{}, nil } var rows []Object err := d.db.WithContext(ctx).Raw(` @@ -238,11 +194,21 @@ AND objects.verified_at IS NOT NULL`, if err != nil { return nil, errors.Wrap(err, "select repo objects by OIDs") } - out := make(map[string]Object, len(rows)) + return verifiedObjectsByOID(rows), nil +} + +// verifiedObjectsByOID converts verified rows, whose size and object_uri are +// guaranteed non-null by the check_verified_consistency constraint. +func verifiedObjectsByOID(rows []Object) map[string]lfs.Object { + out := make(map[string]lfs.Object, len(rows)) for _, r := range rows { - out[r.OID] = r + out[r.OID] = lfs.Object{ + OID: r.OID, + Size: ptrx.Deref(r.Size, -1), + URI: ptrx.Deref(r.ObjectURI, ""), + } } - return out, nil + return out } // SweepOrphanObjects deletes up to limit pending objects (verified_at IS NULL) diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index c8db2a0..2332c5f 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -10,7 +10,7 @@ import ( "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/internal/ptrx" - "unknwon.dev/git-lfs-server/internal/storage" + "unknwon.dev/git-lfs-server/lfs/storage" ) const ( diff --git a/lfs/go.mod b/lfs/go.mod new file mode 100644 index 0000000..ba96a41 --- /dev/null +++ b/lfs/go.mod @@ -0,0 +1,40 @@ +module unknwon.dev/git-lfs-server/lfs + +go 1.26 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 + github.com/cockroachdb/errors v1.13.0 + github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/getsentry/sentry-go v0.46.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/lfs/go.sum b/lfs/go.sum new file mode 100644 index 0000000..a48112f --- /dev/null +++ b/lfs/go.sum @@ -0,0 +1,114 @@ +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= +github.com/cockroachdb/errors v1.13.0 h1:BoCcJeiP9hpBJDETkX19qi8Tb8So37srSsp3stTaDMQ= +github.com/cockroachdb/errors v1.13.0/go.mod h1:bjxt/4E5+OyuAnacpTIU9rn2mzPu1VlthvHP+xpROq0= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= +github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= +github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= +go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/iox/exact_size_reader.go b/lfs/internal/iox/exact_size_reader.go similarity index 100% rename from internal/iox/exact_size_reader.go rename to lfs/internal/iox/exact_size_reader.go diff --git a/internal/iox/exact_size_reader_test.go b/lfs/internal/iox/exact_size_reader_test.go similarity index 100% rename from internal/iox/exact_size_reader_test.go rename to lfs/internal/iox/exact_size_reader_test.go diff --git a/internal/iox/sha256_reader.go b/lfs/internal/iox/sha256_reader.go similarity index 100% rename from internal/iox/sha256_reader.go rename to lfs/internal/iox/sha256_reader.go diff --git a/internal/iox/sha256_reader_test.go b/lfs/internal/iox/sha256_reader_test.go similarity index 100% rename from internal/iox/sha256_reader_test.go rename to lfs/internal/iox/sha256_reader_test.go diff --git a/lfs/lfs.go b/lfs/lfs.go new file mode 100644 index 0000000..548f32e --- /dev/null +++ b/lfs/lfs.go @@ -0,0 +1,123 @@ +// Package lfs implements the server side of the Git LFS Batch API and the +// basic transfer adapter on top of pluggable metadata and object storage. +// +// It leaves HTTP routing, authentication, authorization, and the encoding of +// responses to the caller, so that servers with different URL layouts, +// credentials, and databases can share the transfer logic. +// +// Spec: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md +package lfs + +import ( + "context" + "regexp" +) + +// oidPattern matches the SHA-256 hex digest format used as the LFS object ID: +// exactly 64 lowercase hex characters. +var oidPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// ValidOID reports whether oid is a valid LFS object ID. +func ValidOID(oid string) bool { + return oidPattern.MatchString(oid) +} + +const ( + OperationUpload = "upload" + OperationDownload = "download" + TransferBasic = "basic" +) + +// BatchRequest is the request payload of the batch endpoint. +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Objects []BatchObject `json:"objects"` +} + +// BatchObject is an object requested in a BatchRequest. +type BatchObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// BatchResponse is the response payload of the batch endpoint. +type BatchResponse struct { + Transfer string `json:"transfer"` + Objects []BatchResponseObject `json:"objects"` +} + +// BatchResponseObject is the result for one object of a BatchRequest. Either +// Actions or Error is set, unless the object needs no transfer. +type BatchResponseObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` + Authenticated bool `json:"authenticated,omitempty"` + Actions map[string]Action `json:"actions,omitempty"` + Error *ObjectError `json:"error,omitempty"` +} + +// Action tells the client where and how to transfer an object. +type Action struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` +} + +// ObjectError is the error of one object in a BatchResponse. +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// VerifyRequest is the request payload of the verify endpoint. +type VerifyRequest struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// Error is a client error that the caller should respond with, using Code as +// the HTTP status code. +type Error struct { + Code int + Message string +} + +func (e *Error) Error() string { + return e.Message +} + +// Object is a verified LFS object. +type Object struct { + OID string + Size int64 + // URI is the location of the object content, as returned by the storage + // backend that stores it. + URI string +} + +// Store is the metadata layer of LFS objects. An object is "pending" from the +// start of a presigned upload until it is verified, and only verified objects +// are served. A repository is identified by an opaque string chosen by the +// caller. +type Store interface { + // GetRepoObjects returns the verified objects among oids that are linked to + // repo, keyed by OID. Missing OIDs are omitted. + GetRepoObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) + // GetObjects returns the verified objects among oids regardless of the + // repositories they are linked to, keyed by OID. Missing OIDs are omitted. + // It is only called when Service.CrossRepoDedup is true. + GetObjects(ctx context.Context, oids []string) (map[string]Object, error) + // LinkObject records a verified object whose content is stored at uri and + // links it to repo. It returns the URI the object is recorded with, and when + // that differs from uri, the caller deletes the content at uri as no longer + // referenced. + LinkObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) + // InsertPendingObject records a pending object for oid, unless the object is + // already recorded, so that abandoned presigned uploads can be cleaned up. + InsertPendingObject(ctx context.Context, oid string) error + // VerifyObject marks the pending object for oid as verified with content + // stored at uri and links it to repo. It returns the URI the object is + // recorded with, like LinkObject. It fails when no object is recorded for + // oid. + VerifyObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) +} diff --git a/lfs/moon.yml b/lfs/moon.yml new file mode 100644 index 0000000..7605311 --- /dev/null +++ b/lfs/moon.yml @@ -0,0 +1,45 @@ +$schema: 'https://moonrepo.dev/schemas/project.json' + +language: 'go' +stack: 'backend' +id: 'lfs' + +fileGroups: + sources: + - '**/*.go' + tests: + - '**/*_test.go' + configs: + - 'go.mod' + - 'go.sum' + +tasks: + install: + command: 'go mod tidy' + inputs: + - '@group(configs)' + + lint: + command: 'golangci-lint run' + inputs: + - '@group(sources)' + - '@group(configs)' + deps: + - 'install' + + test: + command: 'go test ./...' + inputs: + - '@group(sources)' + - '@group(tests)' + - '@group(configs)' + deps: + - 'install' + + build: + command: 'go build -o /dev/null ./...' + inputs: + - '@group(sources)' + - '@group(configs)' + deps: + - 'install' diff --git a/lfs/service.go b/lfs/service.go new file mode 100644 index 0000000..6bfd0e5 --- /dev/null +++ b/lfs/service.go @@ -0,0 +1,379 @@ +package lfs + +import ( + "context" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "log/slog" + "maps" + "net/http" + "slices" + "strings" + + "github.com/cockroachdb/errors" + + "unknwon.dev/git-lfs-server/lfs/internal/iox" + "unknwon.dev/git-lfs-server/lfs/storage" +) + +// Service serves LFS operations for repositories. Callers authenticate and +// authorize each request before calling a method, and respond with the Code +// and Message of a returned *Error. Any other returned error is an internal +// error. +type Service struct { + Store Store + // Storage receives uploads and decides the transfer mode: content is + // proxied through the caller for a storage.Proxier, and transferred + // directly by the client through redirects for a storage.Presigner. + Storage storage.Backend + // PreviousStorages are consulted for downloads of objects stored before + // Storage was changed. An object is served by the first backend, starting + // with Storage, whose scheme prefixes the object's URI. + PreviousStorages []storage.Backend + // MaxObjectSize is the maximum size in bytes of an uploaded object. Zero or + // negative means no limit. + MaxObjectSize int64 + // CrossRepoDedup allows an upload to be satisfied by an object that is + // already stored for another repository, without the client sending the + // content again. Without it, a repository only gains an object by uploading + // its full content, so knowing an OID and its size is not enough to read the + // object through a repository the client has write access to. + CrossRepoDedup bool +} + +// Endpoint describes how the client reaches the transfer endpoints of a +// repository: objects at Href/{oid} and verifications at Href/verify. +type Endpoint struct { + Href string + // Header is included in every action, e.g., to forward the credentials of + // the batch request. + Header map[string]string + // Authenticated tells the client that Header authenticates the actions, so + // it does not look up credentials for them. It is required when Storage is a + // storage.Presigner: otherwise the Git LFS client looks up credentials for + // the host of the presigned URL it is redirected to, and fails. + Authenticated bool +} + +// Redirect sends the client to another location to transfer the content. +type Redirect struct { + URL string + // Header must be sent by the client verbatim. + Header map[string]string +} + +// Batch answers a batch request for repo. Callers must check that the actor is +// allowed to perform req.Operation before calling it. +func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, endpoint Endpoint, req BatchRequest) (*BatchResponse, error) { + if req.Operation != OperationUpload && req.Operation != OperationDownload { + return nil, &Error{Code: http.StatusBadRequest, Message: `operation must be "upload" or "download"`} + } + if len(req.Transfers) > 0 && !slices.Contains(req.Transfers, TransferBasic) { + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: `only "basic" transfer is supported`} + } + + oids := make([]string, len(req.Objects)) + for i, o := range req.Objects { + if !ValidOID(o.OID) { + return nil, &Error{Code: http.StatusBadRequest, Message: fmt.Sprintf("invalid oid %q", o.OID)} + } + oids[i] = o.OID + } + + var ( + existing map[string]Object + err error + ) + switch req.Operation { + case OperationDownload: + existing, err = s.Store.GetRepoObjects(ctx, repo, oids) + case OperationUpload: + existing, err = s.uploadedObjects(ctx, repo, oids) + } + if err != nil { + return nil, errors.Wrap(err, "look up objects") + } + + objectHref := func(oid string) string { return endpoint.Href + "/" + oid } + resp := &BatchResponse{ + Transfer: TransferBasic, + Objects: make([]BatchResponseObject, 0, len(req.Objects)), + } + for _, in := range req.Objects { + obj, ok := existing[in.OID] + out := BatchResponseObject{OID: in.OID, Size: in.Size, Authenticated: endpoint.Authenticated} + + switch req.Operation { + case OperationDownload: + switch { + case !ok: + out.Error = &ObjectError{Code: http.StatusNotFound, Message: "object does not exist"} + case obj.Size != in.Size: + out.Error = &ObjectError{Code: http.StatusUnprocessableEntity, Message: "OID exists with different size"} + default: + out.Actions = map[string]Action{ + "download": {Href: objectHref(in.OID), Header: endpoint.Header}, + } + } + case OperationUpload: + switch { + case s.MaxObjectSize > 0 && in.Size > s.MaxObjectSize: + out.Error = &ObjectError{ + Code: http.StatusRequestEntityTooLarge, + Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, s.MaxObjectSize), + } + case ok && obj.Size == in.Size: + // No actions tell the client the object needs no upload. With + // cross-repo dedup, the object may only be linked to another + // repository so far. + if s.CrossRepoDedup { + if _, err := s.Store.LinkObject(ctx, repo, in.OID, in.Size, obj.URI); err != nil { + logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) + out.Error = &ObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} + } + } + case ok && obj.Size != in.Size: + out.Error = &ObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} + default: + rawOID, _ := hex.DecodeString(in.OID) // Validated above. + uploadHeader := make(map[string]string, len(endpoint.Header)+1) + maps.Copy(uploadHeader, endpoint.Header) + // Presigned uploads require the client to send the checksum that + // the storage verifies the content against. + uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) + out.Actions = map[string]Action{ + "upload": {Href: objectHref(in.OID), Header: uploadHeader}, + "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + } + } + } + resp.Objects = append(resp.Objects, out) + } + return resp, nil +} + +// uploadedObjects returns the verified objects among oids that an upload to +// repo does not need to send again. +func (s *Service) uploadedObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) { + if s.CrossRepoDedup { + return s.Store.GetObjects(ctx, oids) + } + return s.Store.GetRepoObjects(ctx, repo, oids) +} + +// UploadResult is the outcome of a successful Service.Upload. +type UploadResult struct { + // Redirect is set when the client must upload the content to another + // location and then call the verify endpoint. + Redirect *Redirect + // Existed reports that the object was already stored and the content was not + // read. + Existed bool +} + +// Upload handles an upload of size bytes for oid to repo. For a +// storage.Proxier, it stores the content read from body after verifying its +// size and hash. For a storage.Presigner, it does not read body. +func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { + if !ValidOID(oid) { + return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + if size < 0 { + return nil, &Error{Code: http.StatusLengthRequired, Message: "Content-Length is required"} + } + if s.MaxObjectSize > 0 && size > s.MaxObjectSize { + return nil, &Error{ + Code: http.StatusRequestEntityTooLarge, + Message: fmt.Sprintf("object size %d exceeds limit %d", size, s.MaxObjectSize), + } + } + + switch b := s.Storage.(type) { + case storage.Proxier: + return s.uploadProxy(ctx, logger, b, repo, oid, size, body) + case storage.Presigner: + return s.uploadPresign(ctx, b, repo, oid, size) + default: + return nil, errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) + } +} + +func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend storage.Proxier, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { + reader := iox.NewSHA256Reader(iox.NewExactSizeReader(body, size), oid) + uri, err := backend.Put(ctx, oid, reader) + if err == nil { + // Put skips reading when the content is already stored, but the client + // must still prove it has the content before the object is linked to + // repo. + _, err = io.Copy(io.Discard, reader) + } + if err != nil { + var sizeErr *iox.SizeMismatchError + var hashErr *iox.SHA256MismatchError + switch { + case errors.As(err, &sizeErr): + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: sizeErr.Error()} + case errors.As(err, &hashErr): + return nil, &Error{Code: http.StatusUnprocessableEntity, Message: hashErr.Error()} + default: + return nil, errors.Wrap(err, "store object") + } + } + + storedURI, err := s.Store.LinkObject(ctx, repo, oid, size, uri) + if err != nil { + return nil, errors.Wrap(err, "link object") + } + if storedURI != uri { + if err := backend.Delete(ctx, uri); err != nil { + logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) + } + } + return &UploadResult{}, nil +} + +func (s *Service) uploadPresign(ctx context.Context, backend storage.Presigner, repo, oid string, size int64) (*UploadResult, error) { + existing, err := s.uploadedObjects(ctx, repo, []string{oid}) + if err != nil { + return nil, errors.Wrap(err, "look up object") + } + if _, ok := existing[oid]; ok { + return &UploadResult{Existed: true}, nil + } + + if err = s.Store.InsertPendingObject(ctx, oid); err != nil { + return nil, errors.Wrap(err, "insert pending object") + } + url, header, err := backend.PresignPut(ctx, oid, size) + if err != nil { + return nil, errors.Wrap(err, "presign upload") + } + return &UploadResult{Redirect: &Redirect{URL: url, Header: header}}, nil +} + +// DownloadResult is the outcome of a successful Service.Download. Exactly one +// of Redirect and Content is set. +type DownloadResult struct { + // Redirect is set when the client must download the content from another + // location. + Redirect *Redirect + // Content is the object content, which the caller must close. + Content io.ReadCloser + // Size is the size of Content in bytes. + Size int64 +} + +// Download handles a download of oid from repo. +func (s *Service) Download(ctx context.Context, repo, oid string) (*DownloadResult, error) { + if !ValidOID(oid) { + return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + + objects, err := s.Store.GetRepoObjects(ctx, repo, []string{oid}) + if err != nil { + return nil, errors.Wrap(err, "look up object") + } + object, ok := objects[oid] + if !ok { + return nil, &Error{Code: http.StatusNotFound, Message: "object does not exist"} + } + + backend := s.backendFor(object.URI) + switch b := backend.(type) { + case storage.Proxier: + rc, err := b.Open(ctx, object.URI) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, errors.Wrapf(err, "object %q is recorded but missing in storage", object.URI) + } + return nil, errors.Wrap(err, "open object") + } + return &DownloadResult{Content: rc, Size: object.Size}, nil + case storage.Presigner: + url, err := b.PresignGet(ctx, object.URI) + if err != nil { + return nil, errors.Wrap(err, "presign download") + } + return &DownloadResult{Redirect: &Redirect{URL: url}}, nil + case nil: + return nil, errors.Newf("no storage backend for uri %q", object.URI) + default: + return nil, errors.Newf("storage backend %T does not implement a known transfer mode", backend) + } +} + +// backendFor returns the backend that stores the object at uri, or nil when +// none does. +func (s *Service) backendFor(uri string) storage.Backend { + if strings.HasPrefix(uri, s.Storage.Scheme()) { + return s.Storage + } + for _, b := range s.PreviousStorages { + if strings.HasPrefix(uri, b.Scheme()) { + return b + } + } + return nil +} + +// Verify handles the verification of an upload to repo. A presigned upload +// becomes downloadable only after it is verified. +func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) error { + if !ValidOID(req.OID) { + return &Error{Code: http.StatusBadRequest, Message: "invalid oid"} + } + + switch b := s.Storage.(type) { + case storage.Proxier: + return s.verifyProxy(ctx, repo, req) + case storage.Presigner: + return s.verifyPresign(ctx, logger, b, repo, req) + default: + return errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) + } +} + +func (s *Service) verifyProxy(ctx context.Context, repo string, req VerifyRequest) error { + objects, err := s.Store.GetRepoObjects(ctx, repo, []string{req.OID}) + if err != nil { + return errors.Wrap(err, "look up object") + } + object, ok := objects[req.OID] + if !ok { + return &Error{Code: http.StatusNotFound, Message: "object does not exist"} + } + if object.Size != req.Size { + return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} + } + return nil +} + +func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo string, req VerifyRequest) error { + uri := backend.URI(req.OID) + headSize, err := backend.Head(ctx, uri) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return &Error{Code: http.StatusUnprocessableEntity, Message: "object does not exist in storage"} + } + return errors.Wrap(err, "head object") + } + // The presigned upload signs Content-Length, which should make this + // impossible, but treat any mismatch as a hard error rather than trusting + // either side. + if headSize != req.Size { + return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} + } + + storedURI, err := s.Store.VerifyObject(ctx, repo, req.OID, req.Size, uri) + if err != nil { + return errors.Wrap(err, "mark object verified") + } + if storedURI != uri { + if err := backend.Delete(ctx, uri); err != nil { + logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) + } + } + return nil +} diff --git a/lfs/service_test.go b/lfs/service_test.go new file mode 100644 index 0000000..bfe39f1 --- /dev/null +++ b/lfs/service_test.go @@ -0,0 +1,373 @@ +package lfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/cockroachdb/errors" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +// memStore is an in-memory Store. +type memStore struct { + mu sync.Mutex + objects map[string]*memObject // Keyed by OID. + links map[string]map[string]bool // Keyed by repo, then OID. +} + +type memObject struct { + size int64 + uri string + verified bool +} + +func newMemStore() *memStore { + return &memStore{objects: map[string]*memObject{}, links: map[string]map[string]bool{}} +} + +func (s *memStore) verifiedObjects(oids []string, keep func(oid string) bool) map[string]Object { + out := map[string]Object{} + for _, oid := range oids { + if o := s.objects[oid]; o != nil && o.verified && keep(oid) { + out[oid] = Object{OID: oid, Size: o.size, URI: o.uri} + } + } + return out +} + +func (s *memStore) GetRepoObjects(_ context.Context, repo string, oids []string) (map[string]Object, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.verifiedObjects(oids, func(oid string) bool { return s.links[repo][oid] }), nil +} + +func (s *memStore) GetObjects(_ context.Context, oids []string) (map[string]Object, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.verifiedObjects(oids, func(string) bool { return true }), nil +} + +func (s *memStore) link(repo, oid string) { + if s.links[repo] == nil { + s.links[repo] = map[string]bool{} + } + s.links[repo][oid] = true +} + +func (s *memStore) LinkObject(_ context.Context, repo, oid string, size int64, uri string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + o := s.objects[oid] + if o == nil { + o = &memObject{} + s.objects[oid] = o + } + if !o.verified { + o.size, o.uri, o.verified = size, uri, true + } + s.link(repo, oid) + return o.uri, nil +} + +func (s *memStore) InsertPendingObject(_ context.Context, oid string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.objects[oid] == nil { + s.objects[oid] = &memObject{} + } + return nil +} + +func (s *memStore) VerifyObject(_ context.Context, repo, oid string, size int64, uri string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + o := s.objects[oid] + if o == nil { + return "", errors.New("object not found") + } + if !o.verified { + o.size, o.uri, o.verified = size, uri, true + } + s.link(repo, oid) + return o.uri, nil +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func newFilesystemBackend(t *testing.T, name, scheme string) *storage.FilesystemBackend { + t.Helper() + root := t.TempDir() + b, err := storage.NewFilesystemBackend(name, scheme, root, filepath.Join(root, ".tmp")) + require.NoError(t, err) + return b +} + +func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { + t.Helper() + mem := s3mem.New() + require.NoError(t, mem.CreateBucket("lfs-objects")) + srv := httptest.NewServer(gofakes3.New(mem).Server()) + t.Cleanup(srv.Close) + b, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) + require.NoError(t, err) + return b +} + +var discardLogger = slog.New(slog.DiscardHandler) + +func requireLFSError(t *testing.T, err error, wantCode int) { + t.Helper() + var lfsErr *Error + require.True(t, errors.As(err, &lfsErr), "want *Error, got %v", err) + assert.Equal(t, wantCode, lfsErr.Code, lfsErr.Message) +} + +func TestService_Proxy(t *testing.T) { + ctx := t.Context() + content := "proxied content" + oid := sha256Hex(content) + size := int64(len(content)) + + t.Run("upload stores content and links it to the repository only", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, &UploadResult{}, result) + + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) + + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + got, err := io.ReadAll(download.Content) + require.NoError(t, download.Content.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + assert.Equal(t, size, download.Size) + + _, err = s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + }) + + t.Run("upload of stored content still verifies the body", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + forged := strings.Repeat("x", len(content)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(forged)) + requireLFSError(t, err, http.StatusUnprocessableEntity) + _, err = s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + + _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(content)) + require.NoError(t, err) + _, err = s.Download(ctx, "repo2", oid) + require.NoError(t, err) + }) + + t.Run("upload rejects invalid input", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), MaxObjectSize: size - 1} + _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", size, strings.NewReader(content)) + requireLFSError(t, err, http.StatusBadRequest) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, -1, strings.NewReader(content)) + requireLFSError(t, err, http.StatusLengthRequired) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + requireLFSError(t, err, http.StatusRequestEntityTooLarge) + + s.MaxObjectSize = 0 + _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content[1:])) + requireLFSError(t, err, http.StatusUnprocessableEntity) + }) + + t.Run("download reads objects from previous storages", func(t *testing.T) { + store := newMemStore() + previous := newFilesystemBackend(t, "old", "old-file://") + s := &Service{Store: store, Storage: previous} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + s = &Service{Store: store, Storage: newFilesystemBackend(t, "new", "file://"), PreviousStorages: []storage.Backend{previous}} + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + got, err := io.ReadAll(download.Content) + require.NoError(t, download.Content.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + + s.PreviousStorages = nil + _, err = s.Download(ctx, "repo1", oid) + require.Error(t, err) + assert.False(t, errors.As(err, new(*Error)), "a recorded object without a backend is an internal error") + }) +} + +func TestService_Batch(t *testing.T) { + ctx := t.Context() + content := "batched content" + oid := sha256Hex(content) + size := int64(len(content)) + endpoint := Endpoint{Href: "https://lfs.example.com/repo/info/lfs/objects"} + + newServiceWithObject := func(t *testing.T, crossRepoDedup bool) *Service { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), CrossRepoDedup: crossRepoDedup} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + return s + } + upload := BatchRequest{Operation: OperationUpload, Objects: []BatchObject{{OID: oid, Size: size}}} + download := BatchRequest{Operation: OperationDownload, Objects: []BatchObject{{OID: oid, Size: size}}} + + t.Run("upload of an object in the repository needs no transfer", func(t *testing.T) { + s := newServiceWithObject(t, false) + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, upload) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + }) + + t.Run("upload of an object in another repository", func(t *testing.T) { + t.Run("needs a transfer without cross-repo dedup", func(t *testing.T) { + s := newServiceWithObject(t, false) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + assert.Contains(t, resp.Objects[0].Actions, "upload") + + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + require.NotNil(t, resp.Objects[0].Error) + assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) + }) + + t.Run("links the object with cross-repo dedup", func(t *testing.T) { + s := newServiceWithObject(t, true) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + assert.Contains(t, resp.Objects[0].Actions, "download") + }) + }) + + t.Run("actions carry the endpoint", func(t *testing.T) { + s := newServiceWithObject(t, false) + endpoint := Endpoint{Href: endpoint.Href, Header: map[string]string{"Authorization": "Basic Zm9vOmJhcg=="}, Authenticated: true} + newContent := "new content" + newOID := sha256Hex(newContent) + + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, BatchRequest{ + Operation: OperationUpload, + Objects: []BatchObject{{OID: newOID, Size: int64(len(newContent))}}, + }) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{ + OID: newOID, + Size: int64(len(newContent)), + Authenticated: true, + Actions: map[string]Action{ + "upload": { + Href: endpoint.Href + "/" + newOID, + Header: map[string]string{ + "Authorization": "Basic Zm9vOmJhcg==", + "x-amz-checksum-sha256": "/jJgjJ71ts9+P5RkgCU/928k9OwGePPQ8H+YRMv/lgE=", + }, + }, + "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + }, + }}, resp.Objects) + + resp, err = s.Batch(ctx, discardLogger, "repo1", endpoint, download) + require.NoError(t, err) + assert.Equal(t, []BatchResponseObject{{ + OID: oid, + Size: size, + Authenticated: true, + Actions: map[string]Action{"download": {Href: endpoint.Href + "/" + oid, Header: endpoint.Header}}, + }}, resp.Objects) + }) + + t.Run("rejects invalid requests", func(t *testing.T) { + s := newServiceWithObject(t, false) + for _, req := range []BatchRequest{ + {Operation: "delete"}, + {Operation: OperationDownload, Objects: []BatchObject{{OID: "not-an-oid"}}}, + } { + _, err := s.Batch(ctx, discardLogger, "repo1", endpoint, req) + requireLFSError(t, err, http.StatusBadRequest) + } + _, err := s.Batch(ctx, discardLogger, "repo1", endpoint, BatchRequest{Operation: OperationUpload, Transfers: []string{"tus"}}) + requireLFSError(t, err, http.StatusUnprocessableEntity) + }) +} + +func TestService_Presign(t *testing.T) { + ctx := t.Context() + content := "presigned content" + oid := sha256Hex(content) + size := int64(len(content)) + + // uploadToRedirect sends the content to the presigned URL like a client. + uploadToRedirect := func(t *testing.T, redirect *Redirect) { + t.Helper() + req, err := http.NewRequestWithContext(ctx, http.MethodPut, redirect.URL, strings.NewReader(content)) + require.NoError(t, err) + for k, v := range redirect.Header { + req.Header.Set(k, v) + } + req.ContentLength = size + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + } + + for _, crossRepoDedup := range []bool{false, true} { + s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t), CrossRepoDedup: crossRepoDedup} + + result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, nil) + require.NoError(t, err) + require.NotNil(t, result.Redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size}), http.StatusUnprocessableEntity) + _, err = s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) + + uploadToRedirect(t, result.Redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + + download, err := s.Download(ctx, "repo1", oid) + require.NoError(t, err) + require.NotNil(t, download.Redirect) + assert.Contains(t, download.Redirect.URL, oid) + + result, err = s.Upload(ctx, discardLogger, "repo1", oid, size, nil) + require.NoError(t, err) + assert.Equal(t, &UploadResult{Existed: true}, result) + + // Without cross-repo dedup, another repository must upload the content + // itself, which the storage checks against the OID. + result, err = s.Upload(ctx, discardLogger, "repo2", oid, size, nil) + require.NoError(t, err) + assert.Equal(t, crossRepoDedup, result.Redirect == nil, "crossRepoDedup=%v", crossRepoDedup) + } +} diff --git a/internal/storage/filesystem.go b/lfs/storage/filesystem.go similarity index 96% rename from internal/storage/filesystem.go rename to lfs/storage/filesystem.go index 430efc8..a86e1b6 100644 --- a/internal/storage/filesystem.go +++ b/lfs/storage/filesystem.go @@ -48,6 +48,11 @@ func (b *FilesystemBackend) storagePath(oid string) string { return filepath.Join(b.root, oid[0:2], oid[2:4], oid) } +// URI returns the file URI of the object under the storage root. +func (b *FilesystemBackend) URI(oid string) string { + return b.uriFromPath(b.storagePath(oid)) +} + func (b *FilesystemBackend) Put(ctx context.Context, oid string, r io.Reader) (string, error) { if len(oid) < 4 { return "", errors.Newf("invalid oid %q", oid) diff --git a/internal/storage/filesystem_test.go b/lfs/storage/filesystem_test.go similarity index 100% rename from internal/storage/filesystem_test.go rename to lfs/storage/filesystem_test.go diff --git a/internal/storage/s3_presign.go b/lfs/storage/s3_presign.go similarity index 100% rename from internal/storage/s3_presign.go rename to lfs/storage/s3_presign.go diff --git a/internal/storage/s3_presign_test.go b/lfs/storage/s3_presign_test.go similarity index 100% rename from internal/storage/s3_presign_test.go rename to lfs/storage/s3_presign_test.go diff --git a/internal/storage/storage.go b/lfs/storage/storage.go similarity index 97% rename from internal/storage/storage.go rename to lfs/storage/storage.go index bdc8bf3..6bb95ae 100644 --- a/internal/storage/storage.go +++ b/lfs/storage/storage.go @@ -20,6 +20,10 @@ type Backend interface { Type() Type // Scheme returns the URI scheme prefix (including "://") that this backend owns. Scheme() string + // URI returns the canonical, non-expiring location of the object. Pure + // function of the OID and the backend's static config (e.g. bucket name), + // safe to compute without I/O. + URI(oid string) string // Delete removes the object at uri. Returns nil if already absent. Delete(ctx context.Context, uri string) error } @@ -40,6 +44,7 @@ type Proxier interface { // Put stores the bytes from the reader under the given OID and returns a URI // that uniquely identifies the stored object, including the backend scheme. + // It may return without reading r when the object is already stored. Put(ctx context.Context, oid string, r io.Reader) (uri string, err error) // Open returns a reader for the object at URI. It returns ErrNotFound when no // matching object exists. The caller MUST close the reader. @@ -51,10 +56,6 @@ type Proxier interface { type Presigner interface { Backend - // URI returns the canonical, non-expiring location of the object. Pure - // function of the OID and the backend's static config (e.g. bucket name), - // safe to compute without I/O. - URI(oid string) string // PresignPut returns a short-lived URL the client PUTs to, along with the // headers the client MUST send verbatim. PresignPut(ctx context.Context, oid string, size int64) (url string, headers map[string]string, err error) From 605bed21c584b633cafa5277d05f69e8d5369f41 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Wed, 11 Nov 2026 14:47:50 +0000 Subject: [PATCH 2/5] lfs: close downloaded content in proxy test Windows cannot remove the test's temp dir while the object file is open. --- lfs/service_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lfs/service_test.go b/lfs/service_test.go index bfe39f1..830bcad 100644 --- a/lfs/service_test.go +++ b/lfs/service_test.go @@ -180,8 +180,9 @@ func TestService_Proxy(t *testing.T) { _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(content)) require.NoError(t, err) - _, err = s.Download(ctx, "repo2", oid) + download, err := s.Download(ctx, "repo2", oid) require.NoError(t, err) + require.NoError(t, download.Content.Close()) }) t.Run("upload rejects invalid input", func(t *testing.T) { From f7d816405b4a3394b459013b4b44620a0ed3f3a4 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Wed, 11 Nov 2026 15:40:24 +0000 Subject: [PATCH 3/5] lfs: serve the LFS HTTP endpoints from the module Add Service.Serve, which handles the batch, upload, download, and verify requests of a repository given an Access that the caller resolved after authenticating the request. It dispatches on the method and the last path segment, so it works with any router that runs net/http handlers and any URL layout. lfsd now only maps its forge permissions to an Access and hands requests to it. Upload actions now set Content-Type to application/octet-stream so the Git LFS client does not send a type detected from the file content. --- README.md | 16 +++- cmd/lfsd/auth.go | 13 +-- cmd/lfsd/lfs.go | 197 +++-------------------------------------- cmd/lfsd/main.go | 11 +-- lfs/http.go | 209 ++++++++++++++++++++++++++++++++++++++++++++ lfs/http_test.go | 132 ++++++++++++++++++++++++++++ lfs/service.go | 11 ++- lfs/service_test.go | 1 + 8 files changed, 388 insertions(+), 202 deletions(-) create mode 100644 lfs/http.go create mode 100644 lfs/http_test.go diff --git a/README.md b/README.md index ac1d589..dcca796 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,21 @@ The allowlist is enforced before the forge token check, so a request for a disal The `unknwon.dev/git-lfs-server/lfs` module implements the Batch API and the basic transfer adapter for other Git servers to embed. It is a separate Go module so that importers do not pull in the dependencies of lfsd (PostgreSQL, Flamego, forge clients). -An `lfs.Service` combines an `lfs.Store`, which records objects and the repositories they are linked to, with a `storage.Backend`, which holds the content. The caller keeps its own routing, authentication, authorization, and response encoding, identifies repositories with any string it likes, and calls `Batch`, `Upload`, `Download`, and `Verify` from its handlers. A `storage.Proxier` backend (e.g., `storage.FilesystemBackend`) streams content through the caller, and a `storage.Presigner` backend (e.g., `storage.S3PresignBackend`) redirects clients to the storage. `PreviousStorages` keeps objects readable after the backend for new uploads changes. +An `lfs.Service` combines an `lfs.Store`, which records objects and the repositories they are linked to, with a `storage.Backend`, which holds the content. A `storage.Proxier` backend (e.g., `storage.FilesystemBackend`) streams content through the server, and a `storage.Presigner` backend (e.g., `storage.S3PresignBackend`) redirects clients to the storage. `PreviousStorages` keeps objects readable after the backend for new uploads changes. + +`Service.Serve` handles the HTTP requests of the batch, upload, download, and verify endpoints. The caller routes the requests, authenticates them, and passes an `lfs.Access` with the repository, the base URL of its endpoints, and whether the actor can read and write. `Serve` works with any router that can run a `net/http` handler, e.g., Flamego, Macaron, or `http.ServeMux`: + +```go +mux.HandleFunc("/{repo}/info/lfs/objects/", func(w http.ResponseWriter, r *http.Request) { + access, ok := authorize(w, r) // Your authentication and authorization. + if !ok { + return + } + service.Serve(logger, w, r, access) +}) +``` + +The lower-level `Batch`, `Upload`, `Download`, and `Verify` methods are available for servers that need their own HTTP handling. ## Local development diff --git a/cmd/lfsd/auth.go b/cmd/lfsd/auth.go index 9d1dbeb..2181d82 100644 --- a/cmd/lfsd/auth.go +++ b/cmd/lfsd/auth.go @@ -15,6 +15,7 @@ import ( "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/logx" + "unknwon.dev/git-lfs-server/lfs" ) func authorize(forges map[string]forge.Provider) flamego.Handler { @@ -25,26 +26,26 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { host := hostFromContext(c) provider, ok := forges[host] if !ok { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "unsupported forge host") + lfs.WriteError(c.ResponseWriter(), http.StatusBadRequest, "unsupported forge host") return } username, token, ok := c.Request().BasicAuth() if !ok || token == "" { c.ResponseWriter().Header().Set("WWW-Authenticate", `Basic realm="git-lfs"`) - writeLFSError(c.ResponseWriter(), http.StatusUnauthorized, "basic auth credentials are required") + lfs.WriteError(c.ResponseWriter(), http.StatusUnauthorized, "basic auth credentials are required") return } repo := strings.ToLower(c.Param("**")) if repo == "" { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, "repository path is missing") + lfs.WriteError(c.ResponseWriter(), http.StatusBadRequest, "repository path is missing") return } if !provider.Allow(repo) { logger.InfoContext(ctx, "Repository rejected by allowlist", "host", host, "repo", repo) - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "repository is not in the allowlist") + lfs.WriteError(c.ResponseWriter(), http.StatusForbidden, "repository is not in the allowlist") return } @@ -62,10 +63,10 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { perm, rawTTL, err := provider.Authorize(ctx, logger.Scoped("forge"), repo, username, token) if err != nil { if errors.Is(err, forge.ErrTokenInvalid) { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "token is invalid or lacks repository access") + lfs.WriteError(c.ResponseWriter(), http.StatusForbidden, "token is invalid or lacks repository access") } else { logger.ErrorContext(ctx, "Failed to verify repository permissions", "repo", repo, "error", err) - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + lfs.WriteError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } return } diff --git a/cmd/lfsd/lfs.go b/cmd/lfsd/lfs.go index c93842a..5f2f8d3 100644 --- a/cmd/lfsd/lfs.go +++ b/cmd/lfsd/lfs.go @@ -1,14 +1,9 @@ package main import ( - "context" - "encoding/json" "fmt" - "io" "net/http" - "strconv" - "github.com/cockroachdb/errors" "github.com/flamego/flamego" "unknwon.dev/git-lfs-server/internal/forge" @@ -16,193 +11,23 @@ import ( "unknwon.dev/git-lfs-server/lfs" ) -const ( - mediaTypeLFS = "application/vnd.git-lfs+json" - - // maxJSONRequestBytes caps JSON request bodies (batch, verify) to prevent - // authenticated callers from exhausting memory with oversized payloads. - maxJSONRequestBytes = 10 << 20 -) - -// mapService maps the *lfs.Service of the forge host to the request context. -func mapService(services map[string]*lfs.Service) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger) { +// serveLFS hands the request to the *lfs.Service of the forge host, with the +// permission resolved by the authorize middleware. +func serveLFS(externalURL string, services map[string]*lfs.Service) flamego.Handler { + return func(c flamego.Context, logger *logx.Logger, perm forge.Permission) { service, ok := services[hostFromContext(c)] if !ok { logger.ErrorContext(c.Request().Context(), "No storage backend configured for host") - writeLFSError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - return - } - c.Map(service) - } -} - -func serveBatch(externalURL string) flamego.Handler { - return func(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { - logger = logger.Scoped("batch") - ctx := c.Request().Context() - - body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) - defer func() { _ = body.Close() }() - - var req lfs.BatchRequest - if err := json.NewDecoder(body).Decode(&req); err != nil { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) - return - } - - switch req.Operation { - case lfs.OperationUpload: - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") - return - } - case lfs.OperationDownload: - if perm != forge.PermissionRead && perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") - return - } - default: - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, `operation must be "upload" or "download"`) + lfs.WriteError(c.ResponseWriter(), http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) return } repoName := repoNameFromContext(c) - endpoint := lfs.Endpoint{ - Href: fmt.Sprintf("%s/%s/info/lfs/objects", externalURL, repoName), - // Every request passes the authorize middleware with Basic Auth, so - // forwarding the header authenticates the actions. - Authenticated: true, - } - if authz := c.Request().Header.Get("Authorization"); authz != "" { - endpoint.Header = map[string]string{"Authorization": authz} - } - - resp, err := service.Batch(ctx, logger.Logger, repoName, endpoint, req) - if err != nil { - writeServiceError(ctx, logger, c.ResponseWriter(), err, "Failed to process batch", "failed to process batch") - return - } - - c.ResponseWriter().Header().Set("Content-Type", mediaTypeLFS) - if err := json.NewEncoder(c.ResponseWriter()).Encode(resp); err != nil { - logger.ErrorContext(ctx, "Failed to encode batch response", "error", err) - } - } -} - -func serveUpload(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { - logger = logger.Scoped("upload") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for upload") - return - } - - body := c.Request().Body().ReadCloser() - defer func() { _ = body.Close() }() - - oid := c.Param("oid") - result, err := service.Upload(ctx, logger.Logger, repoNameFromContext(c), oid, c.Request().ContentLength, body) - if err != nil { - writeServiceError(ctx, logger.With("oid", oid), c.ResponseWriter(), err, "Failed to upload object", http.StatusText(http.StatusInternalServerError)) - return + service.Serve(logger.Scoped("lfs").Logger, c.ResponseWriter(), c.Request().Request, lfs.Access{ + Repo: repoName, + Href: fmt.Sprintf("%s/%s/info/lfs/objects", externalURL, repoName), + CanRead: perm == forge.PermissionRead || perm == forge.PermissionWrite, + CanWrite: perm == forge.PermissionWrite, + }) } - - w := c.ResponseWriter() - switch { - case result.Redirect != nil: - for k, v := range result.Redirect.Header { - w.Header().Set(k, v) - } - w.Header().Set("Location", result.Redirect.URL) - w.WriteHeader(http.StatusTemporaryRedirect) - case result.Existed: - w.WriteHeader(http.StatusOK) - default: - w.WriteHeader(http.StatusNoContent) - } -} - -func serveDownload(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { - logger = logger.Scoped("download") - ctx := c.Request().Context() - - if perm != forge.PermissionRead && perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "read permission required for download") - return - } - - oid := c.Param("oid") - result, err := service.Download(ctx, repoNameFromContext(c), oid) - if err != nil { - writeServiceError(ctx, logger.With("oid", oid), c.ResponseWriter(), err, "Failed to download object", http.StatusText(http.StatusInternalServerError)) - return - } - - w := c.ResponseWriter() - if result.Redirect != nil { - w.Header().Set("Location", result.Redirect.URL) - w.WriteHeader(http.StatusTemporaryRedirect) - return - } - - defer func() { _ = result.Content.Close() }() - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) - w.WriteHeader(http.StatusOK) - if _, err = io.Copy(w, result.Content); err != nil { - logger.ErrorContext(ctx, "Failed to stream object", "oid", oid, "error", err) - } -} - -func serveVerify(c flamego.Context, logger *logx.Logger, perm forge.Permission, service *lfs.Service) { - logger = logger.Scoped("verify") - ctx := c.Request().Context() - - if perm != forge.PermissionWrite { - writeLFSError(c.ResponseWriter(), http.StatusForbidden, "write permission required for verify") - return - } - - body := http.MaxBytesReader(c.ResponseWriter(), c.Request().Body().ReadCloser(), maxJSONRequestBytes) - defer func() { _ = body.Close() }() - - var req lfs.VerifyRequest - if err := json.NewDecoder(body).Decode(&req); err != nil { - writeLFSError(c.ResponseWriter(), http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) - return - } - - if err := service.Verify(ctx, logger.Logger, repoNameFromContext(c), req); err != nil { - writeServiceError(ctx, logger.With("oid", req.OID), c.ResponseWriter(), err, "Failed to verify object", http.StatusText(http.StatusInternalServerError)) - return - } - c.ResponseWriter().WriteHeader(http.StatusNoContent) -} - -// writeServiceError responds with the client error carried by err as an -// *lfs.Error. Any other error is internal, so it is logged with logMessage and -// the response only carries internalMessage. -func writeServiceError(ctx context.Context, logger *logx.Logger, w http.ResponseWriter, err error, logMessage, internalMessage string) { - var lfsErr *lfs.Error - if errors.As(err, &lfsErr) { - writeLFSError(w, lfsErr.Code, lfsErr.Message) - return - } - logger.ErrorContext(ctx, logMessage, "error", err) - writeLFSError(w, http.StatusInternalServerError, internalMessage) -} - -// writeLFSError writes an error response in the format required by the LFS -// Batch API spec: a JSON body of {"message": "..."} with the -// application/vnd.git-lfs+json content type. All LFS endpoints (batch, upload, -// download, verify, and the auth middleware that fronts them) must use this -// helper instead of http.Error so clients get a parseable body and the right -// content type on every error. -func writeLFSError(w http.ResponseWriter, code int, msg string) { - w.Header().Set("Content-Type", mediaTypeLFS) - w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) } diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index b071bd9..dc8cc0d 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -70,12 +70,13 @@ func main() { CrossRepoDedup: true, } } + serve := serveLFS(config.Server.ExternalURL, services) f.Group("/{host}/{**}/info/lfs/objects", func() { - f.Post("/batch", serveBatch(config.Server.ExternalURL)) - f.Get("/{oid}", serveDownload) - f.Put("/{oid}", serveUpload) - f.Post("/verify", serveVerify) - }, authorize(config.Forges), mapService(services)) + f.Post("/batch", serve) + f.Get("/{oid}", serve) + f.Put("/{oid}", serve) + f.Post("/verify", serve) + }, authorize(config.Forges)) ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() diff --git a/lfs/http.go b/lfs/http.go new file mode 100644 index 0000000..2a7e674 --- /dev/null +++ b/lfs/http.go @@ -0,0 +1,209 @@ +package lfs + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "path" + "strconv" + + "github.com/cockroachdb/errors" +) + +const ( + // MediaType is the content type of LFS API requests and responses. + MediaType = "application/vnd.git-lfs+json" + + // maxJSONRequestBytes caps JSON request bodies (batch, verify) to prevent + // authenticated callers from exhausting memory with oversized payloads. + maxJSONRequestBytes = 10 << 20 +) + +// Access is what the caller resolved for a request after authenticating it. +type Access struct { + // Repo identifies the repository to the Store. + Repo string + // Href is the URL that the transfer endpoints of the repository are served + // under, without a trailing slash: objects at Href/{oid} and verifications + // at Href/verify. + Href string + // CanRead allows downloads. + CanRead bool + // CanWrite allows uploads and verifications. + CanWrite bool +} + +// Serve handles an LFS API request to the repository in access. It dispatches +// on the method and the last segment of the URL path, so it serves any URL +// layout whose endpoints end in "batch", "verify", and the OID: +// +// - POST .../batch +// - GET .../{oid} +// - PUT .../{oid} +// - POST .../verify +// +// The request's Authorization header is forwarded in the actions of batch +// responses, so the client authenticates transfers with the same credentials. +func (s *Service) Serve(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + name := path.Base(r.URL.Path) + switch { + case r.Method == http.MethodPost && name == "batch": + s.serveBatch(logger, w, r, access) + case r.Method == http.MethodPost && name == "verify": + s.serveVerify(logger, w, r, access) + case r.Method == http.MethodGet: + s.serveDownload(logger, w, r, access, name) + case r.Method == http.MethodPut: + s.serveUpload(logger, w, r, access, name) + default: + WriteError(w, http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed)) + } +} + +func (s *Service) serveBatch(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + ctx := r.Context() + + body := http.MaxBytesReader(w, r.Body, maxJSONRequestBytes) + defer func() { _ = body.Close() }() + + var req BatchRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + WriteError(w, http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) + return + } + + switch req.Operation { + case OperationUpload: + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for upload") + return + } + case OperationDownload: + if !access.CanRead { + WriteError(w, http.StatusForbidden, "read permission required for download") + return + } + default: + WriteError(w, http.StatusBadRequest, `operation must be "upload" or "download"`) + return + } + + endpoint := Endpoint{Href: access.Href} + if authz := r.Header.Get("Authorization"); authz != "" { + endpoint.Header = map[string]string{"Authorization": authz} + endpoint.Authenticated = true + } + resp, err := s.Batch(ctx, logger, access.Repo, endpoint, req) + if err != nil { + writeServiceError(logger, w, r, err, "failed to process batch") + return + } + + w.Header().Set("Content-Type", MediaType) + if err := json.NewEncoder(w).Encode(resp); err != nil { + logger.ErrorContext(ctx, "Failed to encode batch response", "error", err) + } +} + +func (s *Service) serveUpload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for upload") + return + } + + defer func() { _ = r.Body.Close() }() + result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.ContentLength, r.Body) + if err != nil { + writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + + switch { + case result.Redirect != nil: + writeRedirect(w, result.Redirect) + case result.Existed: + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNoContent) + } +} + +func (s *Service) serveDownload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { + if !access.CanRead { + WriteError(w, http.StatusForbidden, "read permission required for download") + return + } + + result, err := s.Download(r.Context(), access.Repo, oid) + if err != nil { + writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + if result.Redirect != nil { + writeRedirect(w, result.Redirect) + return + } + + defer func() { _ = result.Content.Close() }() + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(result.Size, 10)) + w.WriteHeader(http.StatusOK) + if _, err = io.Copy(w, result.Content); err != nil { + logger.ErrorContext(r.Context(), "Failed to stream object", "oid", oid, "error", err) + } +} + +func (s *Service) serveVerify(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { + if !access.CanWrite { + WriteError(w, http.StatusForbidden, "write permission required for verify") + return + } + + body := http.MaxBytesReader(w, r.Body, maxJSONRequestBytes) + defer func() { _ = body.Close() }() + + var req VerifyRequest + if err := json.NewDecoder(body).Decode(&req); err != nil { + WriteError(w, http.StatusBadRequest, fmt.Sprintf("invalid JSON body: %v", err)) + return + } + + if err := s.Verify(r.Context(), logger, access.Repo, req); err != nil { + writeServiceError(logger.With("oid", req.OID), w, r, err, http.StatusText(http.StatusInternalServerError)) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func writeRedirect(w http.ResponseWriter, redirect *Redirect) { + for k, v := range redirect.Header { + w.Header().Set(k, v) + } + w.Header().Set("Location", redirect.URL) + w.WriteHeader(http.StatusTemporaryRedirect) +} + +// writeServiceError responds with the client error carried by err as an +// *Error. Any other error is internal, so it is logged and the response only +// carries internalMessage. +func writeServiceError(logger *slog.Logger, w http.ResponseWriter, r *http.Request, err error, internalMessage string) { + var lfsErr *Error + if errors.As(err, &lfsErr) { + WriteError(w, lfsErr.Code, lfsErr.Message) + return + } + logger.ErrorContext(r.Context(), "Failed to serve LFS request", "path", r.URL.Path, "error", err) + WriteError(w, http.StatusInternalServerError, internalMessage) +} + +// WriteError writes an error response in the format required by the LFS API: a +// JSON body of {"message": "..."} with the MediaType content type. Callers use +// it for the errors of their own middleware, e.g., authentication, so clients +// get a parseable body on every error. +func WriteError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", MediaType) + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"message": msg}) +} diff --git a/lfs/http_test.go b/lfs/http_test.go new file mode 100644 index 0000000..22cbcea --- /dev/null +++ b/lfs/http_test.go @@ -0,0 +1,132 @@ +package lfs + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestService_Serve(t *testing.T) { + content := "served content" + oid := sha256Hex(content) + size := len(content) + const href = "https://lfs.example.com/repo/info/lfs/objects" + readWrite := Access{Repo: "repo", Href: href, CanRead: true, CanWrite: true} + + serve := func(s *Service, access Access, method, target, body string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, href+target, strings.NewReader(body)) + r.SetBasicAuth("user", "token") + w := httptest.NewRecorder() + s.Serve(discardLogger, w, r, access) + return w + } + requireError := func(t *testing.T, w *httptest.ResponseRecorder, wantCode int) { + t.Helper() + require.Equal(t, wantCode, w.Code, w.Body.String()) + assert.Equal(t, MediaType, w.Header().Get("Content-Type")) + var body struct{ Message string } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.NotEmpty(t, body.Message) + } + batch := func(operation string) string { + return `{"operation":"` + operation + `","objects":[{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}]}` + } + verify := `{"oid":"` + oid + `","size":` + strconv.Itoa(size) + `}` + + t.Run("proxied transfer", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + + w := serve(s, readWrite, http.MethodPost, "/batch", batch(OperationUpload)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + assert.Equal(t, MediaType, w.Header().Get("Content-Type")) + var resp BatchResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.Len(t, resp.Objects, 1) + authz := "Basic dXNlcjp0b2tlbg==" + assert.True(t, resp.Objects[0].Authenticated) + assert.Equal(t, Action{ + Href: href + "/" + oid, + Header: map[string]string{ + "Authorization": authz, + "Content-Type": "application/octet-stream", + "x-amz-checksum-sha256": "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", + }, + }, resp.Objects[0].Actions["upload"]) + assert.Equal(t, Action{Href: href + "/verify", Header: map[string]string{"Authorization": authz}}, resp.Objects[0].Actions["verify"]) + + requireError(t, serve(s, readWrite, http.MethodPut, "/"+oid, strings.Repeat("x", size)), http.StatusUnprocessableEntity) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify", verify), http.StatusNotFound) + + w = serve(s, readWrite, http.MethodPut, "/"+oid, content) + assert.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + w = serve(s, readWrite, http.MethodPost, "/verify", verify) + assert.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + + w = serve(s, readWrite, http.MethodGet, "/"+oid, "") + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + assert.Equal(t, "application/octet-stream", w.Header().Get("Content-Type")) + assert.Equal(t, strconv.Itoa(size), w.Header().Get("Content-Length")) + assert.Equal(t, content, w.Body.String()) + }) + + t.Run("presigned transfer redirects", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t)} + + w := serve(s, readWrite, http.MethodPut, "/"+oid, "") + require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) + assert.Equal(t, "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", w.Header().Get("x-amz-checksum-sha256")) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, w.Header().Get("Location"), strings.NewReader(content)) + require.NoError(t, err) + req.Header.Set("x-amz-checksum-sha256", w.Header().Get("x-amz-checksum-sha256")) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, http.StatusOK, resp.StatusCode) + + w = serve(s, readWrite, http.MethodPost, "/verify", verify) + require.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + + w = serve(s, readWrite, http.MethodGet, "/"+oid, "") + require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) + resp, err = http.Get(w.Header().Get("Location")) + require.NoError(t, err) + got, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + }) + + t.Run("checks permissions", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + readOnly := Access{Repo: "repo", Href: href, CanRead: true} + requireError(t, serve(s, readOnly, http.MethodPost, "/batch", batch(OperationUpload)), http.StatusForbidden) + requireError(t, serve(s, readOnly, http.MethodPut, "/"+oid, content), http.StatusForbidden) + requireError(t, serve(s, readOnly, http.MethodPost, "/verify", verify), http.StatusForbidden) + assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodPost, "/batch", batch(OperationDownload)).Code) + + writeOnly := Access{Repo: "repo", Href: href, CanWrite: true} + requireError(t, serve(s, writeOnly, http.MethodPost, "/batch", batch(OperationDownload)), http.StatusForbidden) + requireError(t, serve(s, writeOnly, http.MethodGet, "/"+oid, ""), http.StatusForbidden) + + _, err := s.Upload(t.Context(), discardLogger, "repo", oid, int64(size), strings.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodGet, "/"+oid, "").Code) + }) + + t.Run("rejects invalid requests", func(t *testing.T) { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + requireError(t, serve(s, readWrite, http.MethodPost, "/batch", "{"), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/batch", `{"operation":"delete"}`), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify", "{"), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodGet, "/not-an-oid", ""), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodDelete, "/"+oid, ""), http.StatusMethodNotAllowed) + }) +} diff --git a/lfs/service.go b/lfs/service.go index 6bfd0e5..d01cde8 100644 --- a/lfs/service.go +++ b/lfs/service.go @@ -19,9 +19,9 @@ import ( ) // Service serves LFS operations for repositories. Callers authenticate and -// authorize each request before calling a method, and respond with the Code -// and Message of a returned *Error. Any other returned error is an internal -// error. +// authorize each request, then either hand it to Serve, or call the operation +// methods and respond with the Code and Message of a returned *Error. Any other +// error returned by an operation method is an internal error. type Service struct { Store Store // Storage receives uploads and decides the transfer mode: content is @@ -138,8 +138,11 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e out.Error = &ObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} default: rawOID, _ := hex.DecodeString(in.OID) // Validated above. - uploadHeader := make(map[string]string, len(endpoint.Header)+1) + uploadHeader := make(map[string]string, len(endpoint.Header)+2) maps.Copy(uploadHeader, endpoint.Header) + // The Git LFS client otherwise sends a Content-Type detected from + // the file content. + uploadHeader["Content-Type"] = "application/octet-stream" // Presigned uploads require the client to send the checksum that // the storage verifies the content against. uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) diff --git a/lfs/service_test.go b/lfs/service_test.go index 830bcad..e3326e7 100644 --- a/lfs/service_test.go +++ b/lfs/service_test.go @@ -290,6 +290,7 @@ func TestService_Batch(t *testing.T) { Href: endpoint.Href + "/" + newOID, Header: map[string]string{ "Authorization": "Basic Zm9vOmJhcg==", + "Content-Type": "application/octet-stream", "x-amz-checksum-sha256": "/jJgjJ71ts9+P5RkgCU/928k9OwGePPQ8H+YRMv/lgE=", }, }, From 239bb24a8945d6de7f2f1748e84caa79ea2ffb79 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 12 Nov 2026 14:35:48 +0000 Subject: [PATCH 4/5] lfs: trace operations and storage calls with OpenTelemetry Service gains a TracerProvider field, defaulting to the global provider, and starts a span for each of Batch, Upload, Download, and Verify, with a child span for every call to the storage backend. Instrumenting the calls in Service traces any plugged-in backend without changes to it. The module depends only on the OpenTelemetry API. Client errors (*Error) are recorded as lfs.error.code and leave the span status unset. Internal errors fail the span. --- lfs/go.mod | 24 ++++++++----- lfs/go.sum | 53 ++++++++++++++++++---------- lfs/service.go | 85 +++++++++++++++++++++++++++++++++++---------- lfs/service_test.go | 72 ++++++++++++++++++++++++++++++++++++++ lfs/tracing.go | 48 +++++++++++++++++++++++++ 5 files changed, 237 insertions(+), 45 deletions(-) create mode 100644 lfs/tracing.go diff --git a/lfs/go.mod b/lfs/go.mod index ba96a41..6fafc18 100644 --- a/lfs/go.mod +++ b/lfs/go.mod @@ -1,6 +1,6 @@ module unknwon.dev/git-lfs-server/lfs -go 1.26 +go 1.26.0 require ( github.com/aws/aws-sdk-go-v2 v1.41.7 @@ -8,7 +8,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 ) require ( @@ -21,20 +24,25 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect github.com/aws/smithy-go v1.25.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.46.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect - golang.org/x/sys v0.31.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + golang.org/x/tools v0.26.0 // indirect ) diff --git a/lfs/go.sum b/lfs/go.sum index a48112f..24aae61 100644 --- a/lfs/go.sum +++ b/lfs/go.sum @@ -24,6 +24,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXq github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/cockroachdb/errors v1.13.0 h1:BoCcJeiP9hpBJDETkX19qi8Tb8So37srSsp3stTaDMQ= @@ -33,16 +35,21 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9D github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -56,24 +63,39 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -89,8 +111,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= @@ -99,16 +121,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lfs/service.go b/lfs/service.go index d01cde8..1a902ea 100644 --- a/lfs/service.go +++ b/lfs/service.go @@ -13,6 +13,8 @@ import ( "strings" "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/lfs/internal/iox" "unknwon.dev/git-lfs-server/lfs/storage" @@ -41,6 +43,10 @@ type Service struct { // its full content, so knowing an OID and its size is not enough to read the // object through a repository the client has write access to. CrossRepoDedup bool + // TracerProvider creates the spans of LFS operations and of the calls to + // storage backends. Nil uses the global provider of + // go.opentelemetry.io/otel. + TracerProvider trace.TracerProvider } // Endpoint describes how the client reaches the transfer endpoints of a @@ -66,7 +72,14 @@ type Redirect struct { // Batch answers a batch request for repo. Callers must check that the actor is // allowed to perform req.Operation before calling it. -func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, endpoint Endpoint, req BatchRequest) (*BatchResponse, error) { +func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, endpoint Endpoint, req BatchRequest) (_ *BatchResponse, err error) { + ctx, span := s.startSpan(ctx, "lfs.Batch", + attribute.String("lfs.repo", repo), + attribute.String("lfs.operation", req.Operation), + attribute.Int("lfs.object_count", len(req.Objects)), + ) + defer func() { endSpan(span, err) }() + if req.Operation != OperationUpload && req.Operation != OperationDownload { return nil, &Error{Code: http.StatusBadRequest, Message: `operation must be "upload" or "download"`} } @@ -82,10 +95,7 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e oids[i] = o.OID } - var ( - existing map[string]Object - err error - ) + var existing map[string]Object switch req.Operation { case OperationDownload: existing, err = s.Store.GetRepoObjects(ctx, repo, oids) @@ -179,7 +189,14 @@ type UploadResult struct { // Upload handles an upload of size bytes for oid to repo. For a // storage.Proxier, it stores the content read from body after verifying its // size and hash. For a storage.Presigner, it does not read body. -func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { +func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (_ *UploadResult, err error) { + ctx, span := s.startSpan(ctx, "lfs.Upload", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", oid), + attribute.Int64("lfs.size", size), + ) + defer func() { endSpan(span, err) }() + if !ValidOID(oid) { return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -205,7 +222,9 @@ func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid str func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend storage.Proxier, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { reader := iox.NewSHA256Reader(iox.NewExactSizeReader(body, size), oid) - uri, err := backend.Put(ctx, oid, reader) + putCtx, span := s.startStorageSpan(ctx, "Put", backend) + uri, err := backend.Put(putCtx, oid, reader) + endSpan(span, err) if err == nil { // Put skips reading when the content is already stored, but the client // must still prove it has the content before the object is linked to @@ -230,9 +249,7 @@ func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend return nil, errors.Wrap(err, "link object") } if storedURI != uri { - if err := backend.Delete(ctx, uri); err != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) - } + s.deleteOrphan(ctx, logger, backend, uri) } return &UploadResult{}, nil } @@ -249,7 +266,9 @@ func (s *Service) uploadPresign(ctx context.Context, backend storage.Presigner, if err = s.Store.InsertPendingObject(ctx, oid); err != nil { return nil, errors.Wrap(err, "insert pending object") } - url, header, err := backend.PresignPut(ctx, oid, size) + presignCtx, span := s.startStorageSpan(ctx, "PresignPut", backend) + url, header, err := backend.PresignPut(presignCtx, oid, size) + endSpan(span, err) if err != nil { return nil, errors.Wrap(err, "presign upload") } @@ -269,7 +288,13 @@ type DownloadResult struct { } // Download handles a download of oid from repo. -func (s *Service) Download(ctx context.Context, repo, oid string) (*DownloadResult, error) { +func (s *Service) Download(ctx context.Context, repo, oid string) (_ *DownloadResult, err error) { + ctx, span := s.startSpan(ctx, "lfs.Download", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", oid), + ) + defer func() { endSpan(span, err) }() + if !ValidOID(oid) { return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -286,7 +311,9 @@ func (s *Service) Download(ctx context.Context, repo, oid string) (*DownloadResu backend := s.backendFor(object.URI) switch b := backend.(type) { case storage.Proxier: - rc, err := b.Open(ctx, object.URI) + openCtx, span := s.startStorageSpan(ctx, "Open", b) + rc, err := b.Open(openCtx, object.URI) + endSpan(span, err) if err != nil { if errors.Is(err, storage.ErrNotFound) { return nil, errors.Wrapf(err, "object %q is recorded but missing in storage", object.URI) @@ -295,7 +322,9 @@ func (s *Service) Download(ctx context.Context, repo, oid string) (*DownloadResu } return &DownloadResult{Content: rc, Size: object.Size}, nil case storage.Presigner: - url, err := b.PresignGet(ctx, object.URI) + presignCtx, span := s.startStorageSpan(ctx, "PresignGet", b) + url, err := b.PresignGet(presignCtx, object.URI) + endSpan(span, err) if err != nil { return nil, errors.Wrap(err, "presign download") } @@ -323,7 +352,14 @@ func (s *Service) backendFor(uri string) storage.Backend { // Verify handles the verification of an upload to repo. A presigned upload // becomes downloadable only after it is verified. -func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) error { +func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) (err error) { + ctx, span := s.startSpan(ctx, "lfs.Verify", + attribute.String("lfs.repo", repo), + attribute.String("lfs.oid", req.OID), + attribute.Int64("lfs.size", req.Size), + ) + defer func() { endSpan(span, err) }() + if !ValidOID(req.OID) { return &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -355,7 +391,9 @@ func (s *Service) verifyProxy(ctx context.Context, repo string, req VerifyReques func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo string, req VerifyRequest) error { uri := backend.URI(req.OID) - headSize, err := backend.Head(ctx, uri) + headCtx, span := s.startStorageSpan(ctx, "Head", backend) + headSize, err := backend.Head(headCtx, uri) + endSpan(span, err) if err != nil { if errors.Is(err, storage.ErrNotFound) { return &Error{Code: http.StatusUnprocessableEntity, Message: "object does not exist in storage"} @@ -374,9 +412,18 @@ func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backen return errors.Wrap(err, "mark object verified") } if storedURI != uri { - if err := backend.Delete(ctx, uri); err != nil { - logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) - } + s.deleteOrphan(ctx, logger, backend, uri) } return nil } + +// deleteOrphan deletes the content at uri that was stored again while the +// object was already recorded with another URI. +func (s *Service) deleteOrphan(ctx context.Context, logger *slog.Logger, backend storage.Backend, uri string) { + ctx, span := s.startStorageSpan(ctx, "Delete", backend) + err := backend.Delete(ctx, uri) + endSpan(span, err) + if err != nil { + logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) + } +} diff --git a/lfs/service_test.go b/lfs/service_test.go index e3326e7..e173ec5 100644 --- a/lfs/service_test.go +++ b/lfs/service_test.go @@ -18,6 +18,10 @@ import ( "github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" "unknwon.dev/git-lfs-server/lfs/storage" ) @@ -373,3 +377,71 @@ func TestService_Presign(t *testing.T) { assert.Equal(t, crossRepoDedup, result.Redirect == nil, "crossRepoDedup=%v", crossRepoDedup) } } + +// failingStore fails every lookup. +type failingStore struct{ *memStore } + +func (failingStore) GetRepoObjects(context.Context, string, []string) (map[string]Object, error) { + return nil, errors.New("store is down") +} + +func TestService_Tracing(t *testing.T) { + ctx := t.Context() + content := "traced content" + oid := sha256Hex(content) + size := int64(len(content)) + + newService := func(store Store) (*Service, *tracetest.SpanRecorder) { + recorder := tracetest.NewSpanRecorder() + return &Service{ + Store: store, + Storage: newFilesystemBackend(t, "fs", "file://"), + TracerProvider: sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)), + }, recorder + } + spansByName := func(recorder *tracetest.SpanRecorder) map[string]sdktrace.ReadOnlySpan { + spans := map[string]sdktrace.ReadOnlySpan{} + for _, span := range recorder.Ended() { + spans[span.Name()] = span + } + return spans + } + + t.Run("storage calls are children of the operation", func(t *testing.T) { + s, recorder := newService(newMemStore()) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + require.NoError(t, err) + + spans := spansByName(recorder) + require.Contains(t, spans, "lfs.Upload") + require.Contains(t, spans, "storage.Put") + upload, put := spans["lfs.Upload"], spans["storage.Put"] + assert.Equal(t, upload.SpanContext().SpanID(), put.Parent().SpanID()) + assert.Contains(t, upload.Attributes(), attribute.String("lfs.repo", "repo1")) + assert.Contains(t, upload.Attributes(), attribute.String("lfs.oid", oid)) + assert.Contains(t, put.Attributes(), attribute.String("lfs.storage.name", "fs")) + assert.Contains(t, put.Attributes(), attribute.String("lfs.storage.type", "filesystem")) + }) + + t.Run("client errors do not fail the span", func(t *testing.T) { + s, recorder := newService(newMemStore()) + _, err := s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) + + download := spansByName(recorder)["lfs.Download"] + require.NotNil(t, download) + assert.Equal(t, codes.Unset, download.Status().Code) + assert.Contains(t, download.Attributes(), attribute.Int("lfs.error.code", http.StatusNotFound)) + }) + + t.Run("internal errors fail the span", func(t *testing.T) { + s, recorder := newService(failingStore{newMemStore()}) + _, err := s.Download(ctx, "repo1", oid) + require.Error(t, err) + + download := spansByName(recorder)["lfs.Download"] + require.NotNil(t, download) + assert.Equal(t, codes.Error, download.Status().Code) + assert.Contains(t, download.Status().Description, "store is down") + }) +} diff --git a/lfs/tracing.go b/lfs/tracing.go new file mode 100644 index 0000000..e670e10 --- /dev/null +++ b/lfs/tracing.go @@ -0,0 +1,48 @@ +package lfs + +import ( + "context" + + "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +const tracerName = "unknwon.dev/git-lfs-server/lfs" + +func (s *Service) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + tp := s.TracerProvider + if tp == nil { + tp = otel.GetTracerProvider() + } + return tp.Tracer(tracerName).Start(ctx, name, trace.WithAttributes(attrs...)) +} + +// startStorageSpan starts the span of a call to backend, so that every backend +// plugged into the Service is traced the same way. +func (s *Service) startStorageSpan(ctx context.Context, method string, backend storage.Backend) (context.Context, trace.Span) { + return s.startSpan(ctx, "storage."+method, + attribute.String("lfs.storage.name", backend.Name()), + attribute.String("lfs.storage.type", string(backend.Type())), + ) +} + +// endSpan ends span with the outcome of err. A client error is the expected +// answer to a bad request rather than a failure of the operation, so it is +// recorded as an attribute and leaves the span status unset. +func endSpan(span trace.Span, err error) { + if err != nil { + var lfsErr *Error + if errors.As(err, &lfsErr) { + span.SetAttributes(attribute.Int("lfs.error.code", lfsErr.Code)) + } else { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + } + span.End() +} From 8853467f3d4beeb85dd25998a17e55b0c238e427 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 12 Nov 2026 14:35:49 +0000 Subject: [PATCH 5/5] lfsd: export OpenTelemetry traces over OTLP Add a [tracing] section (ENABLED, ENDPOINT, SAMPLE_RATIO) that installs a global tracer provider with a parent-based ratio sampler and W3C trace context propagation. Disabled by default, which keeps every tracer no-op. - HTTP: otelhttp wraps the router, spans are named after the matched LFS route, and /healthz is not traced. - Auth: an authorize span records cache hits and the permission, never credentials. Forge API calls are client spans, and the trace context is not sent to the forge. - Database: the GORM OpenTelemetry plugin traces queries without bound values. Startup queries share a database.New trace. - Janitor: each sweep is a janitor.Sweep trace with deleted and failed counts per pass. - Logs: records in a traced context carry traceID and spanID, and error records are attached to the span as events. --- README.md | 22 ++++ cmd/lfsd/auth.go | 13 ++- cmd/lfsd/config.go | 5 + cmd/lfsd/logging.go | 2 +- cmd/lfsd/main.go | 55 ++++++++-- cmd/lfsd/main_test.go | 37 +++++++ go.mod | 51 ++++++++-- go.sum | 152 ++++++++++++++++++++++++---- internal/database/database.go | 16 ++- internal/embed/config.ini | 11 ++ internal/forge/forge.go | 15 +++ internal/forge/github.go | 2 +- internal/forge/github_test.go | 48 +++++++++ internal/forge/gitlab.go | 2 +- internal/janitor/janitor.go | 27 +++-- internal/logx/trace_handler.go | 61 +++++++++++ internal/logx/trace_handler_test.go | 59 +++++++++++ internal/tracing/tracing.go | 69 +++++++++++++ 18 files changed, 595 insertions(+), 52 deletions(-) create mode 100644 cmd/lfsd/main_test.go create mode 100644 internal/logx/trace_handler.go create mode 100644 internal/logx/trace_handler_test.go create mode 100644 internal/tracing/tracing.go diff --git a/README.md b/README.md index dcca796..48b73b5 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,28 @@ mux.HandleFunc("/{repo}/info/lfs/objects/", func(w http.ResponseWriter, r *http. The lower-level `Batch`, `Upload`, `Download`, and `Verify` methods are available for servers that need their own HTTP handling. +The module depends only on the OpenTelemetry API. `Service.TracerProvider` receives the spans of LFS operations (`lfs.Batch`, `lfs.Upload`, `lfs.Download`, `lfs.Verify`) and of every call to the storage backend (`storage.Put`, `storage.Head`, etc.), so custom backends are traced without extra work. When it is nil, the global provider from `go.opentelemetry.io/otel` is used. + +## Tracing + +lfsd exports OpenTelemetry traces over OTLP/HTTP when tracing is enabled: + +```ini +[tracing] +ENABLED = true +ENDPOINT = http://otel-collector:4318/v1/traces +SAMPLE_RATIO = 0.1 +``` + +Leave `ENDPOINT` empty to configure the exporter with the standard `OTEL_EXPORTER_OTLP_*` environment variables, e.g., `OTEL_EXPORTER_OTLP_HEADERS` for an API key. `OTEL_SERVICE_NAME` and `OTEL_RESOURCE_ATTRIBUTES` override the default `service.name=lfsd` and `service.version` (the build commit). Requests that carry a sampled W3C `traceparent` are always sampled, so lfsd joins the traces of a reverse proxy or a client that propagates context. + +Each LFS request produces a span named after its route, e.g., `POST /{host}/{**}/info/lfs/objects/batch`, with these children: + +- `authorize`: whether the permission came from the cache, with the forge API call as a child. Credentials are never recorded, and the trace context is not sent to the forge. +- `lfs.Batch`, `lfs.Upload`, `lfs.Download`, or `lfs.Verify`, with the database queries and storage calls of the operation. + +The janitor records a `janitor.Sweep` trace per sweep. `/healthz` is not traced. Log records written during a traced request carry its `traceID` and `spanID`, and error logs are attached to the span as events. + ## Local development This project uses [moon](https://moonrepo.dev/) as its task runner. Install it with Homebrew: diff --git a/cmd/lfsd/auth.go b/cmd/lfsd/auth.go index 2181d82..f278fdf 100644 --- a/cmd/lfsd/auth.go +++ b/cmd/lfsd/auth.go @@ -12,6 +12,8 @@ import ( "github.com/cockroachdb/errors" "github.com/flamego/cache" "github.com/flamego/flamego" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/logx" @@ -21,9 +23,12 @@ import ( func authorize(forges map[string]forge.Provider) flamego.Handler { return func(c flamego.Context, logger *logx.Logger, cache cache.Cache) { logger = logger.Scoped("authorize") - ctx := c.Request().Context() - host := hostFromContext(c) + // The span never carries credentials: the forge decides access, and only + // its outcome is recorded. + ctx, span := otel.Tracer("unknwon.dev/git-lfs-server/cmd/lfsd").Start(c.Request().Context(), "authorize") + defer span.End() + provider, ok := forges[host] if !ok { lfs.WriteError(c.ResponseWriter(), http.StatusBadRequest, "unsupported forge host") @@ -52,6 +57,7 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { key := authCacheKey(host, repo, username, token) if v, err := cache.Get(ctx, key); err == nil { if perm, ok := v.(forge.Permission); ok { + span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", true), attribute.String("lfs.permission", string(perm))) c.Map(perm) return } @@ -60,9 +66,11 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { logger.WarnContext(ctx, "Auth cache lookup failed", "error", err) } + span.SetAttributes(attribute.Bool("lfs.auth.cache_hit", false)) perm, rawTTL, err := provider.Authorize(ctx, logger.Scoped("forge"), repo, username, token) if err != nil { if errors.Is(err, forge.ErrTokenInvalid) { + span.SetAttributes(attribute.Bool("lfs.auth.token_invalid", true)) lfs.WriteError(c.ResponseWriter(), http.StatusForbidden, "token is invalid or lacks repository access") } else { logger.ErrorContext(ctx, "Failed to verify repository permissions", "repo", repo, "error", err) @@ -81,6 +89,7 @@ func authorize(forges map[string]forge.Provider) flamego.Handler { } } + span.SetAttributes(attribute.String("lfs.permission", string(perm))) c.Map(perm) } } diff --git a/cmd/lfsd/config.go b/cmd/lfsd/config.go index 9a22d35..c7c6461 100644 --- a/cmd/lfsd/config.go +++ b/cmd/lfsd/config.go @@ -10,6 +10,7 @@ import ( "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/embed" "unknwon.dev/git-lfs-server/internal/forge" + "unknwon.dev/git-lfs-server/internal/tracing" "unknwon.dev/git-lfs-server/lfs/storage" ) @@ -22,6 +23,7 @@ type Config struct { Forges map[Host]forge.Provider Storages map[Host]storage.Backend Log LogConfig + Tracing tracing.Config } type ServerConfig struct { @@ -62,6 +64,9 @@ func loadConfig(customPath string) (*Config, error) { if err := f.Section("log").MapTo(&c.Log); err != nil { return nil, errors.Wrap(err, `map [log] config`) } + if err := f.Section("tracing").MapTo(&c.Tracing); err != nil { + return nil, errors.Wrap(err, `map [tracing] config`) + } backendsByName := make(map[string]storage.Backend) for _, s := range f.Sections() { diff --git a/cmd/lfsd/logging.go b/cmd/lfsd/logging.go index d9b8cc2..ab25805 100644 --- a/cmd/lfsd/logging.go +++ b/cmd/lfsd/logging.go @@ -44,5 +44,5 @@ func setupLogging(cfg LogConfig) *logx.Logger { Foreground(lipgloss.Color("226")) handler.SetStyles(styles) - return logx.New(handler) + return logx.New(logx.NewTraceHandler(handler)) } diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index dc8cc0d..851dd32 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -12,12 +12,16 @@ import ( "github.com/flamego/cache" "github.com/flamego/flamego" "github.com/sourcegraph/conc" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/forge" "unknwon.dev/git-lfs-server/internal/janitor" "unknwon.dev/git-lfs-server/internal/logx" "unknwon.dev/git-lfs-server/internal/strx" + "unknwon.dev/git-lfs-server/internal/tracing" "unknwon.dev/git-lfs-server/lfs" ) @@ -39,6 +43,15 @@ func main() { ctx := context.Background() logger := setupLogging(config.Log) + // Set up before anything that creates spans, e.g., the database plugin. + shutdownTracing, err := tracing.Setup(ctx, config.Tracing, buildCommit) + if err != nil { + logger.FatalContext(ctx, "Failed to set up tracing", "error", err) + } + if config.Tracing.Enabled { + logger.InfoContext(ctx, "Tracing enabled", "sampleRatio", config.Tracing.SampleRatio) + } + for host, provider := range config.Forges { logger.InfoContext(ctx, "Forge initialized", "host", host, "type", provider.Type()) if provider.Type() == forge.TypeSkipAuth { @@ -71,19 +84,23 @@ func main() { } } serve := serveLFS(config.Server.ExternalURL, services) - f.Group("/{host}/{**}/info/lfs/objects", func() { - f.Post("/batch", serve) - f.Get("/{oid}", serve) - f.Put("/{oid}", serve) - f.Post("/verify", serve) - }, authorize(config.Forges)) + auth := authorize(config.Forges) + const objectsRoute = "/{host}/{**}/info/lfs/objects" + f.Group(objectsRoute, func() { + f.Post("/batch", traceRoute(objectsRoute+"/batch"), auth, serve) + f.Get("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) + f.Put("/{oid}", traceRoute(objectsRoute+"/{oid}"), auth, serve) + f.Post("/verify", traceRoute(objectsRoute+"/verify"), auth, serve) + }) ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() srv := &http.Server{ - Addr: config.Server.ListenAddress, - Handler: f, + Addr: config.Server.ListenAddress, + Handler: otelhttp.NewHandler(f, "lfsd", + otelhttp.WithFilter(func(r *http.Request) bool { return r.URL.Path != "/healthz" }), + ), } var routines conc.WaitGroup @@ -108,11 +125,31 @@ func main() { "buildDate", buildDate, "buildCommit", buildCommit, "listenAddr", config.Server.ListenAddress, ) - if r := routines.WaitAndRecover(); r != nil { + r := routines.WaitAndRecover() + // Flush the spans of the requests that finished during shutdown. The + // signal context is already canceled, so the flush gets its own. + if err := shutdownTracing(context.Background()); err != nil { + logger.ErrorContext(ctx, "Failed to shut down tracing", "error", err) + } + if r != nil { logger.FatalContext(ctx, "Server panicked", "panic", r.Value, "stack", string(r.Stack)) } } +// traceRoute names the server span of the request after route, because the +// otelhttp handler wraps the router and does not know which route matched. +func traceRoute(route string) flamego.Handler { + return func(c flamego.Context) { + span := trace.SpanFromContext(c.Request().Context()) + span.SetName(c.Request().Method + " " + route) + span.SetAttributes( + attribute.String("http.route", route), + attribute.String("lfs.forge.host", hostFromContext(c)), + attribute.String("lfs.repo", repoNameFromContext(c)), + ) + } +} + func hostFromContext(c flamego.Context) string { return strings.ToLower(c.Param("host")) } diff --git a/cmd/lfsd/main_test.go b/cmd/lfsd/main_test.go new file mode 100644 index 0000000..d00c575 --- /dev/null +++ b/cmd/lfsd/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/flamego/flamego" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestTraceRoute(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + + const route = "/{host}/{**}/info/lfs/objects/{oid}" + f := flamego.New() + f.Get(route, traceRoute(route), func(c flamego.Context) { + c.ResponseWriter().WriteHeader(http.StatusOK) + }) + handler := otelhttp.NewHandler(f, "lfsd", otelhttp.WithTracerProvider(tp)) + + r := httptest.NewRequest(http.MethodGet, "/GitHub.com/Org/Repo/info/lfs/objects/abc", nil) + handler.ServeHTTP(httptest.NewRecorder(), r) + + spans := recorder.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "GET "+route, spans[0].Name()) + assert.Contains(t, spans[0].Attributes(), attribute.String("http.route", route)) + assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.forge.host", "github.com")) + assert.Contains(t, spans[0].Attributes(), attribute.String("lfs.repo", "github.com/org/repo")) +} diff --git a/go.mod b/go.mod index bbb2b84..d26f048 100644 --- a/go.mod +++ b/go.mod @@ -11,17 +11,27 @@ require ( github.com/lib/pq v1.12.3 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 go.bobheadxi.dev/streamline v1.3.2 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 gopkg.in/ini.v1 v1.67.2 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 + gorm.io/plugin/opentelemetry v0.1.16 unknwon.dev/git-lfs-server/lfs v0.0.0-00010101000000-000000000000 ) require ( bitbucket.org/creachadair/shell v0.0.7 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect github.com/alecthomas/participle/v2 v2.1.4 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect @@ -35,6 +45,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect github.com/charmbracelet/log v1.0.0 // indirect @@ -48,14 +60,20 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect github.com/cockroachdb/redact v1.1.8 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/djherbis/buffer v1.2.0 // indirect github.com/djherbis/nio/v3 v3.0.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/getsentry/sentry-go v0.46.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-logfmt/logfmt v0.6.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/itchyny/gojq v0.12.14 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -64,6 +82,7 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect @@ -71,18 +90,32 @@ require ( github.com/mattn/go-runewidth v0.0.23 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.opentelemetry.io/otel v1.11.0 // indirect - go.opentelemetry.io/otel/trace v1.11.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/clickhouse v0.7.0 // indirect + gorm.io/driver/mysql v1.5.7 // indirect ) replace unknwon.dev/git-lfs-server/lfs => ./lfs diff --git a/go.sum b/go.sum index 62a7ab2..3c49c1d 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,20 @@ charm.land/lipgloss/v2 v2.0.3 h1:yM2zJ4Cf5Y51b7RHIwioil4ApI/aypFXXVHSwlM6RzU= charm.land/lipgloss/v2 v2.0.3/go.mod h1:7myLU9iG/3xluAWzpY/fSxYYHCgoKTie7laxk6ATwXA= charm.land/log/v2 v2.0.0 h1:SY3Cey7ipx86/MBXQHwsguOT6X1exT94mmJRdzTNs+s= charm.land/log/v2 v2.0.0/go.mod h1:c3cZSRqm20qUVVAR1WmS/7ab8bgha3C6G7DjPcaVZz0= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= @@ -36,6 +44,10 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= @@ -66,7 +78,6 @@ github.com/cockroachdb/redact v1.1.8 h1:8eVLLj6juKxiKrAEw2b8cJvNqWq++U8WOfQFuL7K github.com/cockroachdb/redact v1.1.8/go.mod h1:GceHHpJ0rMDpYARL5In88Alq/xMBUtVlz7Qxix6ZVkw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= @@ -75,6 +86,8 @@ github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4 github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flamego/cache v1.5.1 h1:2B4QhLFV7je0oUMCVKsAGAT+OyDHlXhozOoUffm+O3s= github.com/flamego/cache v1.5.1/go.mod h1:cTWYm/Ls35KKHo8vwcKgTlJUNXswEhzFWqVCTFzj24s= github.com/flamego/flamego v1.9.7 h1:x3gkGOALg+HkpqFngkxQ3ZMC2vIa3Kze/WIpYTU2L0k= @@ -85,18 +98,37 @@ github.com/getsentry/sentry-go v0.46.1 h1:mZyQFaQYkPxAdDG4HR8gDg6j4CnKYVWt4TF92N github.com/getsentry/sentry-go v0.46.1/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE= github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/autogold/v2 v2.2.1 h1:JPUXuZQGkcQMv7eeDXuNMovjfoRYaa0yVcm+F3voaGY= github.com/hexops/autogold/v2 v2.2.1/go.mod h1:IJwxtUfj1BGLm0YsR/k+dIxYi6xbeLjqGke2bzcOTMI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -123,8 +155,14 @@ github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RM github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= @@ -137,18 +175,25 @@ github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -157,6 +202,10 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= @@ -166,66 +215,118 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4= go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg= -go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= -go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= -go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= -go.opentelemetry.io/otel/sdk v1.11.0/go.mod h1:REusa8RsyKaq0OlyangWXaw97t2VogoO4SSEeKkSTAk= -go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI= -go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d h1:Ns9kd1Rwzw7t0BR8XMphenji4SmIoNZPn8zhYmaVKP8= go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3l++MlthCm+koNi0tcUCX3anayogF0Pa/sp24k= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c h1:ofk4SEqvpf6M5VWh88xRp42IYgdVOh7gffebnDj3OqU= +google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c/go.mod h1:OzYcNQzaMRKcOFRMtyLzG/cfW1SmS2eh+7UmOLLFblk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c h1:XXChExL+Tm7Ubp0x0VXUEncPLHrkXIgoVWqnxmKNbAA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= @@ -233,9 +334,18 @@ gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/clickhouse v0.7.0 h1:BCrqvgONayvZRgtuA6hdya+eAW5P2QVagV3OlEp1vtA= +gorm.io/driver/clickhouse v0.7.0/go.mod h1:TmNo0wcVTsD4BBObiRnCahUgHJHjBIwuRejHwYt3JRs= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +gorm.io/plugin/opentelemetry v0.1.16 h1:Kypj2YYAliJqkIczDZDde6P6sFMhKSlG5IpngMFQGpc= +gorm.io/plugin/opentelemetry v0.1.16/go.mod h1:P3RmTeZXT+9n0F1ccUqR5uuTvEXDxF8k2UpO7mTIB2Y= mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= diff --git a/internal/database/database.go b/internal/database/database.go index 8b8eb16..18133d6 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -7,8 +7,10 @@ import ( "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" "gorm.io/driver/postgres" "gorm.io/gorm" + gormtracing "gorm.io/plugin/opentelemetry/tracing" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -57,6 +59,10 @@ func (d *DB) Ping(ctx context.Context) error { // New opens a PostgreSQL connection via GORM and configures the connection pool. func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { + // Groups the startup queries into one trace. + ctx, span := otel.Tracer("unknwon.dev/git-lfs-server/internal/database").Start(ctx, "database.New") + defer span.End() + dsn := fmt.Sprintf( "postgres://%s:%s@%s:%d/%s?sslmode=%s&application_name=lfsd", url.QueryEscape(cfg.User), @@ -84,6 +90,12 @@ func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { return nil, errors.Wrap(err, "open database connection") } + // Leave bound values out of spans so that they never record request data, + // e.g., object URIs. + if err = gormDB.Use(gormtracing.NewPlugin(gormtracing.WithoutQueryVariables(), gormtracing.WithoutMetrics())); err != nil { + return nil, errors.Wrap(err, "register tracing plugin") + } + sqlDB, err := gormDB.DB() if err != nil { return nil, errors.Wrap(err, "get underlying *sql.DB") @@ -109,14 +121,14 @@ func New(ctx context.Context, logger *logx.Logger, cfg Config) (*DB, error) { } logger.DebugContext(ctx, "Auto-migrating database tables") - if err = gormDB.AutoMigrate(tables...); err != nil { + if err = gormDB.WithContext(ctx).AutoMigrate(tables...); err != nil { return nil, errors.Wrap(err, "auto-migrate database tables") } // AutoMigrate doesn't emit CHECK constraints. Add ours idempotently: // Postgres has no "ADD CONSTRAINT IF NOT EXISTS" form for CHECK, so we // catch duplicate_object via SQLSTATE. - if err = gormDB.Exec(` + if err = gormDB.WithContext(ctx).Exec(` DO $$ BEGIN ALTER TABLE objects ADD CONSTRAINT check_verified_consistency CHECK ( (verified_at IS NULL AND size IS NULL AND object_uri IS NULL) diff --git a/internal/embed/config.ini b/internal/embed/config.ini index 366e8c4..5dd4097 100644 --- a/internal/embed/config.ini +++ b/internal/embed/config.ini @@ -85,3 +85,14 @@ TEMP_DIR = %(ROOT)s/.tmp [log] ; Default log level: debug, info, warn, error. LEVEL = debug + +[tracing] +; Set true to export OpenTelemetry traces over OTLP/HTTP. +ENABLED = false +; URL of the OTLP/HTTP traces endpoint, e.g., "http://localhost:4318/v1/traces". +; Leave empty to use the standard OTEL_EXPORTER_OTLP_* environment variables, +; which also configure headers, timeouts, and TLS. +ENDPOINT = +; Fraction of new traces to sample, from 0 to 1. Requests that arrive with a +; sampled trace context are always sampled. +SAMPLE_RATIO = 1 diff --git a/internal/forge/forge.go b/internal/forge/forge.go index f06067e..0b52b19 100644 --- a/internal/forge/forge.go +++ b/internal/forge/forge.go @@ -2,10 +2,13 @@ package forge import ( "context" + "net/http" "strings" "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/propagation" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -69,6 +72,18 @@ type Provider interface { Authorize(ctx context.Context, logger *logx.Logger, repo, username, token string) (Permission, time.Duration, error) } +// newHTTPClient returns the client for forge API calls. Its calls are traced as +// children of the caller's span, but the trace context is not sent to the +// forge, which is outside of our trust boundary. +func newHTTPClient() *http.Client { + return &http.Client{ + Timeout: 30 * time.Second, + Transport: otelhttp.NewTransport(http.DefaultTransport, + otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator()), + ), + } +} + // ErrTokenInvalid signals that the supplied token is invalid or lacks the // required repository access. var ErrTokenInvalid = errors.New("forge: token is invalid or lacks repository access") diff --git a/internal/forge/github.go b/internal/forge/github.go index 5e9ed5c..a6d36a7 100644 --- a/internal/forge/github.go +++ b/internal/forge/github.go @@ -29,7 +29,7 @@ func NewGitHubProvider(host string, allowlist *RepoAllowlist) *GitHubProvider { return &GitHubProvider{ host: host, apiBase: githubAPIBase(host), - client: &http.Client{Timeout: 30 * time.Second}, + client: newHTTPClient(), allowlist: allowlist, } } diff --git a/internal/forge/github_test.go b/internal/forge/github_test.go index e7ef76c..81d0675 100644 --- a/internal/forge/github_test.go +++ b/internal/forge/github_test.go @@ -2,10 +2,18 @@ package forge import ( "context" + "net/http" + "net/http/httptest" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/logx" ) @@ -57,3 +65,43 @@ func TestGitHubTokenTTL(t *testing.T) { assert.Negative(t, githubTokenTTL(ctx, logger, expiry)) }) } + +func TestGitHubProvider_Tracing(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + // The forge client uses the global provider and must ignore the global + // propagator, so install both to prove that nothing is propagated. + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + const token = "ghp_secret" + var gotHeader http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeader = r.Header.Clone() + _, _ = w.Write([]byte(`{"permissions":{"pull":true,"push":true}}`)) + })) + t.Cleanup(server.Close) + + provider := NewGitHubProvider("github.example.com", nil) + provider.apiBase = server.URL + + ctx, parent := tp.Tracer("test").Start(t.Context(), "parent") + perm, _, err := provider.Authorize(ctx, logx.NewNoopLogger(), "org/repo", "x-access-token", token) + parent.End() + require.NoError(t, err) + assert.Equal(t, PermissionWrite, perm) + + assert.Empty(t, gotHeader.Get("traceparent"), "trace context must not be sent to the forge") + + var client sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + if span.SpanKind() == trace.SpanKindClient { + client = span + } + } + require.NotNil(t, client, "forge call must be traced") + assert.Equal(t, parent.SpanContext().SpanID(), client.Parent().SpanID()) + for _, a := range client.Attributes() { + assert.NotContains(t, a.Value.String(), token, "attribute %s", a.Key) + } +} diff --git a/internal/forge/gitlab.go b/internal/forge/gitlab.go index 0f77da3..67d223c 100644 --- a/internal/forge/gitlab.go +++ b/internal/forge/gitlab.go @@ -28,7 +28,7 @@ func (p *GitLabProvider) Allow(repo string) bool { return p.allowlist.Allow(repo func NewGitLabProvider(host string, allowlist *RepoAllowlist) *GitLabProvider { return &GitLabProvider{ baseURL: "https://" + host, - client: &http.Client{Timeout: 30 * time.Second}, + client: newHTTPClient(), allowlist: allowlist, } } diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index 2332c5f..912517a 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -6,6 +6,9 @@ import ( "time" "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "unknwon.dev/git-lfs-server/internal/database" "unknwon.dev/git-lfs-server/internal/logx" @@ -61,17 +64,25 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) { // Sweep runs both cleanup passes once. Errors are logged but not returned. // The next tick will retry. func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { - if objects, err := j.db.SweepOrphanObjects(ctx, orphanAge, sweepBatchSize); err != nil { - logger.ErrorContext(ctx, "Failed to sweep orphan objects", "error", err) + tracer := otel.Tracer("unknwon.dev/git-lfs-server/internal/janitor") + ctx, span := tracer.Start(ctx, "janitor.Sweep") + defer span.End() + + orphanCtx, orphanSpan := tracer.Start(ctx, "janitor.SweepOrphans") + if objects, err := j.db.SweepOrphanObjects(orphanCtx, orphanAge, sweepBatchSize); err != nil { + logger.ErrorContext(orphanCtx, "Failed to sweep orphan objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(ctx, logger.Scoped("orphan"), objects) + j.deleteObjects(orphanCtx, logger.Scoped("orphan"), objects) } + orphanSpan.End() - if objects, err := j.db.SweepUnreferencedObjects(ctx, sweepBatchSize); err != nil { - logger.ErrorContext(ctx, "Failed to sweep unreferenced objects", "error", err) + unreferencedCtx, unreferencedSpan := tracer.Start(ctx, "janitor.SweepUnreferenced") + if objects, err := j.db.SweepUnreferencedObjects(unreferencedCtx, sweepBatchSize); err != nil { + logger.ErrorContext(unreferencedCtx, "Failed to sweep unreferenced objects", "error", err) } else if len(objects) > 0 { - j.deleteObjects(ctx, logger.Scoped("unreferenced"), objects) + j.deleteObjects(unreferencedCtx, logger.Scoped("unreferenced"), objects) } + unreferencedSpan.End() } // deleteObjects deletes the given list of objects from the storage backends. @@ -106,6 +117,10 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object deletedCount++ } } + trace.SpanFromContext(ctx).SetAttributes( + attribute.Int("janitor.deleted_count", deletedCount), + attribute.Int("janitor.failed_count", failedCount), + ) logFn := logger.InfoContext if failedCount > 0 { logFn = logger.ErrorContext diff --git a/internal/logx/trace_handler.go b/internal/logx/trace_handler.go new file mode 100644 index 0000000..30a11b6 --- /dev/null +++ b/internal/logx/trace_handler.go @@ -0,0 +1,61 @@ +package logx + +import ( + "context" + "log/slog" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// traceHandler wraps a slog.Handler to correlate records with the span in their +// context. Records get the trace and span IDs as attributes, and error records +// are added to the span as events and fail it, so a trace shows why it failed +// without searching the logs. +type traceHandler struct { + handler slog.Handler +} + +// NewTraceHandler returns a handler that correlates records with spans before +// passing them to handler. +func NewTraceHandler(handler slog.Handler) slog.Handler { + return &traceHandler{handler: handler} +} + +func (h *traceHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.handler.Enabled(ctx, level) +} + +func (h *traceHandler) Handle(ctx context.Context, r slog.Record) error { + span := trace.SpanFromContext(ctx) + spanContext := span.SpanContext() + if !spanContext.IsValid() { + return h.handler.Handle(ctx, r) + } + + if r.Level >= slog.LevelError && span.IsRecording() { + attrs := make([]attribute.KeyValue, 0, r.NumAttrs()) + r.Attrs(func(a slog.Attr) bool { + attrs = append(attrs, attribute.String(a.Key, a.Value.String())) + return true + }) + span.AddEvent(r.Message, trace.WithAttributes(attrs...)) + span.SetStatus(codes.Error, r.Message) + } + + r = r.Clone() + r.AddAttrs( + slog.String("traceID", spanContext.TraceID().String()), + slog.String("spanID", spanContext.SpanID().String()), + ) + return h.handler.Handle(ctx, r) +} + +func (h *traceHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &traceHandler{handler: h.handler.WithAttrs(attrs)} +} + +func (h *traceHandler) WithGroup(name string) slog.Handler { + return &traceHandler{handler: h.handler.WithGroup(name)} +} diff --git a/internal/logx/trace_handler_test.go b/internal/logx/trace_handler_test.go new file mode 100644 index 0000000..dedc5cb --- /dev/null +++ b/internal/logx/trace_handler_test.go @@ -0,0 +1,59 @@ +package logx + +import ( + "bytes" + "encoding/json" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func TestTraceHandler(t *testing.T) { + newLogger := func() (*Logger, *bytes.Buffer) { + var buf bytes.Buffer + return New(NewTraceHandler(slog.NewJSONHandler(&buf, nil))).Scoped("test"), &buf + } + decode := func(t *testing.T, buf *bytes.Buffer) map[string]any { + t.Helper() + var record map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &record)) + return record + } + + t.Run("records outside of a span are unchanged", func(t *testing.T) { + logger, buf := newLogger() + logger.InfoContext(t.Context(), "Hello") + record := decode(t, buf) + assert.NotContains(t, record, "traceID") + assert.NotContains(t, record, "spanID") + }) + + t.Run("records in a span carry its IDs and only errors fail it", func(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + ctx, span := tp.Tracer("test").Start(t.Context(), "span") + + logger, buf := newLogger() + logger.WarnContext(ctx, "Slow") + record := decode(t, buf) + assert.Equal(t, span.SpanContext().TraceID().String(), record["traceID"]) + assert.Equal(t, span.SpanContext().SpanID().String(), record["spanID"]) + + logger.ErrorContext(ctx, "Failed", "error", "boom") + span.End() + + ended := recorder.Ended() + require.Len(t, ended, 1) + assert.Equal(t, codes.Error, ended[0].Status().Code) + assert.Equal(t, "[test] Failed", ended[0].Status().Description) + events := ended[0].Events() + require.Len(t, events, 1, "only the error record becomes an event") + assert.Equal(t, "[test] Failed", events[0].Name) + assert.Equal(t, "boom", events[0].Attributes[0].Value.AsString()) + }) +} diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go new file mode 100644 index 0000000..6fa11b9 --- /dev/null +++ b/internal/tracing/tracing.go @@ -0,0 +1,69 @@ +// Package tracing sets up OpenTelemetry tracing for the process. +package tracing + +import ( + "context" + + "github.com/cockroachdb/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" +) + +// Config holds the [tracing] section of the config. +type Config struct { + Enabled bool `ini:"ENABLED"` + // Endpoint is the URL of an OTLP/HTTP traces endpoint. Empty falls back to + // the OTEL_EXPORTER_OTLP_* environment variables. + Endpoint string `ini:"ENDPOINT"` + // SampleRatio is the fraction of new traces to sample. Requests that carry + // a sampled trace context are always sampled. + SampleRatio float64 `ini:"SAMPLE_RATIO"` +} + +// Setup installs the global tracer provider and propagator when tracing is +// enabled, and returns the function that flushes and stops them. Instrumented +// code uses the global provider, so nothing is recorded when tracing is +// disabled. +func Setup(ctx context.Context, cfg Config, serviceVersion string) (shutdown func(context.Context) error, err error) { + if !cfg.Enabled { + return func(context.Context) error { return nil }, nil + } + + var opts []otlptracehttp.Option + if cfg.Endpoint != "" { + opts = append(opts, otlptracehttp.WithEndpointURL(cfg.Endpoint)) + } + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, errors.Wrap(err, "create OTLP trace exporter") + } + + res, err := resource.New(ctx, + resource.WithTelemetrySDK(), + resource.WithAttributes( + semconv.ServiceName("lfsd"), + semconv.ServiceVersion(serviceVersion), + ), + // Applied last so that OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES + // override the defaults above. + resource.WithFromEnv(), + ) + if err != nil { + return nil, errors.Wrap(err, "create resource") + } + + provider := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))), + ) + otel.SetTracerProvider(provider) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + return func(ctx context.Context) error { + return errors.Wrap(provider.Shutdown(ctx), "shut down tracer provider") + }, nil +}