From 9697b22696ea1e2fe3cafe6273a0ac0274e1dcc9 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Fri, 13 Nov 2026 20:03:32 +0000 Subject: [PATCH 1/2] lfs: require proof of upload for presigned storage An OID and its size are public in pointer files, so knowing them must not link an object to a repository. Presigned verify only checked that the object existed at its shared location, so a writer of one repository could skip the upload and gain an object stored by another repository. Each presigned upload now goes to a one-time location under uploads/, named by an upload ID that batch adds to the upload and verify actions. Verify requires the content at that location, copies it to the shared location when missing, links the object, and deletes the upload. The janitor deletes abandoned uploads after a day. Service.CrossRepoDedup and Store.GetObjects are removed, so every repository uploads an object before it is linked. --- README.md | 2 + cmd/lfsd/main.go | 7 +- go.mod | 11 +- go.sum | 12 ++ internal/database/objects.go | 18 --- internal/janitor/janitor.go | 23 ++- internal/janitor/janitor_test.go | 45 ++++++ lfs/go.mod | 1 + lfs/go.sum | 2 + lfs/http.go | 16 +- lfs/http_test.go | 28 +++- lfs/lfs.go | 21 ++- lfs/service.go | 147 ++++++++++--------- lfs/service_test.go | 245 +++++++++++++++++++++---------- lfs/storage/s3_presign.go | 180 ++++++++++++++++++++++- lfs/storage/s3_presign_test.go | 97 +++++++++++- lfs/storage/storage.go | 21 ++- 17 files changed, 670 insertions(+), 206 deletions(-) create mode 100644 internal/janitor/janitor_test.go diff --git a/README.md b/README.md index 445223e..0eed42e 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com At least one `[forge "{host}"]` section must be configured. Each forge references a `[storage "{name}"]` section by name. Multiple forges may share a single storage backend. Supported storage types are `filesystem` and `s3-presign` (compatible with S3, R2, DigitalOcean Spaces, etc.). +An `s3-presign` bucket also holds in-progress uploads under the `uploads/` prefix. They are moved into place when the client verifies the upload, and abandoned ones are deleted automatically after a day. + ### Client Git LFS resolves the LFS endpoint per repository. To point a clone at lfsd, set `lfs.url` in the repository's `.lfsconfig` (committed so collaborators inherit it) to `EXTERNAL_URL` joined with the forge host, the repository path, and the `/info/lfs` suffix: diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index 3d64c86..90ae26b 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -63,10 +63,9 @@ func main() { services := make(map[string]*lfs.Service, len(config.Storages)) for host, backend := range config.Storages { services[host] = &lfs.Service{ - Store: db, - Storage: backend, - MaxObjectSize: config.Server.MaxObjectSize, - CrossRepoDedup: true, + Store: db, + Storage: backend, + MaxObjectSize: config.Server.MaxObjectSize, } } serve := serveLFS(logger.Scoped("lfs"), config.Server.ExternalURL, services) diff --git a/go.mod b/go.mod index 02f7aab..790da06 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,13 @@ go 1.27 require ( charm.land/lipgloss/v2 v2.0.3 charm.land/log/v2 v2.0.0 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 github.com/flamego/cache v1.5.1 github.com/flamego/flamego v1.9.7 + github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/lib/pq v1.12.3 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 @@ -23,9 +27,7 @@ require ( require ( bitbucket.org/creachadair/shell v0.0.7 // indirect github.com/alecthomas/participle/v2 v2.1.4 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect @@ -33,7 +35,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect @@ -57,6 +58,7 @@ require ( github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/itchyny/gojq v0.12.14 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -76,13 +78,16 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/otel v1.11.0 // indirect go.opentelemetry.io/otel/trace v1.11.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.48.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ff17b44..b843784 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= @@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= @@ -97,6 +101,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/autogold/v2 v2.2.1 h1:JPUXuZQGkcQMv7eeDXuNMovjfoRYaa0yVcm+F3voaGY= github.com/hexops/autogold/v2 v2.2.1/go.mod h1:IJwxtUfj1BGLm0YsR/k+dIxYi6xbeLjqGke2bzcOTMI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -161,6 +167,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -172,6 +180,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4= go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= @@ -222,6 +232,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/database/objects.go b/internal/database/objects.go index 9d66b1b..426500f 100644 --- a/internal/database/objects.go +++ b/internal/database/objects.go @@ -156,24 +156,6 @@ WHERE id = @id`, return storedURI, nil } -// GetObjects fetches verified objects for the given OIDs. Results are keyed by -// OID, silently dropping missing OIDs. -func (d *DB) GetObjects(ctx context.Context, oids []string) (map[string]lfs.Object, error) { - if len(oids) == 0 { - return map[string]lfs.Object{}, nil - } - var rows []Object - err := d.db.WithContext(ctx).Raw(` -SELECT * FROM objects -WHERE oid = ANY(@oids) AND verified_at IS NOT NULL`, - map[string]any{"oids": pq.Array(oids)}, - ).Scan(&rows).Error - if err != nil { - return nil, errors.Wrap(err, "select objects by OIDs") - } - return verifiedObjectsByOID(rows), nil -} - // GetRepoObjects fetches verified objects for the given OIDs that are linked to // the repository. Results are keyed by OID, silently dropping missing OIDs. func (d *DB) GetRepoObjects(ctx context.Context, repoName string, oids []string) (map[string]lfs.Object, error) { diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index 81474d1..12d35b1 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -20,7 +20,7 @@ const ( ) // Janitor sweeps orphan (never verified) and unreferenced (repo_count = 0) -// objects. +// objects, and abandoned presigned uploads. type Janitor struct { db *database.DB storages []storage.Backend @@ -58,9 +58,11 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) { } } -// Sweep runs both cleanup passes once. Errors are logged but not returned. +// Sweep runs all cleanup passes once. Errors are logged but not returned. // The next tick will retry. func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { + j.deleteUploads(ctx, logger.Scoped("uploads"), time.Now().Add(-orphanAge)) + if objects, err := j.db.SweepOrphanObjects(ctx, orphanAge, sweepBatchSize); err != nil { logger.ErrorContext(ctx, "Failed to sweep orphan objects", "error", err) } else if len(objects) > 0 { @@ -113,6 +115,23 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object logFn(ctx, "Swept objects", "deletedCount", deletedCount, "failedCount", failedCount) } +// deleteUploads deletes the content of presigned uploads last modified before +// the cutoff, which were never verified or failed to be deleted after. +func (j *Janitor) deleteUploads(ctx context.Context, logger *logx.Logger, before time.Time) { + for _, b := range j.storages { + p, ok := b.(storage.Presigner) + if !ok { + continue + } + deletedCount, err := p.DeleteUploads(ctx, before) + if err != nil { + logger.ErrorContext(ctx, "Failed to delete stale uploads", "storage", b.Name(), "deletedCount", deletedCount, "error", err) + } else if deletedCount > 0 { + logger.InfoContext(ctx, "Deleted stale uploads", "storage", b.Name(), "deletedCount", deletedCount) + } + } +} + func (j *Janitor) deleteByURI(ctx context.Context, uri string) error { for _, b := range j.storages { if strings.HasPrefix(uri, b.Scheme()) { diff --git a/internal/janitor/janitor_test.go b/internal/janitor/janitor_test.go new file mode 100644 index 0000000..ce44cce --- /dev/null +++ b/internal/janitor/janitor_test.go @@ -0,0 +1,45 @@ +package janitor + +import ( + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cockroachdb/errors" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "unknwon.dev/x/logx" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +func TestJanitor_deleteUploads(t *testing.T) { + clock := gofakes3.FixedTimeSource(time.Now()) + mem := s3mem.New(s3mem.WithTimeSource(clock)) + require.NoError(t, mem.CreateBucket("lfs-objects")) + srv := httptest.NewServer(gofakes3.New(mem).Server()) + t.Cleanup(srv.Close) + backend, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) + require.NoError(t, err) + + put := func(key string) { + _, err := mem.PutObject("lfs-objects", key, map[string]string{}, strings.NewReader("x"), 1, nil) + require.NoError(t, err) + } + put("uploads/stale/oid") + clock.Advance(orphanAge + time.Hour) + put("uploads/recent/oid") + + filesystem, err := storage.NewFilesystemBackend("fs", "file://", t.TempDir(), t.TempDir()) + require.NoError(t, err) + j := New(nil, map[string]storage.Backend{"a.example.com": backend, "b.example.com": filesystem}) + j.deleteUploads(t.Context(), logx.NewNoopLogger(), clock.Now().Add(-orphanAge)) + + _, err = backend.Head(t.Context(), backend.UploadURI("stale", "oid")) + assert.True(t, errors.Is(err, storage.ErrNotFound), "a stale upload is deleted") + _, err = backend.Head(t.Context(), backend.UploadURI("recent", "oid")) + assert.NoError(t, err, "a recent upload is kept") +} diff --git a/lfs/go.mod b/lfs/go.mod index 6310341..e3509a7 100644 --- a/lfs/go.mod +++ b/lfs/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 + github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/stretchr/testify v1.11.1 ) diff --git a/lfs/go.sum b/lfs/go.sum index 398ce3a..c81d87f 100644 --- a/lfs/go.sum +++ b/lfs/go.sum @@ -43,6 +43,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/lfs/http.go b/lfs/http.go index 2a7e674..5edcad1 100644 --- a/lfs/http.go +++ b/lfs/http.go @@ -44,6 +44,9 @@ type Access struct { // - PUT .../{oid} // - POST .../verify // +// The upload and verify requests of a storage.Presigner carry an upload ID in +// the "upload" query parameter, which the caller must pass through unchanged. +// // The request's Authorization header is forwarded in the actions of batch // responses, so the client authenticates transfers with the same credentials. func (s *Service) Serve(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { @@ -114,20 +117,17 @@ func (s *Service) serveUpload(logger *slog.Logger, w http.ResponseWriter, r *htt } defer func() { _ = r.Body.Close() }() - result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.ContentLength, r.Body) + result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.URL.Query().Get(uploadQueryParam), r.ContentLength, r.Body) if err != nil { writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) return } - switch { - case result.Redirect != nil: + if result.Redirect != nil { writeRedirect(w, result.Redirect) - case result.Existed: - w.WriteHeader(http.StatusOK) - default: - w.WriteHeader(http.StatusNoContent) + return } + w.WriteHeader(http.StatusNoContent) } func (s *Service) serveDownload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { @@ -170,7 +170,7 @@ func (s *Service) serveVerify(logger *slog.Logger, w http.ResponseWriter, r *htt return } - if err := s.Verify(r.Context(), logger, access.Repo, req); err != nil { + if err := s.Verify(r.Context(), logger, access.Repo, r.URL.Query().Get(uploadQueryParam), req); err != nil { writeServiceError(logger.With("oid", req.OID), w, r, err, http.StatusText(http.StatusInternalServerError)) return } diff --git a/lfs/http_test.go b/lfs/http_test.go index 992bc1e..a93fcbe 100644 --- a/lfs/http_test.go +++ b/lfs/http_test.go @@ -77,9 +77,19 @@ func TestService_Serve(t *testing.T) { }) t.Run("presigned transfer redirects", func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t)} + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} - w := serve(s, readWrite, http.MethodPut, "/"+oid, "") + w := serve(s, readWrite, http.MethodPost, "/batch", batch(OperationUpload)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var batchResp BatchResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &batchResp)) + require.Len(t, batchResp.Objects, 1) + uploadQuery := "?upload=" + uploadIDOf(t, batchResp.Objects[0]) + + requireError(t, serve(s, readWrite, http.MethodPost, "/verify"+uploadQuery, verify), http.StatusUnprocessableEntity) + + w = serve(s, readWrite, http.MethodPut, "/"+oid+uploadQuery, "") require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) assert.Equal(t, "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", w.Header().Get("x-amz-checksum-sha256")) @@ -91,8 +101,9 @@ func TestService_Serve(t *testing.T) { require.NoError(t, resp.Body.Close()) require.Equal(t, http.StatusOK, resp.StatusCode) - w = serve(s, readWrite, http.MethodPost, "/verify", verify) + w = serve(s, readWrite, http.MethodPost, "/verify"+uploadQuery, verify) require.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + assert.Equal(t, []string{oid}, listKeys()) w = serve(s, readWrite, http.MethodGet, "/"+oid, "") require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) @@ -104,6 +115,15 @@ func TestService_Serve(t *testing.T) { assert.Equal(t, content, string(got)) }) + t.Run("presigned transfer requires an upload ID", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + for _, query := range []string{"", "?upload=", "?upload=not-a-uuid"} { + requireError(t, serve(s, readWrite, http.MethodPut, "/"+oid+query, ""), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify"+query, verify), http.StatusBadRequest) + } + }) + t.Run("checks permissions", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} readOnly := Access{Repo: "repo", Href: href, CanRead: true} @@ -116,7 +136,7 @@ func TestService_Serve(t *testing.T) { requireError(t, serve(s, writeOnly, http.MethodPost, "/batch", batch(OperationDownload)), http.StatusForbidden) requireError(t, serve(s, writeOnly, http.MethodGet, "/"+oid, ""), http.StatusForbidden) - _, err := s.Upload(t.Context(), discardLogger, "repo", oid, int64(size), strings.NewReader(content)) + _, err := s.Upload(t.Context(), discardLogger, "repo", oid, "", int64(size), strings.NewReader(content)) require.NoError(t, err) assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodGet, "/"+oid, "").Code) }) diff --git a/lfs/lfs.go b/lfs/lfs.go index 6cf482b..0d00527 100644 --- a/lfs/lfs.go +++ b/lfs/lfs.go @@ -12,6 +12,13 @@ // content through the server, and a storage.Presigner backend (e.g., // storage.S3PresignBackend) redirects clients to the storage. // +// An OID is not secret, because pointer files carry it in plain text. So a +// repository only gains an object by uploading its content, even when the +// object is already stored for another repository. Storage keeps one copy of +// each object, shared by the repositories it is linked to. A presigned upload +// goes to its own upload location, and verification copies it to the location +// of the object. +// // [Service.Serve] handles the HTTP requests of the batch, upload, download, // and verify endpoints. The caller routes the requests, authenticates them, // and passes an [Access] with the repository, the base URL of its endpoints, @@ -121,24 +128,22 @@ type Object struct { } // Store is the metadata layer of LFS objects. An object is "pending" from the -// start of a presigned upload until it is verified, and only verified objects -// are served. A repository is identified by an opaque string chosen by the -// caller. +// verification of a presigned upload until the verification succeeds, and only +// verified objects are served. A repository is identified by an opaque string +// chosen by the caller. type Store interface { // GetRepoObjects returns the verified objects among oids that are linked to // repo, keyed by OID. Missing OIDs are omitted. GetRepoObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) - // GetObjects returns the verified objects among oids regardless of the - // repositories they are linked to, keyed by OID. Missing OIDs are omitted. - // It is only called when Service.CrossRepoDedup is true. - GetObjects(ctx context.Context, oids []string) (map[string]Object, error) // LinkObject records a verified object whose content is stored at uri and // links it to repo. It returns the URI the object is recorded with, and when // that differs from uri, the caller deletes the content at uri as no longer // referenced. LinkObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) // InsertPendingObject records a pending object for oid, unless the object is - // already recorded, so that abandoned presigned uploads can be cleaned up. + // already recorded. It is called before the content of a presigned upload is + // copied to the canonical location of the object, so that the content can be + // cleaned up when the verification does not complete. InsertPendingObject(ctx context.Context, oid string) error // VerifyObject marks the pending object for oid as verified with content // stored at uri and links it to repo. It returns the URI the object is diff --git a/lfs/service.go b/lfs/service.go index be48dfc..23c6f50 100644 --- a/lfs/service.go +++ b/lfs/service.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/cockroachdb/errors" + "github.com/google/uuid" "unknwon.dev/git-lfs-server/lfs/internal/iox" "unknwon.dev/git-lfs-server/lfs/storage" @@ -35,16 +36,13 @@ type Service struct { // MaxObjectSize is the maximum size in bytes of an uploaded object. Zero or // negative means no limit. MaxObjectSize int64 - // CrossRepoDedup allows an upload to be satisfied by an object that is - // already stored for another repository, without the client sending the - // content again. Without it, a repository only gains an object by uploading - // its full content, so knowing an OID and its size is not enough to read the - // object through a repository the client has write access to. - CrossRepoDedup bool } // Endpoint describes how the client reaches the transfer endpoints of a -// repository: objects at Href/{oid} and verifications at Href/verify. +// repository: objects at Href/{oid} and verifications at Href/verify. For a +// storage.Presigner, the upload and verify actions also carry an upload ID in +// the "upload" query parameter, which the caller passes to Service.Upload and +// Service.Verify. type Endpoint struct { Href string // Header is included in every action, e.g., to forward the credentials of @@ -82,19 +80,11 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e oids[i] = o.OID } - var ( - existing map[string]Object - err error - ) - switch req.Operation { - case OperationDownload: - existing, err = s.Store.GetRepoObjects(ctx, repo, oids) - case OperationUpload: - existing, err = s.uploadedObjects(ctx, repo, oids) - } + existing, err := s.Store.GetRepoObjects(ctx, repo, oids) if err != nil { return nil, errors.Wrap(err, "look up objects") } + _, presigned := s.Storage.(storage.Presigner) objectHref := func(oid string) string { return endpoint.Href + "/" + oid } resp := &BatchResponse{ @@ -125,15 +115,7 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, s.MaxObjectSize), } case ok && obj.Size == in.Size: - // No actions tell the client the object needs no upload. With - // cross-repo dedup, the object may only be linked to another - // repository so far. - if s.CrossRepoDedup { - if _, err := s.Store.LinkObject(ctx, repo, in.OID, in.Size, obj.URI); err != nil { - logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) - out.Error = &ObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} - } - } + // No actions tell the client the object needs no upload. case ok && obj.Size != in.Size: out.Error = &ObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} default: @@ -146,9 +128,19 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e // Presigned uploads require the client to send the checksum that // the storage verifies the content against. uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) + uploadHref, verifyHref := objectHref(in.OID), endpoint.Href+"/verify" + if presigned { + uploadID, err := uuid.NewV7() + if err != nil { + return nil, errors.Wrap(err, "generate upload ID") + } + query := "?" + uploadQueryParam + "=" + uploadID.String() + uploadHref += query + verifyHref += query + } out.Actions = map[string]Action{ - "upload": {Href: objectHref(in.OID), Header: uploadHeader}, - "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + "upload": {Href: uploadHref, Header: uploadHeader}, + "verify": {Href: verifyHref, Header: endpoint.Header}, } } } @@ -157,37 +149,33 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e return resp, nil } -// uploadedObjects returns the verified objects among oids that an upload to -// repo does not need to send again. -func (s *Service) uploadedObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) { - if s.CrossRepoDedup { - objects, err := s.Store.GetObjects(ctx, oids) - if err != nil { - return nil, errors.Wrap(err, "get objects") - } - return objects, nil - } - objects, err := s.Store.GetRepoObjects(ctx, repo, oids) - if err != nil { - return nil, errors.Wrap(err, "get repository objects") - } - return objects, nil -} - // UploadResult is the outcome of a successful Service.Upload. type UploadResult struct { // Redirect is set when the client must upload the content to another // location and then call the verify endpoint. Redirect *Redirect - // Existed reports that the object was already stored and the content was not - // read. - Existed bool +} + +// uploadQueryParam is the query parameter of the upload and verify actions that +// carries the upload ID for a storage.Presigner. +const uploadQueryParam = "upload" + +// validateUploadID checks that uploadID has the form of the upload IDs issued +// by Service.Batch, so that it is safe to use in storage locations. +func validateUploadID(uploadID string) error { + id, err := uuid.Parse(uploadID) + if err != nil || id.Version() != 7 || uploadID != id.String() { + return &Error{Code: http.StatusBadRequest, Message: "invalid upload ID"} + } + return nil } // Upload handles an upload of size bytes for oid to repo. For a // storage.Proxier, it stores the content read from body after verifying its -// size and hash. For a storage.Presigner, it does not read body. -func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { +// size and hash, and ignores uploadID. For a storage.Presigner, it does not +// read body and redirects the client to the upload location of uploadID, the +// upload ID from the action that Service.Batch returned. +func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid, uploadID string, size int64, body io.Reader) (*UploadResult, error) { if !ValidOID(oid) { return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -205,7 +193,7 @@ func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid str case storage.Proxier: return s.uploadProxy(ctx, logger, b, repo, oid, size, body) case storage.Presigner: - return s.uploadPresign(ctx, b, repo, oid, size) + return uploadPresign(ctx, b, oid, uploadID, size) default: return nil, errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) } @@ -245,19 +233,11 @@ func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend return &UploadResult{}, nil } -func (s *Service) uploadPresign(ctx context.Context, backend storage.Presigner, repo, oid string, size int64) (*UploadResult, error) { - existing, err := s.uploadedObjects(ctx, repo, []string{oid}) - if err != nil { - return nil, errors.Wrap(err, "look up object") - } - if _, ok := existing[oid]; ok { - return &UploadResult{Existed: true}, nil - } - - if err = s.Store.InsertPendingObject(ctx, oid); err != nil { - return nil, errors.Wrap(err, "insert pending object") +func uploadPresign(ctx context.Context, backend storage.Presigner, oid, uploadID string, size int64) (*UploadResult, error) { + if err := validateUploadID(uploadID); err != nil { + return nil, err } - url, header, err := backend.PresignPut(ctx, oid, size) + url, header, err := backend.PresignPut(ctx, backend.UploadURI(uploadID, oid), oid, size) if err != nil { return nil, errors.Wrap(err, "presign upload") } @@ -330,8 +310,12 @@ func (s *Service) backendFor(uri string) storage.Backend { } // Verify handles the verification of an upload to repo. A presigned upload -// becomes downloadable only after it is verified. -func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) error { +// becomes downloadable only after it is verified, which requires the content +// at the upload location of uploadID, like Service.Upload. Each upload +// location is only used once, so a repository only gains an object by +// uploading its content, even when the object is already stored for another +// repository. +func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo, uploadID string, req VerifyRequest) error { if !ValidOID(req.OID) { return &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -340,7 +324,7 @@ func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, case storage.Proxier: return s.verifyProxy(ctx, repo, req) case storage.Presigner: - return s.verifyPresign(ctx, logger, b, repo, req) + return s.verifyPresign(ctx, logger, b, repo, uploadID, req) default: return errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) } @@ -361,22 +345,40 @@ func (s *Service) verifyProxy(ctx context.Context, repo string, req VerifyReques return nil } -func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo string, req VerifyRequest) error { - uri := backend.URI(req.OID) - headSize, err := backend.Head(ctx, uri) +func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo, uploadID string, req VerifyRequest) error { + if err := validateUploadID(uploadID); err != nil { + return err + } + uploadURI := backend.UploadURI(uploadID, req.OID) + uploadSize, err := backend.Head(ctx, uploadURI) if err != nil { if errors.Is(err, storage.ErrNotFound) { return &Error{Code: http.StatusUnprocessableEntity, Message: "object does not exist in storage"} } - return errors.Wrap(err, "head object") + return errors.Wrap(err, "head upload") } // The presigned upload signs Content-Length, which should make this // impossible, but treat any mismatch as a hard error rather than trusting // either side. - if headSize != req.Size { + if uploadSize != req.Size { return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} } + // Record the object before its content reaches the canonical location, so + // that the content is cleaned up if verification fails from here on. + if err = s.Store.InsertPendingObject(ctx, req.OID); err != nil { + return errors.Wrap(err, "insert pending object") + } + uri := backend.URI(req.OID) + if _, err = backend.Head(ctx, uri); err != nil { + if !errors.Is(err, storage.ErrNotFound) { + return errors.Wrap(err, "head object") + } + if err = backend.Copy(ctx, uploadURI, uri, req.Size); err != nil { + return errors.Wrap(err, "copy upload to object") + } + } + storedURI, err := s.Store.VerifyObject(ctx, repo, req.OID, req.Size, uri) if err != nil { return errors.Wrap(err, "mark object verified") @@ -386,5 +388,8 @@ func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backen logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) } } + if err := backend.Delete(ctx, uploadURI); err != nil { + logger.ErrorContext(ctx, "Failed to delete upload", "uri", uploadURI, "error", err) + } return nil } diff --git a/lfs/service_test.go b/lfs/service_test.go index c1f8501..ba2b453 100644 --- a/lfs/service_test.go +++ b/lfs/service_test.go @@ -4,17 +4,18 @@ import ( "context" "crypto/sha256" "encoding/hex" - "fmt" "io" "log/slog" "net/http" "net/http/httptest" + "net/url" "path/filepath" "strings" "sync" "testing" "github.com/cockroachdb/errors" + "github.com/google/uuid" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/stretchr/testify/assert" @@ -56,12 +57,6 @@ func (s *memStore) GetRepoObjects(_ context.Context, repo string, oids []string) return s.verifiedObjects(oids, func(oid string) bool { return s.links[repo][oid] }), nil } -func (s *memStore) GetObjects(_ context.Context, oids []string) (map[string]Object, error) { - s.mu.Lock() - defer s.mu.Unlock() - return s.verifiedObjects(oids, func(string) bool { return true }), nil -} - func (s *memStore) link(repo, oid string) { if s.links[repo] == nil { s.links[repo] = map[string]bool{} @@ -120,7 +115,9 @@ func newFilesystemBackend(t *testing.T, name, scheme string) *storage.Filesystem return b } -func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { +// newS3PresignBackend returns a backend on a fake S3, and a function that lists +// the keys in its bucket. +func newS3PresignBackend(t *testing.T) (*storage.S3PresignBackend, func() []string) { t.Helper() mem := s3mem.New() require.NoError(t, mem.CreateBucket("lfs-objects")) @@ -128,7 +125,16 @@ func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { t.Cleanup(srv.Close) b, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) require.NoError(t, err) - return b + listKeys := func() []string { + list, err := mem.ListBucket("lfs-objects", nil, gofakes3.ListBucketPage{}) + require.NoError(t, err) + keys := make([]string, 0, len(list.Contents)) + for _, c := range list.Contents { + keys = append(keys, c.Key) + } + return keys + } + return b, listKeys } var discardLogger = slog.New(slog.DiscardHandler) @@ -148,13 +154,13 @@ func TestService_Proxy(t *testing.T) { t.Run("upload stores content and links it to the repository only", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} - result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + result, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) assert.Equal(t, &UploadResult{}, result) - require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", "", VerifyRequest{OID: oid, Size: size})) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", "", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", "", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) download, err := s.Download(ctx, "repo1", oid) require.NoError(t, err) @@ -170,16 +176,16 @@ func TestService_Proxy(t *testing.T) { t.Run("upload of stored content still verifies the body", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) forged := strings.Repeat("x", len(content)) - _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(forged)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, "", size, strings.NewReader(forged)) requireLFSError(t, err, http.StatusUnprocessableEntity) _, err = s.Download(ctx, "repo2", oid) requireLFSError(t, err, http.StatusNotFound) - _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, "", size, strings.NewReader(content)) require.NoError(t, err) download, err := s.Download(ctx, "repo2", oid) require.NoError(t, err) @@ -188,15 +194,15 @@ func TestService_Proxy(t *testing.T) { t.Run("upload rejects invalid input", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), MaxObjectSize: size - 1} - _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", "", size, strings.NewReader(content)) requireLFSError(t, err, http.StatusBadRequest) - _, err = s.Upload(ctx, discardLogger, "repo1", oid, -1, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", -1, strings.NewReader(content)) requireLFSError(t, err, http.StatusLengthRequired) - _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) requireLFSError(t, err, http.StatusRequestEntityTooLarge) s.MaxObjectSize = 0 - _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content[1:])) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content[1:])) requireLFSError(t, err, http.StatusUnprocessableEntity) }) @@ -204,7 +210,7 @@ func TestService_Proxy(t *testing.T) { store := newMemStore() previous := newFilesystemBackend(t, "old", "old-file://") s := &Service{Store: store, Storage: previous} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) s = &Service{Store: store, Storage: newFilesystemBackend(t, "new", "file://"), PreviousStorages: []storage.Backend{previous}} @@ -229,9 +235,9 @@ func TestService_Batch(t *testing.T) { size := int64(len(content)) endpoint := Endpoint{Href: "https://lfs.example.com/repo/info/lfs/objects"} - newServiceWithObject := func(t *testing.T, crossRepoDedup bool) *Service { - s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), CrossRepoDedup: crossRepoDedup} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + newServiceWithObject := func(t *testing.T) *Service { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) return s } @@ -239,40 +245,49 @@ func TestService_Batch(t *testing.T) { download := BatchRequest{Operation: OperationDownload, Objects: []BatchObject{{OID: oid, Size: size}}} t.Run("upload of an object in the repository needs no transfer", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, upload) require.NoError(t, err) assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) }) - t.Run("upload of an object in another repository", func(t *testing.T) { - t.Run("needs a transfer without cross-repo dedup", func(t *testing.T) { - s := newServiceWithObject(t, false) - resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) - require.NoError(t, err) - require.Len(t, resp.Objects, 1) - assert.Contains(t, resp.Objects[0].Actions, "upload") - - resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) - require.NoError(t, err) - require.NotNil(t, resp.Objects[0].Error) - assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) - }) + t.Run("upload of an object in another repository needs a transfer", func(t *testing.T) { + s := newServiceWithObject(t) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + assert.Contains(t, resp.Objects[0].Actions, "upload") - t.Run("links the object with cross-repo dedup", func(t *testing.T) { - s := newServiceWithObject(t, true) - resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) - require.NoError(t, err) - assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + require.NotNil(t, resp.Objects[0].Error) + assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) + }) - resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + t.Run("presigned upload actions carry a new upload ID per object", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + req := BatchRequest{Operation: OperationUpload, Objects: []BatchObject{ + {OID: sha256Hex("a"), Size: 1}, + {OID: sha256Hex("b"), Size: 1}, + }} + seen := map[string]bool{} + for range 2 { + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, req) require.NoError(t, err) - assert.Contains(t, resp.Objects[0].Actions, "download") - }) + require.Len(t, resp.Objects, 2) + for _, o := range resp.Objects { + uploadID := uploadIDOf(t, o) + assert.Equal(t, endpoint.Href+"/"+o.OID+"?upload="+uploadID, o.Actions["upload"].Href) + assert.Equal(t, endpoint.Href+"/verify?upload="+uploadID, o.Actions["verify"].Href) + assert.False(t, seen[uploadID], "upload ID %s is reused", uploadID) + seen[uploadID] = true + } + } }) t.Run("actions carry the endpoint", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) endpoint := Endpoint{Href: endpoint.Href, Header: map[string]string{"Authorization": "Basic Zm9vOmJhcg=="}, Authenticated: true} newContent := "new content" newOID := sha256Hex(newContent) @@ -310,7 +325,7 @@ func TestService_Batch(t *testing.T) { }) t.Run("rejects invalid requests", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) for _, req := range []BatchRequest{ {Operation: "delete"}, {Operation: OperationDownload, Objects: []BatchObject{{OID: "not-an-oid"}}}, @@ -323,14 +338,45 @@ func TestService_Batch(t *testing.T) { }) } +// uploadIDOf returns the upload ID of the upload action of o, after checking +// that the verify action carries the same one. +func uploadIDOf(t *testing.T, o BatchResponseObject) string { + t.Helper() + require.Contains(t, o.Actions, "upload") + uploadHref, err := url.Parse(o.Actions["upload"].Href) + require.NoError(t, err) + verifyHref, err := url.Parse(o.Actions["verify"].Href) + require.NoError(t, err) + uploadID := uploadHref.Query().Get("upload") + require.NotEmpty(t, uploadID) + require.Equal(t, uploadID, verifyHref.Query().Get("upload")) + return uploadID +} + func TestService_Presign(t *testing.T) { ctx := t.Context() content := "presigned content" oid := sha256Hex(content) size := int64(len(content)) + verifyReq := VerifyRequest{OID: oid, Size: size} - // uploadToRedirect sends the content to the presigned URL like a client. - uploadToRedirect := func(t *testing.T, redirect *Redirect) { + // startUpload asks for an upload of the content to repo like a client, and + // returns the upload ID and the presigned URL to send the content to. + startUpload := func(t *testing.T, s *Service, repo string) (string, *Redirect) { + t.Helper() + resp, err := s.Batch(ctx, discardLogger, repo, Endpoint{Href: "https://lfs.example.com/objects"}, BatchRequest{ + Operation: OperationUpload, + Objects: []BatchObject{{OID: oid, Size: size}}, + }) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + uploadID := uploadIDOf(t, resp.Objects[0]) + result, err := s.Upload(ctx, discardLogger, repo, oid, uploadID, size, nil) + require.NoError(t, err) + require.NotNil(t, result.Redirect) + return uploadID, result.Redirect + } + sendContent := func(t *testing.T, redirect *Redirect) { t.Helper() req, err := http.NewRequestWithContext(ctx, http.MethodPut, redirect.URL, strings.NewReader(content)) require.NoError(t, err) @@ -343,36 +389,83 @@ func TestService_Presign(t *testing.T) { require.NoError(t, resp.Body.Close()) require.Equal(t, http.StatusOK, resp.StatusCode) } + upload := func(t *testing.T, s *Service, repo string) { + t.Helper() + uploadID, redirect := startUpload(t, s, repo) + sendContent(t, redirect) + require.NoError(t, s.Verify(ctx, discardLogger, repo, uploadID, verifyReq)) + } + requireDownload := func(t *testing.T, s *Service, repo string) { + t.Helper() + download, err := s.Download(ctx, repo, oid) + require.NoError(t, err) + require.NotNil(t, download.Redirect) + resp, err := http.Get(download.Redirect.URL) + require.NoError(t, err) + got, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + } - for _, crossRepoDedup := range []bool{false, true} { - t.Run(fmt.Sprintf("crossRepoDedup=%v", crossRepoDedup), func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t), CrossRepoDedup: crossRepoDedup} + t.Run("verify requires the content at the upload location", func(t *testing.T) { + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + uploadID, redirect := startUpload(t, s, "repo1") + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq), http.StatusUnprocessableEntity) + _, err := s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) - result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, nil) - require.NoError(t, err) - require.NotNil(t, result.Redirect) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size}), http.StatusUnprocessableEntity) - _, err = s.Download(ctx, "repo1", oid) - requireLFSError(t, err, http.StatusNotFound) + sendContent(t, redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq)) + requireDownload(t, s, "repo1") + assert.Equal(t, []string{oid}, listKeys(), "the upload is moved to the object location") + }) - uploadToRedirect(t, result.Redirect) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) - require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + t.Run("another repository must upload a stored object", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + uploadID, redirect := startUpload(t, s, "repo1") + sendContent(t, redirect) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq)) + + // The OID and size are public in pointer files, and an upload ID is free + // to get, so none of them link the object without the content. + unusedUploadID, _ := startUpload(t, s, "repo2") + for _, uploadID := range []string{unusedUploadID, uploadID} { + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", uploadID, verifyReq), http.StatusUnprocessableEntity) + } + _, err := s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + }) - download, err := s.Download(ctx, "repo1", oid) - require.NoError(t, err) - require.NotNil(t, download.Redirect) - assert.Contains(t, download.Redirect.URL, oid) + t.Run("repositories share one stored copy", func(t *testing.T) { + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + upload(t, s, "repo1") + upload(t, s, "repo2") - result, err = s.Upload(ctx, discardLogger, "repo1", oid, size, nil) - require.NoError(t, err) - assert.Equal(t, &UploadResult{Existed: true}, result) + requireDownload(t, s, "repo1") + requireDownload(t, s, "repo2") + assert.Equal(t, []string{oid}, listKeys()) + }) - // Without cross-repo dedup, another repository must upload the content - // itself, which the storage checks against the OID. - result, err = s.Upload(ctx, discardLogger, "repo2", oid, size, nil) - require.NoError(t, err) - assert.Equal(t, crossRepoDedup, result.Redirect == nil) - }) - } + t.Run("rejects invalid upload IDs", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + v7 := uuid.Must(uuid.NewV7()).String() + for _, uploadID := range []string{ + "", + "not-a-uuid", + uuid.NewString(), // Version 4. + strings.ToUpper(v7), + "urn:uuid:" + v7, + "../" + v7, + } { + _, err := s.Upload(ctx, discardLogger, "repo1", oid, uploadID, size, nil) + requireLFSError(t, err, http.StatusBadRequest) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq), http.StatusBadRequest) + } + }) } diff --git a/lfs/storage/s3_presign.go b/lfs/storage/s3_presign.go index ce3208f..2a76b47 100644 --- a/lfs/storage/s3_presign.go +++ b/lfs/storage/s3_presign.go @@ -7,6 +7,7 @@ import ( "net/url" "strconv" "strings" + "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -24,6 +25,20 @@ import ( // gap to budget for. Keep this short so a leaked URL has limited utility. const presignURLTTL = 60 * time.Second +// uploadsPrefix is the key prefix of upload locations. +const uploadsPrefix = "uploads/" + +const ( + // maxCopyObjectSize is the largest object that a single CopyObject request + // can copy. Larger objects are copied in parts. + maxCopyObjectSize = 5 << 30 + // copyPartSize is the size of each part of a multipart copy. All parts but + // the last have the same size, which Cloudflare R2 requires. + copyPartSize = 512 << 20 + // copyPartConcurrency is the number of parts copied at the same time. + copyPartConcurrency = 8 +) + var _ Presigner = (*S3PresignBackend)(nil) type S3PresignBackend struct { @@ -87,16 +102,26 @@ func (b *S3PresignBackend) URI(oid string) string { return b.scheme + b.bucket + "/" + oid } -// PresignPut returns a presigned PUT URL that pins the body's SHA-256 to oid -// via x-amz-checksum-sha256, so the backend server-side validates the upload -// and rejects mismatches with HTTP 400 BadDigest. Content-Length is also +// UploadURI returns the {scheme}{bucket}/uploads/{uploadID}/{oid} location. +// Pure function, safe to call without I/O. +func (b *S3PresignBackend) UploadURI(uploadID, oid string) string { + return b.scheme + b.bucket + "/" + uploadsPrefix + uploadID + "/" + oid +} + +// PresignPut returns a presigned PUT URL to uri that pins the body's SHA-256 to +// oid via x-amz-checksum-sha256, so the backend server-side validates the +// upload and rejects mismatches with HTTP 400 BadDigest. Content-Length is also // included so the size is signed. // // Note: x-amz-content-sha256 cannot be used here. AWS sigv4 mandates // UNSIGNED-PAYLOAD as the canonical-request HashedPayload for query-string // presigned URLs (S3 sigv4-query-string-auth spec), so signing the body hash // into x-amz-content-sha256 produces a signature that R2 rejects. -func (b *S3PresignBackend) PresignPut(ctx context.Context, oid string, size int64) (string, map[string]string, error) { +func (b *S3PresignBackend) PresignPut(ctx context.Context, uri, oid string, size int64) (string, map[string]string, error) { + key, err := b.keyFromURI(uri) + if err != nil { + return "", nil, err + } rawOID, err := hex.DecodeString(oid) if err != nil { return "", nil, errors.Wrapf(err, "decode oid %q", oid) @@ -105,7 +130,7 @@ func (b *S3PresignBackend) PresignPut(ctx context.Context, oid string, size int6 req, err := b.presignClient.PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(oid), + Key: aws.String(key), ContentLength: aws.Int64(size), ChecksumSHA256: aws.String(checksumB64), }, func(o *s3.PresignOptions) { @@ -189,6 +214,151 @@ func (b *S3PresignBackend) Delete(ctx context.Context, uri string) error { return nil } +// Copy copies the object with CopyObject, or in parts with UploadPartCopy when +// it is larger than what CopyObject supports. +func (b *S3PresignBackend) Copy(ctx context.Context, srcURI, dstURI string, size int64) error { + srcKey, err := b.keyFromURI(srcURI) + if err != nil { + return err + } + dstKey, err := b.keyFromURI(dstURI) + if err != nil { + return err + } + // Keys are made of hex digits, UUIDs, and slashes, which need no escaping. + copySource := b.bucket + "/" + srcKey + if size <= maxCopyObjectSize { + if _, err := b.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + CopySource: aws.String(copySource), + }); err != nil { + return errors.Wrapf(err, "copy object %q to %q", srcKey, dstKey) + } + return nil + } + if err := b.copyMultipart(ctx, copySource, dstKey, size); err != nil { + return errors.Wrapf(err, "copy object %q to %q in parts", srcKey, dstKey) + } + return nil +} + +func (b *S3PresignBackend) copyMultipart(ctx context.Context, copySource, dstKey string, size int64) error { + created, err := b.client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + }) + if err != nil { + return errors.Wrap(err, "create multipart upload") + } + abort := func(cause error) error { + // Abort even when ctx is canceled, so the parts copied so far do not + // linger in the storage. + if _, err := b.client.AbortMultipartUpload(context.WithoutCancel(ctx), &s3.AbortMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + }); err != nil { + return errors.WithSecondaryError(cause, errors.Wrap(err, "abort multipart upload")) + } + return cause + } + + ranges := copyPartRanges(size, copyPartSize) + parts := make([]s3types.CompletedPart, len(ranges)) + partsCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + slots := make(chan struct{}, copyPartConcurrency) + var wg sync.WaitGroup + for i, byteRange := range ranges { + partNumber := aws.Int32(int32(i + 1)) + wg.Go(func() { + select { + case slots <- struct{}{}: + defer func() { <-slots }() + case <-partsCtx.Done(): + return + } + out, err := b.client.UploadPartCopy(partsCtx, &s3.UploadPartCopyInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + PartNumber: partNumber, + CopySource: aws.String(copySource), + CopySourceRange: aws.String(byteRange), + }) + if err != nil { + cancel(errors.Wrapf(err, "copy part %d", *partNumber)) + return + } + parts[i] = s3types.CompletedPart{ETag: out.CopyPartResult.ETag, PartNumber: partNumber} + }) + } + wg.Wait() + if err := context.Cause(partsCtx); err != nil { + return abort(err) + } + + if _, err := b.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts}, + }); err != nil { + return abort(errors.Wrap(err, "complete multipart upload")) + } + return nil +} + +// copyPartRanges splits size bytes into consecutive HTTP byte ranges of +// partSize bytes, except the last which holds the remainder. +func copyPartRanges(size, partSize int64) []string { + ranges := make([]string, 0, (size+partSize-1)/partSize) + for start := int64(0); start < size; start += partSize { + end := min(start+partSize, size) - 1 + ranges = append(ranges, "bytes="+strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(end, 10)) + } + return ranges +} + +// DeleteUploads deletes the objects under the uploads/ prefix that were last +// modified before the cutoff. +func (b *S3PresignBackend) DeleteUploads(ctx context.Context, before time.Time) (int, error) { + var deleted int + pages := s3.NewListObjectsV2Paginator(b.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(b.bucket), + Prefix: aws.String(uploadsPrefix), + }) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + if err != nil { + return deleted, errors.Wrap(err, "list uploads") + } + var stale []s3types.ObjectIdentifier + for _, o := range page.Contents { + if o.LastModified != nil && o.LastModified.Before(before) { + stale = append(stale, s3types.ObjectIdentifier{Key: o.Key}) + } + } + if len(stale) == 0 { + continue + } + out, err := b.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(b.bucket), + Delete: &s3types.Delete{Objects: stale, Quiet: aws.Bool(true)}, + }) + if err != nil { + return deleted, errors.Wrap(err, "delete uploads") + } + deleted += len(stale) - len(out.Errors) + if len(out.Errors) > 0 { + e := out.Errors[0] + return deleted, errors.Newf("delete upload %q: %s", aws.ToString(e.Key), aws.ToString(e.Message)) + } + } + return deleted, nil +} + // keyFromURI extracts the object key from a {scheme}{bucket}/{key} URI. The // bucket is asserted to match the configured backend's bucket so a stale URI // from a different bucket can't accidentally read or write the wrong bytes. diff --git a/lfs/storage/s3_presign_test.go b/lfs/storage/s3_presign_test.go index d48cfb6..70c49b1 100644 --- a/lfs/storage/s3_presign_test.go +++ b/lfs/storage/s3_presign_test.go @@ -9,6 +9,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" @@ -29,10 +30,10 @@ const ( // test bucket, and returns an S3PresignBackend pointed at the fake. The // fake's Server http.Handler accepts S3 API calls from any aws-sdk-go-v2 // client configured with path-style and a custom BaseEndpoint. -func newTestS3PresignBackend(t *testing.T) (*S3PresignBackend, *httptest.Server) { +func newTestS3PresignBackend(t *testing.T, opts ...s3mem.Option) (*S3PresignBackend, *httptest.Server) { t.Helper() - mem := s3mem.New() + mem := s3mem.New(opts...) require.NoError(t, mem.CreateBucket(testBucket)) srv := httptest.NewServer(gofakes3.New(mem).Server()) @@ -52,6 +53,7 @@ func TestS3PresignBackend_URI(t *testing.T) { b, _ := newTestS3PresignBackend(t) oid := sha256Hex("uri-test") assert.Equal(t, "r2://"+testBucket+"/"+oid, b.URI(oid)) + assert.Equal(t, "r2://"+testBucket+"/uploads/upload-id/"+oid, b.UploadURI("upload-id", oid)) } func TestS3PresignBackend_PresignPut(t *testing.T) { @@ -60,7 +62,8 @@ func TestS3PresignBackend_PresignPut(t *testing.T) { oid := sha256Hex(body) size := int64(len(body)) - url, headers, err := b.PresignPut(t.Context(), oid, size) + uploadURI := b.UploadURI("upload-id", oid) + url, headers, err := b.PresignPut(t.Context(), uploadURI, oid, size) require.NoError(t, err) rawOID, err := hex.DecodeString(oid) @@ -90,9 +93,11 @@ func TestS3PresignBackend_PresignPut(t *testing.T) { _ = resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) - gotSize, err := b.Head(t.Context(), b.URI(oid)) + gotSize, err := b.Head(t.Context(), uploadURI) require.NoError(t, err) assert.Equal(t, size, gotSize) + _, err = b.Head(t.Context(), b.URI(oid)) + assert.True(t, errors.Is(err, ErrNotFound), "only the upload location is written") }) } @@ -197,6 +202,90 @@ func TestS3PresignBackend_Delete(t *testing.T) { }) } +func TestS3PresignBackend_Copy(t *testing.T) { + b, _ := newTestS3PresignBackend(t) + body := "copy me" + oid := sha256Hex(body) + src := b.UploadURI("upload-id", oid) + putObject(t, b, "uploads/upload-id/"+oid, body) + + require.NoError(t, b.Copy(t.Context(), src, b.URI(oid), int64(len(body)))) + assert.Equal(t, body, getObject(t, b, oid)) + _, err := b.Head(t.Context(), src) + require.NoError(t, err, "the source is kept") + + err = b.Copy(t.Context(), b.UploadURI("missing", oid), b.URI(sha256Hex("other")), 1) + require.Error(t, err) +} + +func TestCopyPartRanges(t *testing.T) { + tests := []struct { + name string + size, partSize int64 + want []string + }{ + {"remainder in last part", 10, 4, []string{"bytes=0-3", "bytes=4-7", "bytes=8-9"}}, + {"exact multiple", 8, 4, []string{"bytes=0-3", "bytes=4-7"}}, + {"smaller than a part", 1, 4, []string{"bytes=0-0"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, copyPartRanges(tc.size, tc.partSize)) + }) + } + + t.Run("just over the CopyObject limit", func(t *testing.T) { + ranges := copyPartRanges(maxCopyObjectSize+1, copyPartSize) + require.Len(t, ranges, 11) + assert.Equal(t, "bytes=0-536870911", ranges[0]) + assert.Equal(t, "bytes=5368709120-5368709120", ranges[10]) + }) +} + +func TestS3PresignBackend_DeleteUploads(t *testing.T) { + clock := gofakes3.FixedTimeSource(time.Now()) + b, _ := newTestS3PresignBackend(t, s3mem.WithTimeSource(clock)) + oid := sha256Hex("upload") + putObject(t, b, "uploads/stale/"+oid, "upload") + putObject(t, b, oid, "upload") + clock.Advance(2 * time.Hour) + putObject(t, b, "uploads/recent/"+oid, "upload") + + deleted, err := b.DeleteUploads(t.Context(), clock.Now().Add(-time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, deleted) + + _, err = b.Head(t.Context(), b.UploadURI("stale", oid)) + assert.True(t, errors.Is(err, ErrNotFound), "a stale upload is deleted") + _, err = b.Head(t.Context(), b.UploadURI("recent", oid)) + assert.NoError(t, err, "a recent upload is kept") + _, err = b.Head(t.Context(), b.URI(oid)) + assert.NoError(t, err, "an object outside uploads/ is kept regardless of its age") +} + +func putObject(t *testing.T, b *S3PresignBackend, key, body string) { + t.Helper() + _, err := b.client.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(key), + Body: strings.NewReader(body), + }) + require.NoError(t, err) +} + +func getObject(t *testing.T, b *S3PresignBackend, key string) string { + t.Helper() + out, err := b.client.GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(key), + }) + require.NoError(t, err) + defer func() { _ = out.Body.Close() }() + got, err := io.ReadAll(out.Body) + require.NoError(t, err) + return string(got) +} + func TestNewS3PresignBackend(t *testing.T) { t.Run("required fields", func(t *testing.T) { cases := []struct { diff --git a/lfs/storage/storage.go b/lfs/storage/storage.go index 6bb95ae..2bb2ff3 100644 --- a/lfs/storage/storage.go +++ b/lfs/storage/storage.go @@ -3,6 +3,7 @@ package storage import ( "context" "io" + "time" "github.com/cockroachdb/errors" ) @@ -53,15 +54,29 @@ type Proxier interface { // Presigner is implemented by backends that validate object integrity when // directly uploading from clients, and reject uploads with mismatched content. +// +// Each upload goes to its own upload location, so that an object at an upload +// location proves that the upload happened. The server copies the content to +// the canonical location of the object once the upload is verified. type Presigner interface { Backend - // PresignPut returns a short-lived URL the client PUTs to, along with the - // headers the client MUST send verbatim. - PresignPut(ctx context.Context, oid string, size int64) (url string, headers map[string]string, err error) + // UploadURI returns the location that the upload with the given ID sends the + // content of oid to. Pure function, safe to compute without I/O. + UploadURI(uploadID, oid string) string + // PresignPut returns a short-lived URL the client PUTs the content of oid to, + // along with the headers the client MUST send verbatim. The object is stored + // at uri, and only when the content matches oid and size. + PresignPut(ctx context.Context, uri, oid string, size int64) (url string, headers map[string]string, err error) // PresignGet returns a short-lived URL the client GETs from. PresignGet(ctx context.Context, uri string) (url string, err error) // Head returns the size of the object. It returns ErrNotFound when no matching // object exists. Head(ctx context.Context, uri string) (size int64, err error) + // Copy copies the object of size bytes at srcURI to dstURI within the + // storage, replacing any object at dstURI. + Copy(ctx context.Context, srcURI, dstURI string, size int64) error + // DeleteUploads deletes the objects at upload locations that were last + // modified before the cutoff, and returns how many it deleted. + DeleteUploads(ctx context.Context, before time.Time) (deleted int, err error) } From edf06e8bc4094769abbd555571c2c9f8fc1c4937 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Fri, 13 Nov 2026 20:03:32 +0000 Subject: [PATCH 2/2] integration-tests: push the same object from two repositories Clone each repository fresh so that git-lfs downloads through the smudge filter, check that the second repository must upload the object, and check that R2 holds one copy with no upload left behind. --- integration-tests/main_test.go | 111 ++++++++++++++++++++++---- integration-tests/testdata/config.ini | 7 ++ 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/integration-tests/main_test.go b/integration-tests/main_test.go index e185223..54b7fca 100644 --- a/integration-tests/main_test.go +++ b/integration-tests/main_test.go @@ -3,13 +3,16 @@ package main import ( + "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "flag" "fmt" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -19,9 +22,15 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/sourcegraph/run" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.bobheadxi.dev/streamline/streamexec" + + "unknwon.dev/git-lfs-server/lfs" ) var long = flag.Bool("long", false, "run long-running integration tests") @@ -67,10 +76,10 @@ func TestUploadDownloadRoundtrip(t *testing.T) { }) pushDir := t.TempDir() - blob := genRandomBlob(t, 1*1024*1024) - wantSum := sha256.Sum256(blob) + content := genRandomContent(t, 1*1024*1024) + wantSum := sha256.Sum256(content) - require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), blob, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), content, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(pushDir, ".gitattributes"), []byte("*.bin filter=lfs diff=lfs merge=lfs -text\n"), 0o644)) @@ -85,21 +94,91 @@ func TestUploadDownloadRoundtrip(t *testing.T) { git("remote", "add", "origin", remoteURL) git("push", "origin", "HEAD:refs/heads/"+branch) - // Skip smudge during clone so the LFS object isn't fetched as a side effect of - // checkout. Then explicitly pull to exercise the download path against lfsd. - pullDir := t.TempDir() - require.NoError(t, - run.Cmd(ctx, "git", "clone", "--branch", branch, "--depth", "1", remoteURL, pullDir). - Environ([]string{"GIT_LFS_SKIP_SMUDGE=1", "GIT_TERMINAL_PROMPT=0"}).Run().Wait()) + oid := hex.EncodeToString(wantSum[:]) + // Check out in a fresh clone, so the smudge filter downloads the LFS object + // from lfsd. + requireClone := func(lfsURL string) { + t.Helper() + cloneDir := t.TempDir() + require.NoError(t, + run.Cmd(ctx, "git", "clone", "--no-checkout", "--branch", branch, "--depth", "1", remoteURL, cloneDir). + Environ([]string{"GIT_TERMINAL_PROMPT=0"}).Run().Wait()) + cloneGit := gitCmd(ctx, t, cloneDir) + cloneGit("config", "lfs.url", lfsURL) + cloneGit("checkout", branch) + got, err := os.ReadFile(filepath.Join(cloneDir, "large.bin")) + require.NoError(t, err) + require.True(t, bytes.Equal(content, got), "downloaded content differs from the pushed content") + } + requireClone(lfsURL) + + // The same object in another repository is only linked to it by uploading + // the content again. + otherLFSURL := "http://x:x@127.0.0.1:3356/local.test/unknwon/git-lfs-test/info/lfs" + resp := batch(ctx, t, otherLFSURL, lfs.BatchRequest{ + Operation: lfs.OperationDownload, + Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}}, + }) + require.NotNil(t, resp.Objects[0].Error, "another repository must not download the object before uploading it") + resp = batch(ctx, t, otherLFSURL, lfs.BatchRequest{ + Operation: lfs.OperationUpload, + Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}}, + }) + require.Contains(t, resp.Objects[0].Actions, "upload", "another repository must upload the object") - pullGit := gitCmd(ctx, t, pullDir) - pullGit("config", "lfs.url", lfsURL) - pullGit("lfs", "pull") + git("config", "lfs.url", otherLFSURL) + git("lfs", "push", "--object-id", "origin", oid) + requireClone(otherLFSURL) - got, err := os.ReadFile(filepath.Join(pullDir, "large.bin")) + // Both repositories share one copy, and no upload is left behind. + client := s3.NewFromConfig(aws.Config{ + Region: "auto", + Credentials: credentials.NewStaticCredentialsProvider( + os.Getenv("LFSD_R2_ACCESS_KEY_ID"), os.Getenv("LFSD_R2_SECRET_ACCESS_KEY"), ""), + }, func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("https://%s.r2.cloudflarestorage.com", os.Getenv("LFSD_R2_ACCOUNT_ID"))) + o.UsePathStyle = true + }) + bucket := aws.String(os.Getenv("LFSD_R2_BUCKET")) + head, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: bucket, Key: aws.String(oid)}) require.NoError(t, err) - gotSum := sha256.Sum256(got) - require.Equal(t, hex.EncodeToString(wantSum[:]), hex.EncodeToString(gotSum[:])) + assert.Equal(t, int64(len(content)), aws.ToInt64(head.ContentLength)) + pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: bucket, Prefix: aws.String("uploads/")}) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + require.NoError(t, err) + for _, o := range page.Contents { + assert.False(t, strings.HasSuffix(aws.ToString(o.Key), "/"+oid), "upload %q is left behind", aws.ToString(o.Key)) + } + } +} + +// batch sends a batch request to the LFS endpoint at lfsURL, whose user info +// is sent as HTTP Basic Auth credentials. +func batch(ctx context.Context, t *testing.T, lfsURL string, req lfs.BatchRequest) *lfs.BatchResponse { + t.Helper() + u, err := url.Parse(lfsURL + "/objects/batch") + require.NoError(t, err) + password, _ := u.User.Password() + username := u.User.Username() + u.User = nil + + body, err := json.Marshal(req) + require.NoError(t, err) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(body)) + require.NoError(t, err) + httpReq.SetBasicAuth(username, password) + httpReq.Header.Set("Content-Type", lfs.MediaType) + httpReq.Header.Set("Accept", lfs.MediaType) + httpResp, err := http.DefaultClient.Do(httpReq) + require.NoError(t, err) + defer func() { _ = httpResp.Body.Close() }() + require.Equal(t, http.StatusOK, httpResp.StatusCode) + + var resp lfs.BatchResponse + require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&resp)) + require.Len(t, resp.Objects, len(req.Objects)) + return &resp } func requireEnv(t *testing.T, names ...string) { @@ -213,7 +292,7 @@ func gitCmd(ctx context.Context, t *testing.T, dir string) func(args ...string) } } -func genRandomBlob(t *testing.T, size int) []byte { +func genRandomContent(t *testing.T, size int) []byte { t.Helper() buf := make([]byte, size) _, err := rand.Read(buf) diff --git a/integration-tests/testdata/config.ini b/integration-tests/testdata/config.ini index 78391fc..fc1def6 100644 --- a/integration-tests/testdata/config.ini +++ b/integration-tests/testdata/config.ini @@ -17,3 +17,10 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com TYPE = github STORAGE = r2 REPO_ALLOWLIST = unknwon/git-lfs-test + +; Serves the same repository under another name, so that the test uploads the +; same object to two repositories. lfsd only listens on localhost in tests. +[forge "local.test"] +SKIP_AUTH = true +STORAGE = r2 +REPO_ALLOWLIST = unknwon/git-lfs-test