diff --git a/README.md b/README.md index 445223e..0eed42e 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com At least one `[forge "{host}"]` section must be configured. Each forge references a `[storage "{name}"]` section by name. Multiple forges may share a single storage backend. Supported storage types are `filesystem` and `s3-presign` (compatible with S3, R2, DigitalOcean Spaces, etc.). +An `s3-presign` bucket also holds in-progress uploads under the `uploads/` prefix. They are moved into place when the client verifies the upload, and abandoned ones are deleted automatically after a day. + ### Client Git LFS resolves the LFS endpoint per repository. To point a clone at lfsd, set `lfs.url` in the repository's `.lfsconfig` (committed so collaborators inherit it) to `EXTERNAL_URL` joined with the forge host, the repository path, and the `/info/lfs` suffix: diff --git a/cmd/lfsd/main.go b/cmd/lfsd/main.go index 3d64c86..90ae26b 100644 --- a/cmd/lfsd/main.go +++ b/cmd/lfsd/main.go @@ -63,10 +63,9 @@ func main() { services := make(map[string]*lfs.Service, len(config.Storages)) for host, backend := range config.Storages { services[host] = &lfs.Service{ - Store: db, - Storage: backend, - MaxObjectSize: config.Server.MaxObjectSize, - CrossRepoDedup: true, + Store: db, + Storage: backend, + MaxObjectSize: config.Server.MaxObjectSize, } } serve := serveLFS(logger.Scoped("lfs"), config.Server.ExternalURL, services) diff --git a/go.mod b/go.mod index 02f7aab..790da06 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,13 @@ go 1.27 require ( charm.land/lipgloss/v2 v2.0.3 charm.land/log/v2 v2.0.0 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 github.com/flamego/cache v1.5.1 github.com/flamego/flamego v1.9.7 + github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/lib/pq v1.12.3 github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/run v0.12.0 @@ -23,9 +27,7 @@ require ( require ( bitbucket.org/creachadair/shell v0.0.7 // indirect github.com/alecthomas/participle/v2 v2.1.4 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect @@ -33,7 +35,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect github.com/aws/smithy-go v1.25.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect @@ -57,6 +58,7 @@ require ( github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/itchyny/gojq v0.12.14 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -76,13 +78,16 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/otel v1.11.0 // indirect go.opentelemetry.io/otel/trace v1.11.0 // indirect + go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.48.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ff17b44..b843784 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= @@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= +github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= @@ -97,6 +101,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/autogold/v2 v2.2.1 h1:JPUXuZQGkcQMv7eeDXuNMovjfoRYaa0yVcm+F3voaGY= github.com/hexops/autogold/v2 v2.2.1/go.mod h1:IJwxtUfj1BGLm0YsR/k+dIxYi6xbeLjqGke2bzcOTMI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -161,6 +167,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -172,6 +180,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4= go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= @@ -222,6 +232,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration-tests/main_test.go b/integration-tests/main_test.go index e185223..54b7fca 100644 --- a/integration-tests/main_test.go +++ b/integration-tests/main_test.go @@ -3,13 +3,16 @@ package main import ( + "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "flag" "fmt" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -19,9 +22,15 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/sourcegraph/run" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.bobheadxi.dev/streamline/streamexec" + + "unknwon.dev/git-lfs-server/lfs" ) var long = flag.Bool("long", false, "run long-running integration tests") @@ -67,10 +76,10 @@ func TestUploadDownloadRoundtrip(t *testing.T) { }) pushDir := t.TempDir() - blob := genRandomBlob(t, 1*1024*1024) - wantSum := sha256.Sum256(blob) + content := genRandomContent(t, 1*1024*1024) + wantSum := sha256.Sum256(content) - require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), blob, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), content, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(pushDir, ".gitattributes"), []byte("*.bin filter=lfs diff=lfs merge=lfs -text\n"), 0o644)) @@ -85,21 +94,91 @@ func TestUploadDownloadRoundtrip(t *testing.T) { git("remote", "add", "origin", remoteURL) git("push", "origin", "HEAD:refs/heads/"+branch) - // Skip smudge during clone so the LFS object isn't fetched as a side effect of - // checkout. Then explicitly pull to exercise the download path against lfsd. - pullDir := t.TempDir() - require.NoError(t, - run.Cmd(ctx, "git", "clone", "--branch", branch, "--depth", "1", remoteURL, pullDir). - Environ([]string{"GIT_LFS_SKIP_SMUDGE=1", "GIT_TERMINAL_PROMPT=0"}).Run().Wait()) + oid := hex.EncodeToString(wantSum[:]) + // Check out in a fresh clone, so the smudge filter downloads the LFS object + // from lfsd. + requireClone := func(lfsURL string) { + t.Helper() + cloneDir := t.TempDir() + require.NoError(t, + run.Cmd(ctx, "git", "clone", "--no-checkout", "--branch", branch, "--depth", "1", remoteURL, cloneDir). + Environ([]string{"GIT_TERMINAL_PROMPT=0"}).Run().Wait()) + cloneGit := gitCmd(ctx, t, cloneDir) + cloneGit("config", "lfs.url", lfsURL) + cloneGit("checkout", branch) + got, err := os.ReadFile(filepath.Join(cloneDir, "large.bin")) + require.NoError(t, err) + require.True(t, bytes.Equal(content, got), "downloaded content differs from the pushed content") + } + requireClone(lfsURL) + + // The same object in another repository is only linked to it by uploading + // the content again. + otherLFSURL := "http://x:x@127.0.0.1:3356/local.test/unknwon/git-lfs-test/info/lfs" + resp := batch(ctx, t, otherLFSURL, lfs.BatchRequest{ + Operation: lfs.OperationDownload, + Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}}, + }) + require.NotNil(t, resp.Objects[0].Error, "another repository must not download the object before uploading it") + resp = batch(ctx, t, otherLFSURL, lfs.BatchRequest{ + Operation: lfs.OperationUpload, + Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}}, + }) + require.Contains(t, resp.Objects[0].Actions, "upload", "another repository must upload the object") - pullGit := gitCmd(ctx, t, pullDir) - pullGit("config", "lfs.url", lfsURL) - pullGit("lfs", "pull") + git("config", "lfs.url", otherLFSURL) + git("lfs", "push", "--object-id", "origin", oid) + requireClone(otherLFSURL) - got, err := os.ReadFile(filepath.Join(pullDir, "large.bin")) + // Both repositories share one copy, and no upload is left behind. + client := s3.NewFromConfig(aws.Config{ + Region: "auto", + Credentials: credentials.NewStaticCredentialsProvider( + os.Getenv("LFSD_R2_ACCESS_KEY_ID"), os.Getenv("LFSD_R2_SECRET_ACCESS_KEY"), ""), + }, func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("https://%s.r2.cloudflarestorage.com", os.Getenv("LFSD_R2_ACCOUNT_ID"))) + o.UsePathStyle = true + }) + bucket := aws.String(os.Getenv("LFSD_R2_BUCKET")) + head, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: bucket, Key: aws.String(oid)}) require.NoError(t, err) - gotSum := sha256.Sum256(got) - require.Equal(t, hex.EncodeToString(wantSum[:]), hex.EncodeToString(gotSum[:])) + assert.Equal(t, int64(len(content)), aws.ToInt64(head.ContentLength)) + pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: bucket, Prefix: aws.String("uploads/")}) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + require.NoError(t, err) + for _, o := range page.Contents { + assert.False(t, strings.HasSuffix(aws.ToString(o.Key), "/"+oid), "upload %q is left behind", aws.ToString(o.Key)) + } + } +} + +// batch sends a batch request to the LFS endpoint at lfsURL, whose user info +// is sent as HTTP Basic Auth credentials. +func batch(ctx context.Context, t *testing.T, lfsURL string, req lfs.BatchRequest) *lfs.BatchResponse { + t.Helper() + u, err := url.Parse(lfsURL + "/objects/batch") + require.NoError(t, err) + password, _ := u.User.Password() + username := u.User.Username() + u.User = nil + + body, err := json.Marshal(req) + require.NoError(t, err) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(body)) + require.NoError(t, err) + httpReq.SetBasicAuth(username, password) + httpReq.Header.Set("Content-Type", lfs.MediaType) + httpReq.Header.Set("Accept", lfs.MediaType) + httpResp, err := http.DefaultClient.Do(httpReq) + require.NoError(t, err) + defer func() { _ = httpResp.Body.Close() }() + require.Equal(t, http.StatusOK, httpResp.StatusCode) + + var resp lfs.BatchResponse + require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&resp)) + require.Len(t, resp.Objects, len(req.Objects)) + return &resp } func requireEnv(t *testing.T, names ...string) { @@ -213,7 +292,7 @@ func gitCmd(ctx context.Context, t *testing.T, dir string) func(args ...string) } } -func genRandomBlob(t *testing.T, size int) []byte { +func genRandomContent(t *testing.T, size int) []byte { t.Helper() buf := make([]byte, size) _, err := rand.Read(buf) diff --git a/integration-tests/testdata/config.ini b/integration-tests/testdata/config.ini index 78391fc..fc1def6 100644 --- a/integration-tests/testdata/config.ini +++ b/integration-tests/testdata/config.ini @@ -17,3 +17,10 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com TYPE = github STORAGE = r2 REPO_ALLOWLIST = unknwon/git-lfs-test + +; Serves the same repository under another name, so that the test uploads the +; same object to two repositories. lfsd only listens on localhost in tests. +[forge "local.test"] +SKIP_AUTH = true +STORAGE = r2 +REPO_ALLOWLIST = unknwon/git-lfs-test diff --git a/internal/database/objects.go b/internal/database/objects.go index 9d66b1b..426500f 100644 --- a/internal/database/objects.go +++ b/internal/database/objects.go @@ -156,24 +156,6 @@ WHERE id = @id`, return storedURI, nil } -// GetObjects fetches verified objects for the given OIDs. Results are keyed by -// OID, silently dropping missing OIDs. -func (d *DB) GetObjects(ctx context.Context, oids []string) (map[string]lfs.Object, error) { - if len(oids) == 0 { - return map[string]lfs.Object{}, nil - } - var rows []Object - err := d.db.WithContext(ctx).Raw(` -SELECT * FROM objects -WHERE oid = ANY(@oids) AND verified_at IS NOT NULL`, - map[string]any{"oids": pq.Array(oids)}, - ).Scan(&rows).Error - if err != nil { - return nil, errors.Wrap(err, "select objects by OIDs") - } - return verifiedObjectsByOID(rows), nil -} - // GetRepoObjects fetches verified objects for the given OIDs that are linked to // the repository. Results are keyed by OID, silently dropping missing OIDs. func (d *DB) GetRepoObjects(ctx context.Context, repoName string, oids []string) (map[string]lfs.Object, error) { diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index 81474d1..12d35b1 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -20,7 +20,7 @@ const ( ) // Janitor sweeps orphan (never verified) and unreferenced (repo_count = 0) -// objects. +// objects, and abandoned presigned uploads. type Janitor struct { db *database.DB storages []storage.Backend @@ -58,9 +58,11 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) { } } -// Sweep runs both cleanup passes once. Errors are logged but not returned. +// Sweep runs all cleanup passes once. Errors are logged but not returned. // The next tick will retry. func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) { + j.deleteUploads(ctx, logger.Scoped("uploads"), time.Now().Add(-orphanAge)) + if objects, err := j.db.SweepOrphanObjects(ctx, orphanAge, sweepBatchSize); err != nil { logger.ErrorContext(ctx, "Failed to sweep orphan objects", "error", err) } else if len(objects) > 0 { @@ -113,6 +115,23 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object logFn(ctx, "Swept objects", "deletedCount", deletedCount, "failedCount", failedCount) } +// deleteUploads deletes the content of presigned uploads last modified before +// the cutoff, which were never verified or failed to be deleted after. +func (j *Janitor) deleteUploads(ctx context.Context, logger *logx.Logger, before time.Time) { + for _, b := range j.storages { + p, ok := b.(storage.Presigner) + if !ok { + continue + } + deletedCount, err := p.DeleteUploads(ctx, before) + if err != nil { + logger.ErrorContext(ctx, "Failed to delete stale uploads", "storage", b.Name(), "deletedCount", deletedCount, "error", err) + } else if deletedCount > 0 { + logger.InfoContext(ctx, "Deleted stale uploads", "storage", b.Name(), "deletedCount", deletedCount) + } + } +} + func (j *Janitor) deleteByURI(ctx context.Context, uri string) error { for _, b := range j.storages { if strings.HasPrefix(uri, b.Scheme()) { diff --git a/internal/janitor/janitor_test.go b/internal/janitor/janitor_test.go new file mode 100644 index 0000000..ce44cce --- /dev/null +++ b/internal/janitor/janitor_test.go @@ -0,0 +1,45 @@ +package janitor + +import ( + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cockroachdb/errors" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "unknwon.dev/x/logx" + + "unknwon.dev/git-lfs-server/lfs/storage" +) + +func TestJanitor_deleteUploads(t *testing.T) { + clock := gofakes3.FixedTimeSource(time.Now()) + mem := s3mem.New(s3mem.WithTimeSource(clock)) + require.NoError(t, mem.CreateBucket("lfs-objects")) + srv := httptest.NewServer(gofakes3.New(mem).Server()) + t.Cleanup(srv.Close) + backend, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) + require.NoError(t, err) + + put := func(key string) { + _, err := mem.PutObject("lfs-objects", key, map[string]string{}, strings.NewReader("x"), 1, nil) + require.NoError(t, err) + } + put("uploads/stale/oid") + clock.Advance(orphanAge + time.Hour) + put("uploads/recent/oid") + + filesystem, err := storage.NewFilesystemBackend("fs", "file://", t.TempDir(), t.TempDir()) + require.NoError(t, err) + j := New(nil, map[string]storage.Backend{"a.example.com": backend, "b.example.com": filesystem}) + j.deleteUploads(t.Context(), logx.NewNoopLogger(), clock.Now().Add(-orphanAge)) + + _, err = backend.Head(t.Context(), backend.UploadURI("stale", "oid")) + assert.True(t, errors.Is(err, storage.ErrNotFound), "a stale upload is deleted") + _, err = backend.Head(t.Context(), backend.UploadURI("recent", "oid")) + assert.NoError(t, err, "a recent upload is kept") +} diff --git a/lfs/go.mod b/lfs/go.mod index 6310341..e3509a7 100644 --- a/lfs/go.mod +++ b/lfs/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/cockroachdb/errors v1.13.0 + github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 github.com/stretchr/testify v1.11.1 ) diff --git a/lfs/go.sum b/lfs/go.sum index 398ce3a..c81d87f 100644 --- a/lfs/go.sum +++ b/lfs/go.sum @@ -43,6 +43,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73 h1:0xkWp+RMC2ImuKacheMHEAtrbOTMOa0kYkxyzM1Z/II= github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73/go.mod h1:S4S9jGBVlLri0OeqrSSbCGG5vsI6he06UJyuz1WT1EE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/lfs/http.go b/lfs/http.go index 2a7e674..5edcad1 100644 --- a/lfs/http.go +++ b/lfs/http.go @@ -44,6 +44,9 @@ type Access struct { // - PUT .../{oid} // - POST .../verify // +// The upload and verify requests of a storage.Presigner carry an upload ID in +// the "upload" query parameter, which the caller must pass through unchanged. +// // The request's Authorization header is forwarded in the actions of batch // responses, so the client authenticates transfers with the same credentials. func (s *Service) Serve(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access) { @@ -114,20 +117,17 @@ func (s *Service) serveUpload(logger *slog.Logger, w http.ResponseWriter, r *htt } defer func() { _ = r.Body.Close() }() - result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.ContentLength, r.Body) + result, err := s.Upload(r.Context(), logger, access.Repo, oid, r.URL.Query().Get(uploadQueryParam), r.ContentLength, r.Body) if err != nil { writeServiceError(logger.With("oid", oid), w, r, err, http.StatusText(http.StatusInternalServerError)) return } - switch { - case result.Redirect != nil: + if result.Redirect != nil { writeRedirect(w, result.Redirect) - case result.Existed: - w.WriteHeader(http.StatusOK) - default: - w.WriteHeader(http.StatusNoContent) + return } + w.WriteHeader(http.StatusNoContent) } func (s *Service) serveDownload(logger *slog.Logger, w http.ResponseWriter, r *http.Request, access Access, oid string) { @@ -170,7 +170,7 @@ func (s *Service) serveVerify(logger *slog.Logger, w http.ResponseWriter, r *htt return } - if err := s.Verify(r.Context(), logger, access.Repo, req); err != nil { + if err := s.Verify(r.Context(), logger, access.Repo, r.URL.Query().Get(uploadQueryParam), req); err != nil { writeServiceError(logger.With("oid", req.OID), w, r, err, http.StatusText(http.StatusInternalServerError)) return } diff --git a/lfs/http_test.go b/lfs/http_test.go index 992bc1e..a93fcbe 100644 --- a/lfs/http_test.go +++ b/lfs/http_test.go @@ -77,9 +77,19 @@ func TestService_Serve(t *testing.T) { }) t.Run("presigned transfer redirects", func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t)} + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} - w := serve(s, readWrite, http.MethodPut, "/"+oid, "") + w := serve(s, readWrite, http.MethodPost, "/batch", batch(OperationUpload)) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var batchResp BatchResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &batchResp)) + require.Len(t, batchResp.Objects, 1) + uploadQuery := "?upload=" + uploadIDOf(t, batchResp.Objects[0]) + + requireError(t, serve(s, readWrite, http.MethodPost, "/verify"+uploadQuery, verify), http.StatusUnprocessableEntity) + + w = serve(s, readWrite, http.MethodPut, "/"+oid+uploadQuery, "") require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) assert.Equal(t, "97OiZ/pYWtQ/MOiPfTFzQY0nD/6zXoSoWwc00FhqN48=", w.Header().Get("x-amz-checksum-sha256")) @@ -91,8 +101,9 @@ func TestService_Serve(t *testing.T) { require.NoError(t, resp.Body.Close()) require.Equal(t, http.StatusOK, resp.StatusCode) - w = serve(s, readWrite, http.MethodPost, "/verify", verify) + w = serve(s, readWrite, http.MethodPost, "/verify"+uploadQuery, verify) require.Equal(t, http.StatusNoContent, w.Code, w.Body.String()) + assert.Equal(t, []string{oid}, listKeys()) w = serve(s, readWrite, http.MethodGet, "/"+oid, "") require.Equal(t, http.StatusTemporaryRedirect, w.Code, w.Body.String()) @@ -104,6 +115,15 @@ func TestService_Serve(t *testing.T) { assert.Equal(t, content, string(got)) }) + t.Run("presigned transfer requires an upload ID", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + for _, query := range []string{"", "?upload=", "?upload=not-a-uuid"} { + requireError(t, serve(s, readWrite, http.MethodPut, "/"+oid+query, ""), http.StatusBadRequest) + requireError(t, serve(s, readWrite, http.MethodPost, "/verify"+query, verify), http.StatusBadRequest) + } + }) + t.Run("checks permissions", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} readOnly := Access{Repo: "repo", Href: href, CanRead: true} @@ -116,7 +136,7 @@ func TestService_Serve(t *testing.T) { requireError(t, serve(s, writeOnly, http.MethodPost, "/batch", batch(OperationDownload)), http.StatusForbidden) requireError(t, serve(s, writeOnly, http.MethodGet, "/"+oid, ""), http.StatusForbidden) - _, err := s.Upload(t.Context(), discardLogger, "repo", oid, int64(size), strings.NewReader(content)) + _, err := s.Upload(t.Context(), discardLogger, "repo", oid, "", int64(size), strings.NewReader(content)) require.NoError(t, err) assert.Equal(t, http.StatusOK, serve(s, readOnly, http.MethodGet, "/"+oid, "").Code) }) diff --git a/lfs/lfs.go b/lfs/lfs.go index 6cf482b..0d00527 100644 --- a/lfs/lfs.go +++ b/lfs/lfs.go @@ -12,6 +12,13 @@ // content through the server, and a storage.Presigner backend (e.g., // storage.S3PresignBackend) redirects clients to the storage. // +// An OID is not secret, because pointer files carry it in plain text. So a +// repository only gains an object by uploading its content, even when the +// object is already stored for another repository. Storage keeps one copy of +// each object, shared by the repositories it is linked to. A presigned upload +// goes to its own upload location, and verification copies it to the location +// of the object. +// // [Service.Serve] handles the HTTP requests of the batch, upload, download, // and verify endpoints. The caller routes the requests, authenticates them, // and passes an [Access] with the repository, the base URL of its endpoints, @@ -121,24 +128,22 @@ type Object struct { } // Store is the metadata layer of LFS objects. An object is "pending" from the -// start of a presigned upload until it is verified, and only verified objects -// are served. A repository is identified by an opaque string chosen by the -// caller. +// verification of a presigned upload until the verification succeeds, and only +// verified objects are served. A repository is identified by an opaque string +// chosen by the caller. type Store interface { // GetRepoObjects returns the verified objects among oids that are linked to // repo, keyed by OID. Missing OIDs are omitted. GetRepoObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) - // GetObjects returns the verified objects among oids regardless of the - // repositories they are linked to, keyed by OID. Missing OIDs are omitted. - // It is only called when Service.CrossRepoDedup is true. - GetObjects(ctx context.Context, oids []string) (map[string]Object, error) // LinkObject records a verified object whose content is stored at uri and // links it to repo. It returns the URI the object is recorded with, and when // that differs from uri, the caller deletes the content at uri as no longer // referenced. LinkObject(ctx context.Context, repo, oid string, size int64, uri string) (storedURI string, err error) // InsertPendingObject records a pending object for oid, unless the object is - // already recorded, so that abandoned presigned uploads can be cleaned up. + // already recorded. It is called before the content of a presigned upload is + // copied to the canonical location of the object, so that the content can be + // cleaned up when the verification does not complete. InsertPendingObject(ctx context.Context, oid string) error // VerifyObject marks the pending object for oid as verified with content // stored at uri and links it to repo. It returns the URI the object is diff --git a/lfs/service.go b/lfs/service.go index be48dfc..23c6f50 100644 --- a/lfs/service.go +++ b/lfs/service.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/cockroachdb/errors" + "github.com/google/uuid" "unknwon.dev/git-lfs-server/lfs/internal/iox" "unknwon.dev/git-lfs-server/lfs/storage" @@ -35,16 +36,13 @@ type Service struct { // MaxObjectSize is the maximum size in bytes of an uploaded object. Zero or // negative means no limit. MaxObjectSize int64 - // CrossRepoDedup allows an upload to be satisfied by an object that is - // already stored for another repository, without the client sending the - // content again. Without it, a repository only gains an object by uploading - // its full content, so knowing an OID and its size is not enough to read the - // object through a repository the client has write access to. - CrossRepoDedup bool } // Endpoint describes how the client reaches the transfer endpoints of a -// repository: objects at Href/{oid} and verifications at Href/verify. +// repository: objects at Href/{oid} and verifications at Href/verify. For a +// storage.Presigner, the upload and verify actions also carry an upload ID in +// the "upload" query parameter, which the caller passes to Service.Upload and +// Service.Verify. type Endpoint struct { Href string // Header is included in every action, e.g., to forward the credentials of @@ -82,19 +80,11 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e oids[i] = o.OID } - var ( - existing map[string]Object - err error - ) - switch req.Operation { - case OperationDownload: - existing, err = s.Store.GetRepoObjects(ctx, repo, oids) - case OperationUpload: - existing, err = s.uploadedObjects(ctx, repo, oids) - } + existing, err := s.Store.GetRepoObjects(ctx, repo, oids) if err != nil { return nil, errors.Wrap(err, "look up objects") } + _, presigned := s.Storage.(storage.Presigner) objectHref := func(oid string) string { return endpoint.Href + "/" + oid } resp := &BatchResponse{ @@ -125,15 +115,7 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e Message: fmt.Sprintf("object size %d exceeds limit %d", in.Size, s.MaxObjectSize), } case ok && obj.Size == in.Size: - // No actions tell the client the object needs no upload. With - // cross-repo dedup, the object may only be linked to another - // repository so far. - if s.CrossRepoDedup { - if _, err := s.Store.LinkObject(ctx, repo, in.OID, in.Size, obj.URI); err != nil { - logger.ErrorContext(ctx, "Failed to link repo to existing object", "oid", in.OID, "error", err) - out.Error = &ObjectError{Code: http.StatusInternalServerError, Message: "failed to link object to repo"} - } - } + // No actions tell the client the object needs no upload. case ok && obj.Size != in.Size: out.Error = &ObjectError{Code: http.StatusConflict, Message: "OID exists with different size"} default: @@ -146,9 +128,19 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e // Presigned uploads require the client to send the checksum that // the storage verifies the content against. uploadHeader["x-amz-checksum-sha256"] = base64.StdEncoding.EncodeToString(rawOID) + uploadHref, verifyHref := objectHref(in.OID), endpoint.Href+"/verify" + if presigned { + uploadID, err := uuid.NewV7() + if err != nil { + return nil, errors.Wrap(err, "generate upload ID") + } + query := "?" + uploadQueryParam + "=" + uploadID.String() + uploadHref += query + verifyHref += query + } out.Actions = map[string]Action{ - "upload": {Href: objectHref(in.OID), Header: uploadHeader}, - "verify": {Href: endpoint.Href + "/verify", Header: endpoint.Header}, + "upload": {Href: uploadHref, Header: uploadHeader}, + "verify": {Href: verifyHref, Header: endpoint.Header}, } } } @@ -157,37 +149,33 @@ func (s *Service) Batch(ctx context.Context, logger *slog.Logger, repo string, e return resp, nil } -// uploadedObjects returns the verified objects among oids that an upload to -// repo does not need to send again. -func (s *Service) uploadedObjects(ctx context.Context, repo string, oids []string) (map[string]Object, error) { - if s.CrossRepoDedup { - objects, err := s.Store.GetObjects(ctx, oids) - if err != nil { - return nil, errors.Wrap(err, "get objects") - } - return objects, nil - } - objects, err := s.Store.GetRepoObjects(ctx, repo, oids) - if err != nil { - return nil, errors.Wrap(err, "get repository objects") - } - return objects, nil -} - // UploadResult is the outcome of a successful Service.Upload. type UploadResult struct { // Redirect is set when the client must upload the content to another // location and then call the verify endpoint. Redirect *Redirect - // Existed reports that the object was already stored and the content was not - // read. - Existed bool +} + +// uploadQueryParam is the query parameter of the upload and verify actions that +// carries the upload ID for a storage.Presigner. +const uploadQueryParam = "upload" + +// validateUploadID checks that uploadID has the form of the upload IDs issued +// by Service.Batch, so that it is safe to use in storage locations. +func validateUploadID(uploadID string) error { + id, err := uuid.Parse(uploadID) + if err != nil || id.Version() != 7 || uploadID != id.String() { + return &Error{Code: http.StatusBadRequest, Message: "invalid upload ID"} + } + return nil } // Upload handles an upload of size bytes for oid to repo. For a // storage.Proxier, it stores the content read from body after verifying its -// size and hash. For a storage.Presigner, it does not read body. -func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid string, size int64, body io.Reader) (*UploadResult, error) { +// size and hash, and ignores uploadID. For a storage.Presigner, it does not +// read body and redirects the client to the upload location of uploadID, the +// upload ID from the action that Service.Batch returned. +func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid, uploadID string, size int64, body io.Reader) (*UploadResult, error) { if !ValidOID(oid) { return nil, &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -205,7 +193,7 @@ func (s *Service) Upload(ctx context.Context, logger *slog.Logger, repo, oid str case storage.Proxier: return s.uploadProxy(ctx, logger, b, repo, oid, size, body) case storage.Presigner: - return s.uploadPresign(ctx, b, repo, oid, size) + return uploadPresign(ctx, b, oid, uploadID, size) default: return nil, errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) } @@ -245,19 +233,11 @@ func (s *Service) uploadProxy(ctx context.Context, logger *slog.Logger, backend return &UploadResult{}, nil } -func (s *Service) uploadPresign(ctx context.Context, backend storage.Presigner, repo, oid string, size int64) (*UploadResult, error) { - existing, err := s.uploadedObjects(ctx, repo, []string{oid}) - if err != nil { - return nil, errors.Wrap(err, "look up object") - } - if _, ok := existing[oid]; ok { - return &UploadResult{Existed: true}, nil - } - - if err = s.Store.InsertPendingObject(ctx, oid); err != nil { - return nil, errors.Wrap(err, "insert pending object") +func uploadPresign(ctx context.Context, backend storage.Presigner, oid, uploadID string, size int64) (*UploadResult, error) { + if err := validateUploadID(uploadID); err != nil { + return nil, err } - url, header, err := backend.PresignPut(ctx, oid, size) + url, header, err := backend.PresignPut(ctx, backend.UploadURI(uploadID, oid), oid, size) if err != nil { return nil, errors.Wrap(err, "presign upload") } @@ -330,8 +310,12 @@ func (s *Service) backendFor(uri string) storage.Backend { } // Verify handles the verification of an upload to repo. A presigned upload -// becomes downloadable only after it is verified. -func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, req VerifyRequest) error { +// becomes downloadable only after it is verified, which requires the content +// at the upload location of uploadID, like Service.Upload. Each upload +// location is only used once, so a repository only gains an object by +// uploading its content, even when the object is already stored for another +// repository. +func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo, uploadID string, req VerifyRequest) error { if !ValidOID(req.OID) { return &Error{Code: http.StatusBadRequest, Message: "invalid oid"} } @@ -340,7 +324,7 @@ func (s *Service) Verify(ctx context.Context, logger *slog.Logger, repo string, case storage.Proxier: return s.verifyProxy(ctx, repo, req) case storage.Presigner: - return s.verifyPresign(ctx, logger, b, repo, req) + return s.verifyPresign(ctx, logger, b, repo, uploadID, req) default: return errors.Newf("storage backend %T does not implement a known transfer mode", s.Storage) } @@ -361,22 +345,40 @@ func (s *Service) verifyProxy(ctx context.Context, repo string, req VerifyReques return nil } -func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo string, req VerifyRequest) error { - uri := backend.URI(req.OID) - headSize, err := backend.Head(ctx, uri) +func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backend storage.Presigner, repo, uploadID string, req VerifyRequest) error { + if err := validateUploadID(uploadID); err != nil { + return err + } + uploadURI := backend.UploadURI(uploadID, req.OID) + uploadSize, err := backend.Head(ctx, uploadURI) if err != nil { if errors.Is(err, storage.ErrNotFound) { return &Error{Code: http.StatusUnprocessableEntity, Message: "object does not exist in storage"} } - return errors.Wrap(err, "head object") + return errors.Wrap(err, "head upload") } // The presigned upload signs Content-Length, which should make this // impossible, but treat any mismatch as a hard error rather than trusting // either side. - if headSize != req.Size { + if uploadSize != req.Size { return &Error{Code: http.StatusUnprocessableEntity, Message: "size mismatch"} } + // Record the object before its content reaches the canonical location, so + // that the content is cleaned up if verification fails from here on. + if err = s.Store.InsertPendingObject(ctx, req.OID); err != nil { + return errors.Wrap(err, "insert pending object") + } + uri := backend.URI(req.OID) + if _, err = backend.Head(ctx, uri); err != nil { + if !errors.Is(err, storage.ErrNotFound) { + return errors.Wrap(err, "head object") + } + if err = backend.Copy(ctx, uploadURI, uri, req.Size); err != nil { + return errors.Wrap(err, "copy upload to object") + } + } + storedURI, err := s.Store.VerifyObject(ctx, repo, req.OID, req.Size, uri) if err != nil { return errors.Wrap(err, "mark object verified") @@ -386,5 +388,8 @@ func (s *Service) verifyPresign(ctx context.Context, logger *slog.Logger, backen logger.ErrorContext(ctx, "Failed to delete orphan object", "uri", uri, "error", err) } } + if err := backend.Delete(ctx, uploadURI); err != nil { + logger.ErrorContext(ctx, "Failed to delete upload", "uri", uploadURI, "error", err) + } return nil } diff --git a/lfs/service_test.go b/lfs/service_test.go index c1f8501..ba2b453 100644 --- a/lfs/service_test.go +++ b/lfs/service_test.go @@ -4,17 +4,18 @@ import ( "context" "crypto/sha256" "encoding/hex" - "fmt" "io" "log/slog" "net/http" "net/http/httptest" + "net/url" "path/filepath" "strings" "sync" "testing" "github.com/cockroachdb/errors" + "github.com/google/uuid" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/stretchr/testify/assert" @@ -56,12 +57,6 @@ func (s *memStore) GetRepoObjects(_ context.Context, repo string, oids []string) return s.verifiedObjects(oids, func(oid string) bool { return s.links[repo][oid] }), nil } -func (s *memStore) GetObjects(_ context.Context, oids []string) (map[string]Object, error) { - s.mu.Lock() - defer s.mu.Unlock() - return s.verifiedObjects(oids, func(string) bool { return true }), nil -} - func (s *memStore) link(repo, oid string) { if s.links[repo] == nil { s.links[repo] = map[string]bool{} @@ -120,7 +115,9 @@ func newFilesystemBackend(t *testing.T, name, scheme string) *storage.Filesystem return b } -func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { +// newS3PresignBackend returns a backend on a fake S3, and a function that lists +// the keys in its bucket. +func newS3PresignBackend(t *testing.T) (*storage.S3PresignBackend, func() []string) { t.Helper() mem := s3mem.New() require.NoError(t, mem.CreateBucket("lfs-objects")) @@ -128,7 +125,16 @@ func newS3PresignBackend(t *testing.T) *storage.S3PresignBackend { t.Cleanup(srv.Close) b, err := storage.NewS3PresignBackend("s3", "s3://", "lfs-objects", "test", "test", srv.URL) require.NoError(t, err) - return b + listKeys := func() []string { + list, err := mem.ListBucket("lfs-objects", nil, gofakes3.ListBucketPage{}) + require.NoError(t, err) + keys := make([]string, 0, len(list.Contents)) + for _, c := range list.Contents { + keys = append(keys, c.Key) + } + return keys + } + return b, listKeys } var discardLogger = slog.New(slog.DiscardHandler) @@ -148,13 +154,13 @@ func TestService_Proxy(t *testing.T) { t.Run("upload stores content and links it to the repository only", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} - result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + result, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) assert.Equal(t, &UploadResult{}, result) - require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", "", VerifyRequest{OID: oid, Size: size})) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", "", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", "", VerifyRequest{OID: oid, Size: size}), http.StatusNotFound) download, err := s.Download(ctx, "repo1", oid) require.NoError(t, err) @@ -170,16 +176,16 @@ func TestService_Proxy(t *testing.T) { t.Run("upload of stored content still verifies the body", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) forged := strings.Repeat("x", len(content)) - _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(forged)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, "", size, strings.NewReader(forged)) requireLFSError(t, err, http.StatusUnprocessableEntity) _, err = s.Download(ctx, "repo2", oid) requireLFSError(t, err, http.StatusNotFound) - _, err = s.Upload(ctx, discardLogger, "repo2", oid, size, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo2", oid, "", size, strings.NewReader(content)) require.NoError(t, err) download, err := s.Download(ctx, "repo2", oid) require.NoError(t, err) @@ -188,15 +194,15 @@ func TestService_Proxy(t *testing.T) { t.Run("upload rejects invalid input", func(t *testing.T) { s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), MaxObjectSize: size - 1} - _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", "not-an-oid", "", size, strings.NewReader(content)) requireLFSError(t, err, http.StatusBadRequest) - _, err = s.Upload(ctx, discardLogger, "repo1", oid, -1, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", -1, strings.NewReader(content)) requireLFSError(t, err, http.StatusLengthRequired) - _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) requireLFSError(t, err, http.StatusRequestEntityTooLarge) s.MaxObjectSize = 0 - _, err = s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content[1:])) + _, err = s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content[1:])) requireLFSError(t, err, http.StatusUnprocessableEntity) }) @@ -204,7 +210,7 @@ func TestService_Proxy(t *testing.T) { store := newMemStore() previous := newFilesystemBackend(t, "old", "old-file://") s := &Service{Store: store, Storage: previous} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) s = &Service{Store: store, Storage: newFilesystemBackend(t, "new", "file://"), PreviousStorages: []storage.Backend{previous}} @@ -229,9 +235,9 @@ func TestService_Batch(t *testing.T) { size := int64(len(content)) endpoint := Endpoint{Href: "https://lfs.example.com/repo/info/lfs/objects"} - newServiceWithObject := func(t *testing.T, crossRepoDedup bool) *Service { - s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://"), CrossRepoDedup: crossRepoDedup} - _, err := s.Upload(ctx, discardLogger, "repo1", oid, size, strings.NewReader(content)) + newServiceWithObject := func(t *testing.T) *Service { + s := &Service{Store: newMemStore(), Storage: newFilesystemBackend(t, "fs", "file://")} + _, err := s.Upload(ctx, discardLogger, "repo1", oid, "", size, strings.NewReader(content)) require.NoError(t, err) return s } @@ -239,40 +245,49 @@ func TestService_Batch(t *testing.T) { download := BatchRequest{Operation: OperationDownload, Objects: []BatchObject{{OID: oid, Size: size}}} t.Run("upload of an object in the repository needs no transfer", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, upload) require.NoError(t, err) assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) }) - t.Run("upload of an object in another repository", func(t *testing.T) { - t.Run("needs a transfer without cross-repo dedup", func(t *testing.T) { - s := newServiceWithObject(t, false) - resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) - require.NoError(t, err) - require.Len(t, resp.Objects, 1) - assert.Contains(t, resp.Objects[0].Actions, "upload") - - resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) - require.NoError(t, err) - require.NotNil(t, resp.Objects[0].Error) - assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) - }) + t.Run("upload of an object in another repository needs a transfer", func(t *testing.T) { + s := newServiceWithObject(t) + resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + assert.Contains(t, resp.Objects[0].Actions, "upload") - t.Run("links the object with cross-repo dedup", func(t *testing.T) { - s := newServiceWithObject(t, true) - resp, err := s.Batch(ctx, discardLogger, "repo2", endpoint, upload) - require.NoError(t, err) - assert.Equal(t, []BatchResponseObject{{OID: oid, Size: size}}, resp.Objects) + resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + require.NoError(t, err) + require.NotNil(t, resp.Objects[0].Error) + assert.Equal(t, http.StatusNotFound, resp.Objects[0].Error.Code) + }) - resp, err = s.Batch(ctx, discardLogger, "repo2", endpoint, download) + t.Run("presigned upload actions carry a new upload ID per object", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + req := BatchRequest{Operation: OperationUpload, Objects: []BatchObject{ + {OID: sha256Hex("a"), Size: 1}, + {OID: sha256Hex("b"), Size: 1}, + }} + seen := map[string]bool{} + for range 2 { + resp, err := s.Batch(ctx, discardLogger, "repo1", endpoint, req) require.NoError(t, err) - assert.Contains(t, resp.Objects[0].Actions, "download") - }) + require.Len(t, resp.Objects, 2) + for _, o := range resp.Objects { + uploadID := uploadIDOf(t, o) + assert.Equal(t, endpoint.Href+"/"+o.OID+"?upload="+uploadID, o.Actions["upload"].Href) + assert.Equal(t, endpoint.Href+"/verify?upload="+uploadID, o.Actions["verify"].Href) + assert.False(t, seen[uploadID], "upload ID %s is reused", uploadID) + seen[uploadID] = true + } + } }) t.Run("actions carry the endpoint", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) endpoint := Endpoint{Href: endpoint.Href, Header: map[string]string{"Authorization": "Basic Zm9vOmJhcg=="}, Authenticated: true} newContent := "new content" newOID := sha256Hex(newContent) @@ -310,7 +325,7 @@ func TestService_Batch(t *testing.T) { }) t.Run("rejects invalid requests", func(t *testing.T) { - s := newServiceWithObject(t, false) + s := newServiceWithObject(t) for _, req := range []BatchRequest{ {Operation: "delete"}, {Operation: OperationDownload, Objects: []BatchObject{{OID: "not-an-oid"}}}, @@ -323,14 +338,45 @@ func TestService_Batch(t *testing.T) { }) } +// uploadIDOf returns the upload ID of the upload action of o, after checking +// that the verify action carries the same one. +func uploadIDOf(t *testing.T, o BatchResponseObject) string { + t.Helper() + require.Contains(t, o.Actions, "upload") + uploadHref, err := url.Parse(o.Actions["upload"].Href) + require.NoError(t, err) + verifyHref, err := url.Parse(o.Actions["verify"].Href) + require.NoError(t, err) + uploadID := uploadHref.Query().Get("upload") + require.NotEmpty(t, uploadID) + require.Equal(t, uploadID, verifyHref.Query().Get("upload")) + return uploadID +} + func TestService_Presign(t *testing.T) { ctx := t.Context() content := "presigned content" oid := sha256Hex(content) size := int64(len(content)) + verifyReq := VerifyRequest{OID: oid, Size: size} - // uploadToRedirect sends the content to the presigned URL like a client. - uploadToRedirect := func(t *testing.T, redirect *Redirect) { + // startUpload asks for an upload of the content to repo like a client, and + // returns the upload ID and the presigned URL to send the content to. + startUpload := func(t *testing.T, s *Service, repo string) (string, *Redirect) { + t.Helper() + resp, err := s.Batch(ctx, discardLogger, repo, Endpoint{Href: "https://lfs.example.com/objects"}, BatchRequest{ + Operation: OperationUpload, + Objects: []BatchObject{{OID: oid, Size: size}}, + }) + require.NoError(t, err) + require.Len(t, resp.Objects, 1) + uploadID := uploadIDOf(t, resp.Objects[0]) + result, err := s.Upload(ctx, discardLogger, repo, oid, uploadID, size, nil) + require.NoError(t, err) + require.NotNil(t, result.Redirect) + return uploadID, result.Redirect + } + sendContent := func(t *testing.T, redirect *Redirect) { t.Helper() req, err := http.NewRequestWithContext(ctx, http.MethodPut, redirect.URL, strings.NewReader(content)) require.NoError(t, err) @@ -343,36 +389,83 @@ func TestService_Presign(t *testing.T) { require.NoError(t, resp.Body.Close()) require.Equal(t, http.StatusOK, resp.StatusCode) } + upload := func(t *testing.T, s *Service, repo string) { + t.Helper() + uploadID, redirect := startUpload(t, s, repo) + sendContent(t, redirect) + require.NoError(t, s.Verify(ctx, discardLogger, repo, uploadID, verifyReq)) + } + requireDownload := func(t *testing.T, s *Service, repo string) { + t.Helper() + download, err := s.Download(ctx, repo, oid) + require.NoError(t, err) + require.NotNil(t, download.Redirect) + resp, err := http.Get(download.Redirect.URL) + require.NoError(t, err) + got, err := io.ReadAll(resp.Body) + require.NoError(t, resp.Body.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + } - for _, crossRepoDedup := range []bool{false, true} { - t.Run(fmt.Sprintf("crossRepoDedup=%v", crossRepoDedup), func(t *testing.T) { - s := &Service{Store: newMemStore(), Storage: newS3PresignBackend(t), CrossRepoDedup: crossRepoDedup} + t.Run("verify requires the content at the upload location", func(t *testing.T) { + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + uploadID, redirect := startUpload(t, s, "repo1") + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq), http.StatusUnprocessableEntity) + _, err := s.Download(ctx, "repo1", oid) + requireLFSError(t, err, http.StatusNotFound) - result, err := s.Upload(ctx, discardLogger, "repo1", oid, size, nil) - require.NoError(t, err) - require.NotNil(t, result.Redirect) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size}), http.StatusUnprocessableEntity) - _, err = s.Download(ctx, "repo1", oid) - requireLFSError(t, err, http.StatusNotFound) + sendContent(t, redirect) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq)) + requireDownload(t, s, "repo1") + assert.Equal(t, []string{oid}, listKeys(), "the upload is moved to the object location") + }) - uploadToRedirect(t, result.Redirect) - requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size + 1}), http.StatusUnprocessableEntity) - require.NoError(t, s.Verify(ctx, discardLogger, "repo1", VerifyRequest{OID: oid, Size: size})) + t.Run("another repository must upload a stored object", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + uploadID, redirect := startUpload(t, s, "repo1") + sendContent(t, redirect) + require.NoError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq)) + + // The OID and size are public in pointer files, and an upload ID is free + // to get, so none of them link the object without the content. + unusedUploadID, _ := startUpload(t, s, "repo2") + for _, uploadID := range []string{unusedUploadID, uploadID} { + requireLFSError(t, s.Verify(ctx, discardLogger, "repo2", uploadID, verifyReq), http.StatusUnprocessableEntity) + } + _, err := s.Download(ctx, "repo2", oid) + requireLFSError(t, err, http.StatusNotFound) + }) - download, err := s.Download(ctx, "repo1", oid) - require.NoError(t, err) - require.NotNil(t, download.Redirect) - assert.Contains(t, download.Redirect.URL, oid) + t.Run("repositories share one stored copy", func(t *testing.T) { + b, listKeys := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + upload(t, s, "repo1") + upload(t, s, "repo2") - result, err = s.Upload(ctx, discardLogger, "repo1", oid, size, nil) - require.NoError(t, err) - assert.Equal(t, &UploadResult{Existed: true}, result) + requireDownload(t, s, "repo1") + requireDownload(t, s, "repo2") + assert.Equal(t, []string{oid}, listKeys()) + }) - // Without cross-repo dedup, another repository must upload the content - // itself, which the storage checks against the OID. - result, err = s.Upload(ctx, discardLogger, "repo2", oid, size, nil) - require.NoError(t, err) - assert.Equal(t, crossRepoDedup, result.Redirect == nil) - }) - } + t.Run("rejects invalid upload IDs", func(t *testing.T) { + b, _ := newS3PresignBackend(t) + s := &Service{Store: newMemStore(), Storage: b} + v7 := uuid.Must(uuid.NewV7()).String() + for _, uploadID := range []string{ + "", + "not-a-uuid", + uuid.NewString(), // Version 4. + strings.ToUpper(v7), + "urn:uuid:" + v7, + "../" + v7, + } { + _, err := s.Upload(ctx, discardLogger, "repo1", oid, uploadID, size, nil) + requireLFSError(t, err, http.StatusBadRequest) + requireLFSError(t, s.Verify(ctx, discardLogger, "repo1", uploadID, verifyReq), http.StatusBadRequest) + } + }) } diff --git a/lfs/storage/s3_presign.go b/lfs/storage/s3_presign.go index ce3208f..2a76b47 100644 --- a/lfs/storage/s3_presign.go +++ b/lfs/storage/s3_presign.go @@ -7,6 +7,7 @@ import ( "net/url" "strconv" "strings" + "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -24,6 +25,20 @@ import ( // gap to budget for. Keep this short so a leaked URL has limited utility. const presignURLTTL = 60 * time.Second +// uploadsPrefix is the key prefix of upload locations. +const uploadsPrefix = "uploads/" + +const ( + // maxCopyObjectSize is the largest object that a single CopyObject request + // can copy. Larger objects are copied in parts. + maxCopyObjectSize = 5 << 30 + // copyPartSize is the size of each part of a multipart copy. All parts but + // the last have the same size, which Cloudflare R2 requires. + copyPartSize = 512 << 20 + // copyPartConcurrency is the number of parts copied at the same time. + copyPartConcurrency = 8 +) + var _ Presigner = (*S3PresignBackend)(nil) type S3PresignBackend struct { @@ -87,16 +102,26 @@ func (b *S3PresignBackend) URI(oid string) string { return b.scheme + b.bucket + "/" + oid } -// PresignPut returns a presigned PUT URL that pins the body's SHA-256 to oid -// via x-amz-checksum-sha256, so the backend server-side validates the upload -// and rejects mismatches with HTTP 400 BadDigest. Content-Length is also +// UploadURI returns the {scheme}{bucket}/uploads/{uploadID}/{oid} location. +// Pure function, safe to call without I/O. +func (b *S3PresignBackend) UploadURI(uploadID, oid string) string { + return b.scheme + b.bucket + "/" + uploadsPrefix + uploadID + "/" + oid +} + +// PresignPut returns a presigned PUT URL to uri that pins the body's SHA-256 to +// oid via x-amz-checksum-sha256, so the backend server-side validates the +// upload and rejects mismatches with HTTP 400 BadDigest. Content-Length is also // included so the size is signed. // // Note: x-amz-content-sha256 cannot be used here. AWS sigv4 mandates // UNSIGNED-PAYLOAD as the canonical-request HashedPayload for query-string // presigned URLs (S3 sigv4-query-string-auth spec), so signing the body hash // into x-amz-content-sha256 produces a signature that R2 rejects. -func (b *S3PresignBackend) PresignPut(ctx context.Context, oid string, size int64) (string, map[string]string, error) { +func (b *S3PresignBackend) PresignPut(ctx context.Context, uri, oid string, size int64) (string, map[string]string, error) { + key, err := b.keyFromURI(uri) + if err != nil { + return "", nil, err + } rawOID, err := hex.DecodeString(oid) if err != nil { return "", nil, errors.Wrapf(err, "decode oid %q", oid) @@ -105,7 +130,7 @@ func (b *S3PresignBackend) PresignPut(ctx context.Context, oid string, size int6 req, err := b.presignClient.PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(oid), + Key: aws.String(key), ContentLength: aws.Int64(size), ChecksumSHA256: aws.String(checksumB64), }, func(o *s3.PresignOptions) { @@ -189,6 +214,151 @@ func (b *S3PresignBackend) Delete(ctx context.Context, uri string) error { return nil } +// Copy copies the object with CopyObject, or in parts with UploadPartCopy when +// it is larger than what CopyObject supports. +func (b *S3PresignBackend) Copy(ctx context.Context, srcURI, dstURI string, size int64) error { + srcKey, err := b.keyFromURI(srcURI) + if err != nil { + return err + } + dstKey, err := b.keyFromURI(dstURI) + if err != nil { + return err + } + // Keys are made of hex digits, UUIDs, and slashes, which need no escaping. + copySource := b.bucket + "/" + srcKey + if size <= maxCopyObjectSize { + if _, err := b.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + CopySource: aws.String(copySource), + }); err != nil { + return errors.Wrapf(err, "copy object %q to %q", srcKey, dstKey) + } + return nil + } + if err := b.copyMultipart(ctx, copySource, dstKey, size); err != nil { + return errors.Wrapf(err, "copy object %q to %q in parts", srcKey, dstKey) + } + return nil +} + +func (b *S3PresignBackend) copyMultipart(ctx context.Context, copySource, dstKey string, size int64) error { + created, err := b.client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + }) + if err != nil { + return errors.Wrap(err, "create multipart upload") + } + abort := func(cause error) error { + // Abort even when ctx is canceled, so the parts copied so far do not + // linger in the storage. + if _, err := b.client.AbortMultipartUpload(context.WithoutCancel(ctx), &s3.AbortMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + }); err != nil { + return errors.WithSecondaryError(cause, errors.Wrap(err, "abort multipart upload")) + } + return cause + } + + ranges := copyPartRanges(size, copyPartSize) + parts := make([]s3types.CompletedPart, len(ranges)) + partsCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + slots := make(chan struct{}, copyPartConcurrency) + var wg sync.WaitGroup + for i, byteRange := range ranges { + partNumber := aws.Int32(int32(i + 1)) + wg.Go(func() { + select { + case slots <- struct{}{}: + defer func() { <-slots }() + case <-partsCtx.Done(): + return + } + out, err := b.client.UploadPartCopy(partsCtx, &s3.UploadPartCopyInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + PartNumber: partNumber, + CopySource: aws.String(copySource), + CopySourceRange: aws.String(byteRange), + }) + if err != nil { + cancel(errors.Wrapf(err, "copy part %d", *partNumber)) + return + } + parts[i] = s3types.CompletedPart{ETag: out.CopyPartResult.ETag, PartNumber: partNumber} + }) + } + wg.Wait() + if err := context.Cause(partsCtx); err != nil { + return abort(err) + } + + if _, err := b.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(dstKey), + UploadId: created.UploadId, + MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts}, + }); err != nil { + return abort(errors.Wrap(err, "complete multipart upload")) + } + return nil +} + +// copyPartRanges splits size bytes into consecutive HTTP byte ranges of +// partSize bytes, except the last which holds the remainder. +func copyPartRanges(size, partSize int64) []string { + ranges := make([]string, 0, (size+partSize-1)/partSize) + for start := int64(0); start < size; start += partSize { + end := min(start+partSize, size) - 1 + ranges = append(ranges, "bytes="+strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(end, 10)) + } + return ranges +} + +// DeleteUploads deletes the objects under the uploads/ prefix that were last +// modified before the cutoff. +func (b *S3PresignBackend) DeleteUploads(ctx context.Context, before time.Time) (int, error) { + var deleted int + pages := s3.NewListObjectsV2Paginator(b.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(b.bucket), + Prefix: aws.String(uploadsPrefix), + }) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + if err != nil { + return deleted, errors.Wrap(err, "list uploads") + } + var stale []s3types.ObjectIdentifier + for _, o := range page.Contents { + if o.LastModified != nil && o.LastModified.Before(before) { + stale = append(stale, s3types.ObjectIdentifier{Key: o.Key}) + } + } + if len(stale) == 0 { + continue + } + out, err := b.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(b.bucket), + Delete: &s3types.Delete{Objects: stale, Quiet: aws.Bool(true)}, + }) + if err != nil { + return deleted, errors.Wrap(err, "delete uploads") + } + deleted += len(stale) - len(out.Errors) + if len(out.Errors) > 0 { + e := out.Errors[0] + return deleted, errors.Newf("delete upload %q: %s", aws.ToString(e.Key), aws.ToString(e.Message)) + } + } + return deleted, nil +} + // keyFromURI extracts the object key from a {scheme}{bucket}/{key} URI. The // bucket is asserted to match the configured backend's bucket so a stale URI // from a different bucket can't accidentally read or write the wrong bytes. diff --git a/lfs/storage/s3_presign_test.go b/lfs/storage/s3_presign_test.go index d48cfb6..70c49b1 100644 --- a/lfs/storage/s3_presign_test.go +++ b/lfs/storage/s3_presign_test.go @@ -9,6 +9,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" @@ -29,10 +30,10 @@ const ( // test bucket, and returns an S3PresignBackend pointed at the fake. The // fake's Server http.Handler accepts S3 API calls from any aws-sdk-go-v2 // client configured with path-style and a custom BaseEndpoint. -func newTestS3PresignBackend(t *testing.T) (*S3PresignBackend, *httptest.Server) { +func newTestS3PresignBackend(t *testing.T, opts ...s3mem.Option) (*S3PresignBackend, *httptest.Server) { t.Helper() - mem := s3mem.New() + mem := s3mem.New(opts...) require.NoError(t, mem.CreateBucket(testBucket)) srv := httptest.NewServer(gofakes3.New(mem).Server()) @@ -52,6 +53,7 @@ func TestS3PresignBackend_URI(t *testing.T) { b, _ := newTestS3PresignBackend(t) oid := sha256Hex("uri-test") assert.Equal(t, "r2://"+testBucket+"/"+oid, b.URI(oid)) + assert.Equal(t, "r2://"+testBucket+"/uploads/upload-id/"+oid, b.UploadURI("upload-id", oid)) } func TestS3PresignBackend_PresignPut(t *testing.T) { @@ -60,7 +62,8 @@ func TestS3PresignBackend_PresignPut(t *testing.T) { oid := sha256Hex(body) size := int64(len(body)) - url, headers, err := b.PresignPut(t.Context(), oid, size) + uploadURI := b.UploadURI("upload-id", oid) + url, headers, err := b.PresignPut(t.Context(), uploadURI, oid, size) require.NoError(t, err) rawOID, err := hex.DecodeString(oid) @@ -90,9 +93,11 @@ func TestS3PresignBackend_PresignPut(t *testing.T) { _ = resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) - gotSize, err := b.Head(t.Context(), b.URI(oid)) + gotSize, err := b.Head(t.Context(), uploadURI) require.NoError(t, err) assert.Equal(t, size, gotSize) + _, err = b.Head(t.Context(), b.URI(oid)) + assert.True(t, errors.Is(err, ErrNotFound), "only the upload location is written") }) } @@ -197,6 +202,90 @@ func TestS3PresignBackend_Delete(t *testing.T) { }) } +func TestS3PresignBackend_Copy(t *testing.T) { + b, _ := newTestS3PresignBackend(t) + body := "copy me" + oid := sha256Hex(body) + src := b.UploadURI("upload-id", oid) + putObject(t, b, "uploads/upload-id/"+oid, body) + + require.NoError(t, b.Copy(t.Context(), src, b.URI(oid), int64(len(body)))) + assert.Equal(t, body, getObject(t, b, oid)) + _, err := b.Head(t.Context(), src) + require.NoError(t, err, "the source is kept") + + err = b.Copy(t.Context(), b.UploadURI("missing", oid), b.URI(sha256Hex("other")), 1) + require.Error(t, err) +} + +func TestCopyPartRanges(t *testing.T) { + tests := []struct { + name string + size, partSize int64 + want []string + }{ + {"remainder in last part", 10, 4, []string{"bytes=0-3", "bytes=4-7", "bytes=8-9"}}, + {"exact multiple", 8, 4, []string{"bytes=0-3", "bytes=4-7"}}, + {"smaller than a part", 1, 4, []string{"bytes=0-0"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, copyPartRanges(tc.size, tc.partSize)) + }) + } + + t.Run("just over the CopyObject limit", func(t *testing.T) { + ranges := copyPartRanges(maxCopyObjectSize+1, copyPartSize) + require.Len(t, ranges, 11) + assert.Equal(t, "bytes=0-536870911", ranges[0]) + assert.Equal(t, "bytes=5368709120-5368709120", ranges[10]) + }) +} + +func TestS3PresignBackend_DeleteUploads(t *testing.T) { + clock := gofakes3.FixedTimeSource(time.Now()) + b, _ := newTestS3PresignBackend(t, s3mem.WithTimeSource(clock)) + oid := sha256Hex("upload") + putObject(t, b, "uploads/stale/"+oid, "upload") + putObject(t, b, oid, "upload") + clock.Advance(2 * time.Hour) + putObject(t, b, "uploads/recent/"+oid, "upload") + + deleted, err := b.DeleteUploads(t.Context(), clock.Now().Add(-time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, deleted) + + _, err = b.Head(t.Context(), b.UploadURI("stale", oid)) + assert.True(t, errors.Is(err, ErrNotFound), "a stale upload is deleted") + _, err = b.Head(t.Context(), b.UploadURI("recent", oid)) + assert.NoError(t, err, "a recent upload is kept") + _, err = b.Head(t.Context(), b.URI(oid)) + assert.NoError(t, err, "an object outside uploads/ is kept regardless of its age") +} + +func putObject(t *testing.T, b *S3PresignBackend, key, body string) { + t.Helper() + _, err := b.client.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(key), + Body: strings.NewReader(body), + }) + require.NoError(t, err) +} + +func getObject(t *testing.T, b *S3PresignBackend, key string) string { + t.Helper() + out, err := b.client.GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String(testBucket), + Key: aws.String(key), + }) + require.NoError(t, err) + defer func() { _ = out.Body.Close() }() + got, err := io.ReadAll(out.Body) + require.NoError(t, err) + return string(got) +} + func TestNewS3PresignBackend(t *testing.T) { t.Run("required fields", func(t *testing.T) { cases := []struct { diff --git a/lfs/storage/storage.go b/lfs/storage/storage.go index 6bb95ae..2bb2ff3 100644 --- a/lfs/storage/storage.go +++ b/lfs/storage/storage.go @@ -3,6 +3,7 @@ package storage import ( "context" "io" + "time" "github.com/cockroachdb/errors" ) @@ -53,15 +54,29 @@ type Proxier interface { // Presigner is implemented by backends that validate object integrity when // directly uploading from clients, and reject uploads with mismatched content. +// +// Each upload goes to its own upload location, so that an object at an upload +// location proves that the upload happened. The server copies the content to +// the canonical location of the object once the upload is verified. type Presigner interface { Backend - // PresignPut returns a short-lived URL the client PUTs to, along with the - // headers the client MUST send verbatim. - PresignPut(ctx context.Context, oid string, size int64) (url string, headers map[string]string, err error) + // UploadURI returns the location that the upload with the given ID sends the + // content of oid to. Pure function, safe to compute without I/O. + UploadURI(uploadID, oid string) string + // PresignPut returns a short-lived URL the client PUTs the content of oid to, + // along with the headers the client MUST send verbatim. The object is stored + // at uri, and only when the content matches oid and size. + PresignPut(ctx context.Context, uri, oid string, size int64) (url string, headers map[string]string, err error) // PresignGet returns a short-lived URL the client GETs from. PresignGet(ctx context.Context, uri string) (url string, err error) // Head returns the size of the object. It returns ErrNotFound when no matching // object exists. Head(ctx context.Context, uri string) (size int64, err error) + // Copy copies the object of size bytes at srcURI to dstURI within the + // storage, replacing any object at dstURI. + Copy(ctx context.Context, srcURI, dstURI string, size int64) error + // DeleteUploads deletes the objects at upload locations that were last + // modified before the cutoff, and returns how many it deleted. + DeleteUploads(ctx context.Context, before time.Time) (deleted int, err error) }