Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com

At least one `[forge "{host}"]` section must be configured. Each forge references a `[storage "{name}"]` section by name. Multiple forges may share a single storage backend. Supported storage types are `filesystem` and `s3-presign` (compatible with S3, R2, DigitalOcean Spaces, etc.).

An `s3-presign` bucket also holds in-progress uploads under the `uploads/` prefix. They are moved into place when the client verifies the upload, and abandoned ones are deleted automatically after a day.

### Client

Git LFS resolves the LFS endpoint per repository. To point a clone at lfsd, set `lfs.url` in the repository's `.lfsconfig` (committed so collaborators inherit it) to `EXTERNAL_URL` joined with the forge host, the repository path, and the `/info/lfs` suffix:
Expand Down
7 changes: 3 additions & 4 deletions cmd/lfsd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,9 @@ func main() {
services := make(map[string]*lfs.Service, len(config.Storages))
for host, backend := range config.Storages {
services[host] = &lfs.Service{
Store: db,
Storage: backend,
MaxObjectSize: config.Server.MaxObjectSize,
CrossRepoDedup: true,
Store: db,
Storage: backend,
MaxObjectSize: config.Server.MaxObjectSize,
}
}
serve := serveLFS(logger.Scoped("lfs"), config.Server.ExternalURL, services)
Expand Down
11 changes: 8 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ go 1.27
require (
charm.land/lipgloss/v2 v2.0.3
charm.land/log/v2 v2.0.0
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1
github.com/cockroachdb/errors v1.13.0
github.com/flamego/cache v1.5.1
github.com/flamego/flamego v1.9.7
github.com/johannesboyne/gofakes3 v0.0.0-20260208201424-4c385a1f6a73
github.com/lib/pq v1.12.3
github.com/sourcegraph/conc v0.3.0
github.com/sourcegraph/run v0.12.0
Expand All @@ -23,17 +27,14 @@ require (
require (
bitbucket.org/creachadair/shell v0.0.7 // indirect
github.com/alecthomas/participle/v2 v2.1.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 // indirect
github.com/aws/smithy-go v1.25.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/colorprofile v0.4.3 // indirect
Expand All @@ -57,6 +58,7 @@ require (
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/itchyny/gojq v0.12.14 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand All @@ -76,13 +78,16 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/otel v1.11.0 // indirect
go.opentelemetry.io/otel/trace v1.11.0 // indirect
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/tools v0.48.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

Expand Down
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
Expand All @@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q=
github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q=
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
Expand Down Expand Up @@ -97,6 +101,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hexops/autogold/v2 v2.2.1 h1:JPUXuZQGkcQMv7eeDXuNMovjfoRYaa0yVcm+F3voaGY=
github.com/hexops/autogold/v2 v2.2.1/go.mod h1:IJwxtUfj1BGLm0YsR/k+dIxYi6xbeLjqGke2bzcOTMI=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
Expand Down Expand Up @@ -161,6 +167,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I=
github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA=
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand All @@ -172,6 +180,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.bobheadxi.dev/streamline v1.3.2 h1:EwbDggkws9Qo/fl4Zo801Z8mgf4xZBOX7/bbfTthsy4=
go.bobheadxi.dev/streamline v1.3.2/go.mod h1:QSS0MlQm+3ABEr0uMYrkAGYiILwJvTIGFygVARxDFdg=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk=
go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk=
go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo=
Expand Down Expand Up @@ -222,6 +232,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
111 changes: 95 additions & 16 deletions integration-tests/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
package main

import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
Expand All @@ -19,9 +22,15 @@ import (
"testing"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/sourcegraph/run"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.bobheadxi.dev/streamline/streamexec"

"unknwon.dev/git-lfs-server/lfs"
)

var long = flag.Bool("long", false, "run long-running integration tests")
Expand Down Expand Up @@ -67,10 +76,10 @@ func TestUploadDownloadRoundtrip(t *testing.T) {
})

pushDir := t.TempDir()
blob := genRandomBlob(t, 1*1024*1024)
wantSum := sha256.Sum256(blob)
content := genRandomContent(t, 1*1024*1024)
wantSum := sha256.Sum256(content)

require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), blob, 0o644))
require.NoError(t, os.WriteFile(filepath.Join(pushDir, "large.bin"), content, 0o644))
require.NoError(t, os.WriteFile(filepath.Join(pushDir, ".gitattributes"),
[]byte("*.bin filter=lfs diff=lfs merge=lfs -text\n"), 0o644))

Expand All @@ -85,21 +94,91 @@ func TestUploadDownloadRoundtrip(t *testing.T) {
git("remote", "add", "origin", remoteURL)
git("push", "origin", "HEAD:refs/heads/"+branch)

// Skip smudge during clone so the LFS object isn't fetched as a side effect of
// checkout. Then explicitly pull to exercise the download path against lfsd.
pullDir := t.TempDir()
require.NoError(t,
run.Cmd(ctx, "git", "clone", "--branch", branch, "--depth", "1", remoteURL, pullDir).
Environ([]string{"GIT_LFS_SKIP_SMUDGE=1", "GIT_TERMINAL_PROMPT=0"}).Run().Wait())
oid := hex.EncodeToString(wantSum[:])
// Check out in a fresh clone, so the smudge filter downloads the LFS object
// from lfsd.
requireClone := func(lfsURL string) {
t.Helper()
cloneDir := t.TempDir()
require.NoError(t,
run.Cmd(ctx, "git", "clone", "--no-checkout", "--branch", branch, "--depth", "1", remoteURL, cloneDir).
Environ([]string{"GIT_TERMINAL_PROMPT=0"}).Run().Wait())
cloneGit := gitCmd(ctx, t, cloneDir)
cloneGit("config", "lfs.url", lfsURL)
cloneGit("checkout", branch)
got, err := os.ReadFile(filepath.Join(cloneDir, "large.bin"))
require.NoError(t, err)
require.True(t, bytes.Equal(content, got), "downloaded content differs from the pushed content")
}
requireClone(lfsURL)

// The same object in another repository is only linked to it by uploading
// the content again.
otherLFSURL := "http://x:x@127.0.0.1:3356/local.test/unknwon/git-lfs-test/info/lfs"
resp := batch(ctx, t, otherLFSURL, lfs.BatchRequest{
Operation: lfs.OperationDownload,
Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}},
})
require.NotNil(t, resp.Objects[0].Error, "another repository must not download the object before uploading it")
resp = batch(ctx, t, otherLFSURL, lfs.BatchRequest{
Operation: lfs.OperationUpload,
Objects: []lfs.BatchObject{{OID: oid, Size: int64(len(content))}},
})
require.Contains(t, resp.Objects[0].Actions, "upload", "another repository must upload the object")

pullGit := gitCmd(ctx, t, pullDir)
pullGit("config", "lfs.url", lfsURL)
pullGit("lfs", "pull")
git("config", "lfs.url", otherLFSURL)
git("lfs", "push", "--object-id", "origin", oid)
requireClone(otherLFSURL)

got, err := os.ReadFile(filepath.Join(pullDir, "large.bin"))
// Both repositories share one copy, and no upload is left behind.
client := s3.NewFromConfig(aws.Config{
Region: "auto",
Credentials: credentials.NewStaticCredentialsProvider(
os.Getenv("LFSD_R2_ACCESS_KEY_ID"), os.Getenv("LFSD_R2_SECRET_ACCESS_KEY"), ""),
}, func(o *s3.Options) {
o.BaseEndpoint = aws.String(fmt.Sprintf("https://%s.r2.cloudflarestorage.com", os.Getenv("LFSD_R2_ACCOUNT_ID")))
o.UsePathStyle = true
})
bucket := aws.String(os.Getenv("LFSD_R2_BUCKET"))
head, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: bucket, Key: aws.String(oid)})
require.NoError(t, err)
gotSum := sha256.Sum256(got)
require.Equal(t, hex.EncodeToString(wantSum[:]), hex.EncodeToString(gotSum[:]))
assert.Equal(t, int64(len(content)), aws.ToInt64(head.ContentLength))
pages := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: bucket, Prefix: aws.String("uploads/")})
for pages.HasMorePages() {
page, err := pages.NextPage(ctx)
require.NoError(t, err)
for _, o := range page.Contents {
assert.False(t, strings.HasSuffix(aws.ToString(o.Key), "/"+oid), "upload %q is left behind", aws.ToString(o.Key))
}
}
}

// batch sends a batch request to the LFS endpoint at lfsURL, whose user info
// is sent as HTTP Basic Auth credentials.
func batch(ctx context.Context, t *testing.T, lfsURL string, req lfs.BatchRequest) *lfs.BatchResponse {
t.Helper()
u, err := url.Parse(lfsURL + "/objects/batch")
require.NoError(t, err)
password, _ := u.User.Password()
username := u.User.Username()
u.User = nil

body, err := json.Marshal(req)
require.NoError(t, err)
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(body))
require.NoError(t, err)
httpReq.SetBasicAuth(username, password)
httpReq.Header.Set("Content-Type", lfs.MediaType)
httpReq.Header.Set("Accept", lfs.MediaType)
httpResp, err := http.DefaultClient.Do(httpReq)
require.NoError(t, err)
defer func() { _ = httpResp.Body.Close() }()
require.Equal(t, http.StatusOK, httpResp.StatusCode)

var resp lfs.BatchResponse
require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&resp))
require.Len(t, resp.Objects, len(req.Objects))
return &resp
}

func requireEnv(t *testing.T, names ...string) {
Expand Down Expand Up @@ -213,7 +292,7 @@ func gitCmd(ctx context.Context, t *testing.T, dir string) func(args ...string)
}
}

func genRandomBlob(t *testing.T, size int) []byte {
func genRandomContent(t *testing.T, size int) []byte {
t.Helper()
buf := make([]byte, size)
_, err := rand.Read(buf)
Expand Down
7 changes: 7 additions & 0 deletions integration-tests/testdata/config.ini
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,10 @@ ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com
TYPE = github
STORAGE = r2
REPO_ALLOWLIST = unknwon/git-lfs-test

; Serves the same repository under another name, so that the test uploads the
; same object to two repositories. lfsd only listens on localhost in tests.
[forge "local.test"]
SKIP_AUTH = true
STORAGE = r2
REPO_ALLOWLIST = unknwon/git-lfs-test
18 changes: 0 additions & 18 deletions internal/database/objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,24 +156,6 @@ WHERE id = @id`,
return storedURI, nil
}

// GetObjects fetches verified objects for the given OIDs. Results are keyed by
// OID, silently dropping missing OIDs.
func (d *DB) GetObjects(ctx context.Context, oids []string) (map[string]lfs.Object, error) {
if len(oids) == 0 {
return map[string]lfs.Object{}, nil
}
var rows []Object
err := d.db.WithContext(ctx).Raw(`
SELECT * FROM objects
WHERE oid = ANY(@oids) AND verified_at IS NOT NULL`,
map[string]any{"oids": pq.Array(oids)},
).Scan(&rows).Error
if err != nil {
return nil, errors.Wrap(err, "select objects by OIDs")
}
return verifiedObjectsByOID(rows), nil
}

// GetRepoObjects fetches verified objects for the given OIDs that are linked to
// the repository. Results are keyed by OID, silently dropping missing OIDs.
func (d *DB) GetRepoObjects(ctx context.Context, repoName string, oids []string) (map[string]lfs.Object, error) {
Expand Down
23 changes: 21 additions & 2 deletions internal/janitor/janitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const (
)

// Janitor sweeps orphan (never verified) and unreferenced (repo_count = 0)
// objects.
// objects, and abandoned presigned uploads.
type Janitor struct {
db *database.DB
storages []storage.Backend
Expand Down Expand Up @@ -58,9 +58,11 @@ func (j *Janitor) Run(ctx context.Context, logger *logx.Logger) {
}
}

// Sweep runs both cleanup passes once. Errors are logged but not returned.
// Sweep runs all cleanup passes once. Errors are logged but not returned.
// The next tick will retry.
func (j *Janitor) Sweep(ctx context.Context, logger *logx.Logger) {
j.deleteUploads(ctx, logger.Scoped("uploads"), time.Now().Add(-orphanAge))

if objects, err := j.db.SweepOrphanObjects(ctx, orphanAge, sweepBatchSize); err != nil {
logger.ErrorContext(ctx, "Failed to sweep orphan objects", "error", err)
} else if len(objects) > 0 {
Expand Down Expand Up @@ -113,6 +115,23 @@ func (j *Janitor) deleteObjects(ctx context.Context, logger *logx.Logger, object
logFn(ctx, "Swept objects", "deletedCount", deletedCount, "failedCount", failedCount)
}

// deleteUploads deletes the content of presigned uploads last modified before
// the cutoff, which were never verified or failed to be deleted after.
func (j *Janitor) deleteUploads(ctx context.Context, logger *logx.Logger, before time.Time) {
for _, b := range j.storages {
p, ok := b.(storage.Presigner)
if !ok {
continue
}
deletedCount, err := p.DeleteUploads(ctx, before)
if err != nil {
logger.ErrorContext(ctx, "Failed to delete stale uploads", "storage", b.Name(), "deletedCount", deletedCount, "error", err)
} else if deletedCount > 0 {
logger.InfoContext(ctx, "Deleted stale uploads", "storage", b.Name(), "deletedCount", deletedCount)
}
}
}

func (j *Janitor) deleteByURI(ctx context.Context, uri string) error {
for _, b := range j.storages {
if strings.HasPrefix(uri, b.Scheme()) {
Expand Down
Loading
Loading