diff --git a/docs/configuration.md b/docs/configuration.md index d3d828e4..7af560b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2137,7 +2137,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Versions numbered by year, such as certifi's `2025.1.31`, are all one line. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI, conda and gem versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist; conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type; and gems of the same platform. +- PyPI, conda and gem versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist; conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type; and gems of the same platform. When some of those files of the newest release are cached and others aren't, say because a run failed on one, the next run fetches the missing ones. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`; without it the proxy logs a warning at startup and refreshes the other ecosystems only. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 1f1ca12a..92cb50a0 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/metadata/pypi" "github.com/git-pkgs/registries" @@ -257,6 +258,21 @@ func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool return true } +// missing reports whether a file of release that pv selects isn't among +// the cached files, such as one a refresh failed to fetch after fetching +// another of the release. +func (e fileEcosystem) missing(pv PackageVersion, release registries.Version, cached []database.UsedFile) bool { + for _, file := range release.Artifacts { + if !e.selects(pv, file) { + continue + } + if _, _, filename, ok := e.cacheKey(pv, file); ok && !slices.ContainsFunc(cached, func(f database.UsedFile) bool { return f.Filename == filename }) { + return true + } + } + return false +} + // fileQualifiers keeps the qualifiers of a PURL that select files of its // ecosystem, or returns nil. func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string { diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index b1909726..54a4dc24 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -155,22 +155,34 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt) } - var items []PackageVersion - for _, version := range newestPerCachedLine(ecosystem, cached, releases, allowed) { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: version} - if eco, ok := fileEcosystems[ecosystem]; ok { - pv.variants = map[string]bool{} - for _, f := range pkg.Files { - for _, v := range eco.cachedVariants(f.Filename) { - pv.variants[v] = true - } - } + eco, files := fileEcosystems[ecosystem] + var variants map[string]bool + var incomplete func(registries.Version) bool + if files { + variants = cachedVariants(eco, pkg.Files) + incomplete = func(r registries.Version) bool { + return eco.missing(PackageVersion{Ecosystem: ecosystem, Name: name, Version: r.Number, variants: variants}, r, pkg.Files) } - items = append(items, pv) + } + + var items []PackageVersion + for _, version := range newestPerCachedLine(ecosystem, cached, releases, allowed, incomplete) { + items = append(items, PackageVersion{Ecosystem: ecosystem, Name: name, Version: version, variants: variants}) } return items, nil } +// cachedVariants returns the variants of the cached files. +func cachedVariants(eco fileEcosystem, files []database.UsedFile) map[string]bool { + variants := map[string]bool{} + for _, f := range files { + for _, v := range eco.cachedVariants(f.Filename) { + variants[v] = true + } + } + return variants +} + // refreshable reports whether a refresh lists the releases of an // ecosystem's packages. Images follow mirror.schedules, OS packages their // repository indexes, and Julia files are cached by tree hash rather than by diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 0af855d7..c8440188 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -140,7 +140,7 @@ func TestNewestPerCachedLineGo(t *testing.T) { {"pseudo-versions are prereleases", []string{"v1.2.3"}, releases("v1.2.3", "v1.2.4-0.20260101000000-abcdefabcdef"), nil}, } { t.Run(tt.name, func(t *testing.T) { - if got := newestPerCachedLine("golang", tt.cached, tt.releases, allowAll); !reflect.DeepEqual(got, tt.want) { + if got := newestPerCachedLine("golang", tt.cached, tt.releases, allowAll, nil); !reflect.DeepEqual(got, tt.want) { t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) } }) diff --git a/internal/mirror/refresh_retry_test.go b/internal/mirror/refresh_retry_test.go new file mode 100644 index 00000000..7c02c074 --- /dev/null +++ b/internal/mirror/refresh_retry_test.go @@ -0,0 +1,151 @@ +package mirror + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/git-pkgs/registries" +) + +// cachedFile is a file a client downloaded: the version and filename the +// download handler cached it under. +type cachedFile struct{ version, filename string } + +// refreshRetries caches cached of name the way a client would, lists +// releases, and runs three refreshes: the first with the URLs containing +// failing failing, then two without. It checks the first fails one version +// and returns what the second and third fetched. +func refreshRetries(t *testing.T, ecosystem, name string, cached []cachedFile, releases []registries.Version, failing string) (second, third []string) { + t.Helper() + m, fetcher := setupRefreshMirror(t, "file") + ctx := context.Background() + for _, f := range cached { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, ecosystem, name, f.version, f.filename, "https://files.example/"+f.filename, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + for i := range releases { + for j := range releases[i].Artifacts { + releases[i].Artifacts[j].URL = "https://files.example/" + releases[i].Artifacts[j].Filename + } + } + m.Versions = stubReleases{ecosystem + "/" + name: releases} + + refresh := func() (Progress, []string) { + fetcher.reset() + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + got := fetcher.fetched() + slices.Sort(got) + return *progress, got + } + fetcher.failing(failing) + if progress, got := refresh(); progress.Failed != 1 { + t.Fatalf("first refresh: progress = %+v, fetched %v; want one failure", progress, got) + } + fetcher.failing("") + progress, second := refresh() + if progress.Failed != 0 { + t.Errorf("second refresh: progress = %+v, want no failures", progress) + } + _, third = refresh() + return second, third +} + +func release(number string, age time.Duration, files ...registries.Artifact) registries.Version { + return registries.Version{Number: number, PublishedAt: time.Now().Add(-age), Artifacts: files} +} + +const ( + olderRelease = 60 * 24 * time.Hour + newerRelease = 30 * 24 * time.Hour +) + +func TestRefreshRetriesTheMissingFilesOfARelease(t *testing.T) { + conda := func(subdir, filename string) registries.Artifact { + return registries.Artifact{Filename: filename, Metadata: map[string]any{"subdir": subdir, "build": "h5f976f7_0"}} + } + gem := func(filename, platform string) registries.Artifact { + return registries.Artifact{Filename: filename, Metadata: map[string]any{"platform": platform}} + } + tests := []struct { + name, ecosystem, pkg string + cached []cachedFile + releases []registries.Version + failing string + }{ + { + name: "pypi", ecosystem: "pypi", pkg: "requests", + cached: []cachedFile{{"2.32.3", "requests-2.32.3.tar.gz"}, {"2.32.3", "requests-2.32.3-py3-none-any.whl"}}, + releases: []registries.Version{ + release("2.32.3", olderRelease, wheels("requests-2.32.3.tar.gz", "requests-2.32.3-py3-none-any.whl")...), + release("2.32.4", newerRelease, wheels("requests-2.32.4.tar.gz", "requests-2.32.4-py3-none-any.whl")...), + }, + failing: "requests-2.32.4-py3-none-any.whl", + }, + { + name: "conda", ecosystem: "conda", pkg: "conda-forge/zlib", + cached: []cachedFile{{"1.3.0", "linux-64/zlib-1.3.0-h5f976f7_0.conda"}, {"1.3.0", "osx-arm64/zlib-1.3.0-h5f976f7_0.conda"}}, + releases: []registries.Version{ + release("1.3.0", olderRelease, conda("linux-64", "zlib-1.3.0-h5f976f7_0.conda"), conda("osx-arm64", "zlib-1.3.0-h5f976f7_0.conda")), + release("1.3.1", newerRelease, conda("linux-64", "zlib-1.3.1-h5f976f7_0.conda"), conda("osx-arm64", "zlib-1.3.1-h99e199e_0.conda")), + }, + failing: "zlib-1.3.1-h99e199e_0.conda", + }, + { + name: "gem", ecosystem: "gem", pkg: "nokogiri", + cached: []cachedFile{{"1.13.6", "nokogiri-1.13.6.gem"}, {"1.13.6-x86_64-linux", "nokogiri-1.13.6-x86_64-linux.gem"}}, + releases: []registries.Version{ + release("1.13.6", olderRelease, gem("nokogiri-1.13.6.gem", "ruby"), gem("nokogiri-1.13.6-x86_64-linux.gem", "x86_64-linux")), + release("1.13.7", newerRelease, gem("nokogiri-1.13.7.gem", "ruby"), gem("nokogiri-1.13.7-x86_64-linux.gem", "x86_64-linux")), + }, + failing: "nokogiri-1.13.7-x86_64-linux.gem", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + second, third := refreshRetries(t, tt.ecosystem, tt.pkg, tt.cached, tt.releases, tt.failing) + if want := []string{"https://files.example/" + tt.failing}; !slices.Equal(second, want) { + t.Errorf("second refresh fetched %v, want %v", second, want) + } + if len(third) != 0 { + t.Errorf("third refresh fetched %v, want nothing", third) + } + }) + } +} + +// A release missing only kinds of files it doesn't publish, or files the +// download handler wouldn't cache, has nothing to retry, so a refresh +// doesn't look it up again. +func TestRefreshLeavesACompleteReleaseAlone(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + ctx := context.Background() + for _, f := range []cachedFile{{"1.13.6-java", "nokogiri-1.13.6-java.gem"}, {"1.13.7", "nokogiri-1.13.7.gem"}} { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "gem", "nokogiri", f.version, f.filename, "https://rubygems.example/downloads/"+f.filename, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + newest := gemBuilds("1.13.7", newerRelease, "ruby", "arm64-darwin") + newest.Artifacts = append(newest.Artifacts, registries.Artifact{ + URL: "https://rubygems.example/downloads/unreadable", Filename: "unreadable", Metadata: map[string]any{"platform": "ruby"}, + }) + m.Versions = stubReleases{"gem/nokogiri": {gemBuilds("1.13.6", olderRelease, "ruby", "java"), newest}} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetcher.fetched(); len(got) != 0 || progress.Total != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing to do", progress, got) + } +} diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 97624d6d..3bf7c5cb 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -16,11 +16,13 @@ import ( "github.com/git-pkgs/registries/fetch" ) -// servingFetcher answers every fetch with body and records the URLs. +// servingFetcher answers every fetch with body and records the URLs. It +// fails the URLs that contain fail, when set. type servingFetcher struct { body string mu sync.Mutex urls []string + fail string } func (f *servingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { @@ -30,7 +32,11 @@ func (f *servingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact func (f *servingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { f.mu.Lock() f.urls = append(f.urls, url) + fail := f.fail != "" && strings.Contains(url, f.fail) f.mu.Unlock() + if fail { + return nil, fmt.Errorf("fetching %s: timeout", url) + } return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.body)), Size: int64(len(f.body)), ContentType: "application/octet-stream"}, nil } @@ -50,6 +56,12 @@ func (f *servingFetcher) reset() { f.urls = nil } +func (f *servingFetcher) failing(part string) { + f.mu.Lock() + defer f.mu.Unlock() + f.fail = part +} + // stubReleases lists releases by "ecosystem/name". type stubReleases map[string][]registries.Version diff --git a/internal/mirror/refresh_versions.go b/internal/mirror/refresh_versions.go index 22fe7c17..15c11856 100644 --- a/internal/mirror/refresh_versions.go +++ b/internal/mirror/refresh_versions.go @@ -10,11 +10,12 @@ import ( // newestPerCachedLine returns, for each major line a cached version is on, // the newest release of that line that allowed accepts, when it is newer -// than every cached version of the line. A line is the major version, or +// than every cached version of the line, or is the newest cached version +// and incomplete, when given, reports files of it missing. A line is the major version, or // 0.minor below 1.0, as caret ranges read them, and calendar versions are // one line. Prereleases and yanked, // retracted or deprecated releases are skipped. -func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed func(registries.Version) bool) []string { +func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed, incomplete func(registries.Version) bool) []string { scheme := purl.EcosystemToPURLType(ecosystem) newestCached := map[string]string{} var lines []string @@ -32,7 +33,7 @@ func newestPerCachedLine(ecosystem string, cached []string, releases []registrie } } - newest := map[string]string{} + newest := map[string]registries.Version{} for _, r := range releases { line, ok := versionLine(r.Number) if !ok || r.Status != registries.StatusNone || vers.IsPrereleaseWithScheme(r.Number, scheme) { @@ -41,15 +42,17 @@ func newestPerCachedLine(ecosystem string, cached []string, releases []registrie if _, cachedLine := newestCached[line]; !cachedLine || !allowed(r) { continue } - if current, seen := newest[line]; !seen || vers.CompareWithScheme(r.Number, current, scheme) > 0 { - newest[line] = r.Number + if current, seen := newest[line]; !seen || vers.CompareWithScheme(r.Number, current.Number, scheme) > 0 { + newest[line] = r } } var out []string for _, line := range lines { - if v, ok := newest[line]; ok && vers.CompareWithScheme(v, newestCached[line], scheme) > 0 { - out = append(out, v) + if r, ok := newest[line]; ok { + if c := vers.CompareWithScheme(r.Number, newestCached[line], scheme); c > 0 || c == 0 && incomplete != nil && incomplete(r) { + out = append(out, r.Number) + } } } return out diff --git a/internal/mirror/refresh_versions_test.go b/internal/mirror/refresh_versions_test.go index bc889474..47651511 100644 --- a/internal/mirror/refresh_versions_test.go +++ b/internal/mirror/refresh_versions_test.go @@ -58,7 +58,7 @@ func TestNewestPerCachedLine(t *testing.T) { if allowed == nil { allowed = allowAll } - got := newestPerCachedLine("npm", tt.cached, tt.releases, allowed) + got := newestPerCachedLine("npm", tt.cached, tt.releases, allowed, nil) if !reflect.DeepEqual(got, tt.want) { t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) } @@ -68,11 +68,11 @@ func TestNewestPerCachedLine(t *testing.T) { func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) { // Maven orders 1.0.10 after 1.0.9, and a gem's 1.0.0.rc1 is a prerelease. - got := newestPerCachedLine("maven", []string{"1.0.9"}, releases("1.0.9", "1.0.10"), func(registries.Version) bool { return true }) + got := newestPerCachedLine("maven", []string{"1.0.9"}, releases("1.0.9", "1.0.10"), func(registries.Version) bool { return true }, nil) if !reflect.DeepEqual(got, []string{"1.0.10"}) { t.Errorf("maven: got %v, want [1.0.10]", got) } - got = newestPerCachedLine("gem", []string{"1.0.0"}, releases("1.0.0", "1.0.1.rc1"), func(registries.Version) bool { return true }) + got = newestPerCachedLine("gem", []string{"1.0.0"}, releases("1.0.0", "1.0.1.rc1"), func(registries.Version) bool { return true }, nil) if got != nil { t.Errorf("gem: got %v, want nothing", got) } @@ -81,11 +81,11 @@ func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) { func TestNewestPerCachedLineCalendarVersions(t *testing.T) { allowAll := func(registries.Version) bool { return true } // certifi and tzdata number releases by year: 2025.1.31 follows 2024.12.14. - got := newestPerCachedLine("pypi", []string{"2024.8.30"}, releases("2024.8.30", "2024.12.14", "2025.1.31"), allowAll) + got := newestPerCachedLine("pypi", []string{"2024.8.30"}, releases("2024.8.30", "2024.12.14", "2025.1.31"), allowAll, nil) if !reflect.DeepEqual(got, []string{"2025.1.31"}) { t.Errorf("certifi: got %v, want [2025.1.31]", got) } - got = newestPerCachedLine("pypi", []string{"2024.2"}, releases("2024.2", "2025.1"), allowAll) + got = newestPerCachedLine("pypi", []string{"2024.2"}, releases("2024.2", "2025.1"), allowAll, nil) if !reflect.DeepEqual(got, []string{"2025.1"}) { t.Errorf("tzdata: got %v, want [2025.1]", got) }