diff --git a/docs/configuration.md b/docs/configuration.md index e8a7b1ec..dd9b2132 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1966,6 +1966,8 @@ A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdi A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. +A gem version is mirrored with each of its platform builds, such as `nokogiri-1.13.6-x86_64-linux.gem` next to `nokogiri-1.13.6.gem`, and each build is cached the way `/gems/{file}` serves it. The `platform` qualifier selects one, for example `pkg:gem/nokogiri@1.13.6?platform=x86_64-linux`, or `platform=ruby` for the build that installs anywhere. A version with the platform appended, as `Gemfile.lock` records it, selects that build too: `pkg:gem/nokogiri@1.13.6-x86_64-linux`. + A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. @@ -2107,7 +2109,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Versions numbered by year, such as certifi's `2025.1.31`, are all one line. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. +- PyPI, conda and gem versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist; conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type; and gems of the same platform. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`; without it the proxy logs a warning at startup and refreshes the other ecosystems only. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. diff --git a/go.mod b/go.mod index c3409e24..c552f50e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 - github.com/git-pkgs/registries v0.13.2 + github.com/git-pkgs/registries v0.14.0 github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 @@ -75,7 +75,7 @@ require ( github.com/dustin/go-humanize v1.1.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/pom v0.1.9 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.8.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index 4a2270eb..93e42071 100644 --- a/go.sum +++ b/go.sum @@ -131,12 +131,12 @@ github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZ github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= -github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/pom v0.1.9 h1:gP7nqAS+UXITeRXjHhpyrMHRl9K3sRRBzRLV3+TeqHU= +github.com/git-pkgs/pom v0.1.9/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.13.2 h1:3fWinubUZbBEJ+kDii3O5kzqHJMJBmJK1X9FefAow+I= -github.com/git-pkgs/registries v0.13.2/go.mod h1:UlX6jcAd4ftRBVVNzB4mQg32tfSNCk/L2pO4Wtplnho= +github.com/git-pkgs/registries v0.14.0 h1:PQbfRFghM6cN2xu1FcDHS2F73yOMATVIBaq+zY/6uSI= +github.com/git-pkgs/registries v0.14.0/go.mod h1:zgtz1lneDi8olIzAJxnSvSMT7PzChp9dsbFxJVbfyfQ= github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.6 h1:tbkqeYLKgAcDV1qPZNAkNxpNMTx9EfoNQQhcVEHApzs= diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 25983cdc..3e7ce4c2 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -81,12 +81,14 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { suffix: ".gem", parseErr: "could not parse gem filename", fetchErr: "failed to fetch gem", - parse: h.parseGemFilename, + parse: GemFilenameIdentity, publishTime: h.publishTime, }) } -// parseGemFilename extracts name and version from a gem filename. +// GemFilenameIdentity returns the name and version a gem file is cached +// under, read from its filename. A platform build's version keeps its +// platform. // e.g., "rails-7.1.0.gem" -> ("rails", "7.1.0") // e.g., "aws-sdk-s3-1.142.0.gem" -> ("aws-sdk-s3", "1.142.0") // e.g., "sorbet-static-0.5.11002-universal-darwin-22.gem" -> ("sorbet-static", "0.5.11002-universal-darwin-22") @@ -103,7 +105,7 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // // Trying every "-" costs time quadratic in the filename's length, so // a filename longer than maxGemFilenameLength is refused. -func (h *GemHandler) parseGemFilename(filename string) (name, version string) { +func GemFilenameIdentity(filename string) (name, version string) { if len(filename) > maxGemFilenameLength { return "", "" } @@ -126,7 +128,7 @@ func (h *GemHandler) parseGemFilename(filename string) (name, version string) { return base[:first], base[first+1:] } -// maxGemFilenameLength bounds the gem filenames parseGemFilename accepts. +// maxGemFilenameLength bounds the gem filenames GemFilenameIdentity accepts. // Real gem filenames are a small fraction of this. const maxGemFilenameLength = 512 diff --git a/internal/handler/gem_test.go b/internal/handler/gem_test.go index ba62ef8b..3812094e 100644 --- a/internal/handler/gem_test.go +++ b/internal/handler/gem_test.go @@ -4,7 +4,6 @@ import ( "compress/gzip" "encoding/json" "fmt" - "log/slog" "net/http" "net/http/httptest" "strings" @@ -15,8 +14,6 @@ import ( ) func TestGemParseFilename(t *testing.T) { - h := &GemHandler{proxy: &Proxy{Logger: slog.Default()}} - tests := []struct { filename string wantName string @@ -42,9 +39,9 @@ func TestGemParseFilename(t *testing.T) { } for _, tt := range tests { - name, version := h.parseGemFilename(tt.filename) + name, version := GemFilenameIdentity(tt.filename) if name != tt.wantName || version != tt.wantVersion { - t.Errorf("parseGemFilename(%q) = (%q, %q), want (%q, %q)", + t.Errorf("GemFilenameIdentity(%q) = (%q, %q), want (%q, %q)", tt.filename, name, version, tt.wantName, tt.wantVersion) } } diff --git a/internal/mirror/files.go b/internal/mirror/files.go index a4109765..1f1ca12a 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -2,6 +2,7 @@ package mirror import ( "fmt" + "maps" "path" "regexp" "slices" @@ -31,9 +32,56 @@ type fileEcosystem struct { // cachedVariants names them for the filename cacheKey returns. variants func(file registries.Artifact) []string cachedVariants func(cacheFilename string) []string + // split, when set, moves a file selection the version carries into the + // qualifiers, returning the version's release number. + split func(pv PackageVersion) (PackageVersion, error) + // release, when set, returns the number of the release whose files are + // cached under cachedVersion, for an ecosystem where the two differ. + release func(cachedVersion string) string } var fileEcosystems = map[string]fileEcosystem{ + // Each platform build of a gem is an artifact of its version. The gem + // handler caches a platform build under its version with the platform + // appended, as it reads it from the filename. + "gem": { + qualifiers: []string{"platform"}, + attrs: func(file registries.Artifact) map[string]string { + return map[string]string{"platform": gemArtifactPlatform(file)} + }, + cacheKey: func(_ PackageVersion, file registries.Artifact) (string, string, string, bool) { + name, version := handler.GemFilenameIdentity(file.Filename) + return name, version, file.Filename, name != "" + }, + variants: func(file registries.Artifact) []string { return []string{gemArtifactPlatform(file)} }, + cachedVariants: func(cacheFilename string) []string { + _, version := handler.GemFilenameIdentity(cacheFilename) + _, platform := gemVersionPlatform(version) + return []string{platform} + }, + // Gemfile.lock gives a platform build's version with its platform + // appended, and so do SBOMs made from one. + split: func(pv PackageVersion) (PackageVersion, error) { + number, platform, ok := strings.Cut(pv.Version, "-") + if !ok { + return pv, nil + } + if q := pv.Qualifiers["platform"]; q != "" && q != platform { + return pv, fmt.Errorf("version %s is for platform %s, not platform=%s", pv.Version, platform, q) + } + qualifiers := maps.Clone(pv.Qualifiers) + if qualifiers == nil { + qualifiers = map[string]string{} + } + qualifiers["platform"] = platform + pv.Version, pv.Qualifiers = number, qualifiers + return pv, nil + }, + release: func(cachedVersion string) string { + number, _ := gemVersionPlatform(cachedVersion) + return number + }, + }, "pypi": { qualifiers: []string{"file_name"}, attrs: func(file registries.Artifact) map[string]string { @@ -82,6 +130,28 @@ var fileEcosystems = map[string]fileEcosystem{ }, } +// gemRubyPlatform is the platform of a gem built for none, which installs +// on all of them. +const gemRubyPlatform = "ruby" + +// gemArtifactPlatform returns the platform of a gem build, which the +// RubyGems adapter gives as "ruby" for a build for none. +func gemArtifactPlatform(file registries.Artifact) string { + platform, _ := file.Metadata["platform"].(string) + return platform +} + +// gemVersionPlatform splits a version a gem file is cached under into the +// release's number and the build's platform. A gem version has no "-", as +// RubyGems writes one as ".pre.", so the platform follows the first. +func gemVersionPlatform(version string) (number, platform string) { + number, platform, ok := strings.Cut(version, "-") + if !ok { + platform = gemRubyPlatform + } + return number, platform +} + // pypiVariants names a wheel by each combination of its python, ABI and // platform tags, which are alternatives where a tag lists several joined by // dots, with the platform reduced to OS and architecture. Every sdist is diff --git a/internal/mirror/gem_test.go b/internal/mirror/gem_test.go new file mode 100644 index 00000000..04a4c874 --- /dev/null +++ b/internal/mirror/gem_test.go @@ -0,0 +1,226 @@ +package mirror + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/registries" +) + +// gemBuilds returns a version of nokogiri whose artifacts are the given +// platform builds, as the RubyGems adapter lists them: one per platform, with +// "ruby" for the build that installs anywhere. +func gemBuilds(number string, age time.Duration, platforms ...string) registries.Version { + const name = "nokogiri" + v := registries.Version{Number: number, PublishedAt: time.Now().Add(-age)} + for _, platform := range platforms { + file := name + "-" + number + "-" + platform + ".gem" + if platform == "ruby" { + file = name + "-" + number + ".gem" + } + v.Artifacts = append(v.Artifacts, registries.Artifact{ + URL: "https://rubygems.example/downloads/" + file, + Filename: file, + Metadata: map[string]any{"platform": platform}, + }) + } + return v +} + +func TestMirrorGemPlatformBuilds(t *testing.T) { + tests := []struct { + purl string + want []string + }{ + {"pkg:gem/nokogiri@1.13.6", []string{"nokogiri-1.13.6-arm64-darwin.gem", "nokogiri-1.13.6-x86_64-linux.gem", "nokogiri-1.13.6.gem"}}, + {"pkg:gem/nokogiri@1.13.6?platform=x86_64-linux", []string{"nokogiri-1.13.6-x86_64-linux.gem"}}, + {"pkg:gem/nokogiri@1.13.6?platform=ruby", []string{"nokogiri-1.13.6.gem"}}, + // Gemfile.lock, and SBOMs made from one, give a platform build's + // version with the platform appended. + {"pkg:gem/nokogiri@1.13.6-x86_64-linux", []string{"nokogiri-1.13.6-x86_64-linux.gem"}}, + {"pkg:gem/nokogiri@1.13.6-x86_64-linux?platform=x86_64-linux", []string{"nokogiri-1.13.6-x86_64-linux.gem"}}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 30*24*time.Hour, "ruby", "x86_64-linux", "arm64-darwin"), + }} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{tt.purl}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Completed != 1 || progress.Failed != 0 { + t.Errorf("progress = %+v, want one version mirrored", progress) + } + got := fetcher.fetched() + slices.Sort(got) + want := make([]string, len(tt.want)) + for i, f := range tt.want { + want[i] = "https://rubygems.example/downloads/" + f + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } + + // Each build is cached where the gem handler looks for it: a + // platform build under its version with the platform appended. + for _, f := range tt.want { + version := strings.TrimSuffix(strings.TrimPrefix(f, "nokogiri-"), ".gem") + a, err := m.db.GetArtifact("pkg:gem/nokogiri@"+version, f) + if err != nil || a == nil { + t.Errorf("%s not cached under version %s: %v", f, version, err) + } + } + }) + } +} + +func TestMirrorGemPlatformMatchingNoBuild(t *testing.T) { + m, _ := setupRefreshMirror(t, "gem") + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 30*24*time.Hour, "ruby", "x86_64-linux"), + }} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:gem/nokogiri@1.13.6?platform=java"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "platform=java") { + t.Errorf("progress = %+v, want a failure naming platform=java", progress) + } +} + +// A platform in the version that has no build fails under the version as +// given. +func TestMirrorGemVersionPlatformMatchingNoBuild(t *testing.T) { + m, _ := setupRefreshMirror(t, "gem") + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 30*24*time.Hour, "ruby", "x86_64-linux"), + }} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:gem/nokogiri@1.13.6-java"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "platform=java") || + progress.Errors[0].Version != "1.13.6-java" { + t.Errorf("progress = %+v, want a failure of version 1.13.6-java naming platform=java", progress) + } +} + +// A platform in the version and a different one in the qualifier name no +// build, so the PURL is refused. +func TestMirrorGemPlatformConflictsWithVersion(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 30*24*time.Hour, "ruby", "x86_64-linux", "arm64-darwin"), + }} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:gem/nokogiri@1.13.6-x86_64-linux?platform=arm64-darwin"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "platform=arm64-darwin") || + len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want a failure naming platform=arm64-darwin and nothing fetched", progress, fetcher.fetched()) + } +} + +// A file whose name doesn't read as a gem's is not cached, as the gem +// handler could not serve it. +func TestMirrorGemSkipsUnreadableFilename(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + v := gemBuilds("1.13.6", 30*24*time.Hour, "ruby") + v.Artifacts[0].Filename = "nokogiri.gem" + m.Versions = stubReleases{"gem/nokogiri": {v}} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:gem/nokogiri@1.13.6"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want a failure and nothing fetched", progress, fetcher.fetched()) + } +} + +// A refresh mirrors the platforms clients fetched of an older version, and +// compares the versions without their platforms. +func TestRefreshMirrorsTheGemPlatformsClientsFetched(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + ctx := context.Background() + // The gem handler caches a platform build under its version with the + // platform appended, as it reads it from the filename. + for _, f := range [][2]string{ + {"1.13.6", "nokogiri-1.13.6.gem"}, + {"1.13.6-x86_64-linux", "nokogiri-1.13.6-x86_64-linux.gem"}, + } { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "gem", "nokogiri", f[0], f[1], "https://rubygems.example/downloads/"+f[1], "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 60*24*time.Hour, "ruby", "x86_64-linux", "arm64-darwin"), + gemBuilds("1.13.7", 30*24*time.Hour, "ruby", "x86_64-linux", "arm64-darwin", "java"), + }} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + got := fetcher.fetched() + slices.Sort(got) + want := []string{ + "https://rubygems.example/downloads/nokogiri-1.13.7-x86_64-linux.gem", + "https://rubygems.example/downloads/nokogiri-1.13.7.gem", + } + if progress.Completed != 1 || progress.Failed != 0 || !slices.Equal(got, want) { + t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want) + } + + // Once 1.13.7 is cached, a refresh has nothing left to fetch. + fetcher.reset() + progress, err = m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetcher.fetched(); len(got) != 0 || progress.Completed+progress.Skipped != 0 { + t.Errorf("second refresh: progress = %+v, fetched %v; want nothing to do", progress, got) + } +} + +// With only platform builds cached, a refresh still reads their version as +// the release's number: 1.13.7 follows 1.13.6-java, and once it is cached +// there is nothing left to fetch. +func TestRefreshComparesGemVersionsWithoutPlatform(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "gem") + ctx := context.Background() + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "gem", "nokogiri", "1.13.6-java", "nokogiri-1.13.6-java.gem", "https://rubygems.example/downloads/nokogiri-1.13.6-java.gem", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + m.Versions = stubReleases{"gem/nokogiri": { + gemBuilds("1.13.6", 60*24*time.Hour, "ruby", "java"), + gemBuilds("1.13.7", 30*24*time.Hour, "ruby", "java"), + }} + + for i, want := range [][]string{{"https://rubygems.example/downloads/nokogiri-1.13.7-java.gem"}, nil} { + fetcher.reset() + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + got := fetcher.fetched() + if progress.Failed != 0 || progress.Skipped != 0 || !slices.Equal(got, want) { + t.Errorf("refresh %d: progress = %+v, fetched %v; want %v", i+1, progress, got, want) + } + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 1dded24d..70fc93ab 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -379,7 +379,17 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEco return } } - v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, pv.Name, pv.Version) + // sel is pv with the selection its version carries moved into the + // qualifiers; failures and logs keep pv as it was asked for. + sel := pv + if eco.split != nil { + var err error + if sel, err = eco.split(pv); err != nil { + fail(err) + return + } + } + v, err := m.versionFinder().FindVersion(ctx, sel.Ecosystem, sel.Name, sel.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return @@ -387,10 +397,10 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEco var matched, fetched int var size int64 for _, file := range v.Artifacts { - if !eco.selects(pv, file) { + if !eco.selects(sel, file) { continue } - name, version, filename, ok := eco.cacheKey(pv, file) + name, version, filename, ok := eco.cacheKey(sel, file) if !ok { continue } @@ -408,7 +418,7 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEco size += result.Artifact.Size } } - switch selection := eco.selection(pv); { + switch selection := eco.selection(sel); { case matched == 0 && selection != "": fail(fmt.Errorf("no file matches %s", selection)) case matched == 0: diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 3c2e0942..b1909726 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -142,9 +142,13 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] var cached []string seen := map[string]bool{} for _, f := range pkg.Files { - if !seen[f.Version] { - seen[f.Version] = true - cached = append(cached, f.Version) + version := f.Version + if eco, ok := fileEcosystems[ecosystem]; ok && eco.release != nil { + version = eco.release(version) + } + if !seen[version] { + seen[version] = true + cached = append(cached, version) } } allowed := func(v registries.Version) bool {