diff --git a/docs/configuration.md b/docs/configuration.md index cb70bf5a..b74a64c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -348,6 +348,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.oci_default` | `PROXY_UPSTREAM_OCI_DEFAULT` | `https://registry-1.docker.io` | | `upstream.debian` | `PROXY_UPSTREAM_DEBIAN` | `http://deb.debian.org/debian` | | `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | +| `upstream.rpm_repository_paths` | - | `["updates/{release}/Everything/{arch}", "releases/{release}/Everything/{arch}/os"]` | | `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` | | `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` | | `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) | @@ -2006,6 +2007,8 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An RPM package is mirrored from the repositories below `upstream.rpm` that `upstream.rpm_repository_paths` names, in order, as syft writes it in image SBOMs: `pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=0`. The `distro` qualifier gives the release, which replaces `{release}` (`40`, `9.4` for `rocky-9.4`, or `15.6` for `opensuse-leap-15.6`) and `{major}` (`9`, `15`), and `arch` the architecture, which replaces `{arch}`; packages without either are refused. A `noarch` package is looked up in the `x86_64` repositories. By default the paths are Fedora's on dl.fedoraproject.org, its updates and then its release; another distribution's tree needs its own, such as `["{major}/BaseOS/{arch}/os", "{major}/AppStream/{arch}/os"]` for Rocky Linux under `https://dl.rockylinux.org/pub/rocky`. The package is taken from the first repository whose primary metadata lists it, with the `epoch` qualifier picking among builds that differ only in their epoch, checked against its SHA-256 where the metadata gives one, and cached under its path below `upstream.rpm`, which `/rpm/` serves it from. The metadata isn't mirrored. Give a version: Fedora's update repositories keep only the newest build. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/go.mod b/go.mod index c3409e24..c552f50e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 - github.com/git-pkgs/registries v0.13.2 + github.com/git-pkgs/registries v0.14.0 github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 @@ -75,7 +75,7 @@ require ( github.com/dustin/go-humanize v1.1.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/pom v0.1.9 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.8.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index 4a2270eb..93e42071 100644 --- a/go.sum +++ b/go.sum @@ -131,12 +131,12 @@ github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZ github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= -github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/pom v0.1.9 h1:gP7nqAS+UXITeRXjHhpyrMHRl9K3sRRBzRLV3+TeqHU= +github.com/git-pkgs/pom v0.1.9/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.13.2 h1:3fWinubUZbBEJ+kDii3O5kzqHJMJBmJK1X9FefAow+I= -github.com/git-pkgs/registries v0.13.2/go.mod h1:UlX6jcAd4ftRBVVNzB4mQg32tfSNCk/L2pO4Wtplnho= +github.com/git-pkgs/registries v0.14.0 h1:PQbfRFghM6cN2xu1FcDHS2F73yOMATVIBaq+zY/6uSI= +github.com/git-pkgs/registries v0.14.0/go.mod h1:zgtz1lneDi8olIzAJxnSvSMT7PzChp9dsbFxJVbfyfQ= github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.6 h1:tbkqeYLKgAcDV1qPZNAkNxpNMTx9EfoNQQhcVEHApzs= diff --git a/internal/config/config.go b/internal/config/config.go index e5106a75..62dea7af 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -63,6 +63,7 @@ import ( _ "embed" "encoding/base64" "encoding/json" + "errors" "fmt" "maps" "math" @@ -831,6 +832,13 @@ type UpstreamConfig struct { // Default: https://dl.fedoraproject.org/pub/fedora/linux RPM string `json:"rpm" yaml:"rpm"` + // RPMRepositoryPaths are the repository directories below upstream.rpm + // that proxy mirror looks an RPM package up in, in order, with + // {release}, {major} and {arch} taken from the package's distro and + // arch qualifiers. Default: Fedora's updates/{release}/Everything/{arch} + // and releases/{release}/Everything/{arch}/os. + RPMRepositoryPaths []string `json:"rpm_repository_paths" yaml:"rpm_repository_paths"` + // HomebrewAPI is the upstream Homebrew JSON API URL. // Default: https://formulae.brew.sh/api HomebrewAPI string `json:"homebrew_api" yaml:"homebrew_api"` @@ -1027,6 +1035,39 @@ func (u *UpstreamConfig) Validate() error { ) } } + return validateRPMRepositoryPaths(u.RPMRepositoryPaths) +} + +func validateRPMRepositoryPaths(paths []string) error { + for _, p := range paths { + if err := validateRPMRepositoryPath(p); err != nil { + return fmt.Errorf("invalid upstream.rpm_repository_paths entry %q: %w", p, err) + } + } + return nil +} + +// rpmPathPlaceholder matches the placeholders of an RPM repository path. +var rpmPathPlaceholder = regexp.MustCompile(`\{(release|major|arch)\}`) + +// validateRPMRepositoryPath checks a path below upstream.rpm: relative, +// without dot segments or a query, and with no placeholders but {release}, +// {major} and {arch}. +func validateRPMRepositoryPath(p string) error { + rest := rpmPathPlaceholder.ReplaceAllString(p, "x") + switch { + case p == "": + return errors.New("empty") + case strings.HasPrefix(p, "/") || strings.Contains(p, "://"): + return errors.New("not a path relative to upstream.rpm") + case strings.ContainsAny(rest, "{}?#\\"): + return errors.New("only {release}, {major} and {arch} can be replaced, and it can't have a query") + } + for _, segment := range strings.Split(p, "/") { + if segment == "." || segment == ".." { + return errors.New("dot segments aren't allowed") + } + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6ec28ae3..48ce5d31 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1975,6 +1975,34 @@ func TestGenericUpstreamNamedHosted(t *testing.T) { } } +func TestValidateRPMRepositoryPaths(t *testing.T) { + tests := []struct { + name string + paths []string + wantErr bool + }{ + {"Fedora", []string{"updates/{release}/Everything/{arch}", "releases/{release}/Everything/{arch}/os"}, false}, + {"major release", []string{"{major}/BaseOS/{arch}/os"}, false}, + {"empty path", []string{""}, true}, + {"absolute path", []string{"/updates/{release}"}, true}, + {"URL", []string{"https://example.com/{release}"}, true}, + {"dot dot segment", []string{"updates/../{release}"}, true}, + {"dot segment", []string{"./{release}"}, true}, + {"unknown placeholder", []string{"{version}/{arch}"}, true}, + {"unclosed placeholder", []string{"{release/{arch}"}, true}, + {"query", []string{"{release}?x=1"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Upstream.RPMRepositoryPaths = tt.paths + if err := cfg.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func TestValidateFreeBSDUpstreams(t *testing.T) { tests := []struct { name string diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 7363cca1..3a177408 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -52,10 +52,16 @@ func NewRPMHandler(proxy *Proxy, proxyURL string) *RPMHandler { } } +// RPMUpstreamURL returns upstream.rpm, the default Fedora tree when none is +// configured. Package files are cached under their path below it. +func RPMUpstreamURL(configured string) string { + return configuredUpstreamURL(configured, defaultRPMUpstream) +} + // NewRPMHandlerWithUpstream creates an RPM handler with a custom upstream. func NewRPMHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *RPMHandler { h := NewRPMHandler(proxy, proxyURL) - h.upstreamURL = configuredUpstreamURL(upstreamURL, defaultRPMUpstream) + h.upstreamURL = RPMUpstreamURL(upstreamURL) return h } diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 1dded24d..dc23f207 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -53,12 +53,18 @@ type Mirror struct { // the official Alpine mirror as "alpine". APKRepositories map[string]string + // RPMUpstream is upstream.rpm, and RPMRepositoryPaths are + // upstream.rpm_repository_paths. Empty ones use Fedora's tree. + RPMUpstream string + RPMRepositoryPaths []string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex apk apkIndexes + rpm rpmIndexes } // New creates a new Mirror with the given dependencies. @@ -297,6 +303,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "rpm" { + m.mirrorRPM(ctx, pv, tracker) + return + } if pv.Ecosystem == "git" { m.mirrorGit(ctx, pv, tracker) return diff --git a/internal/mirror/rpm.go b/internal/mirror/rpm.go new file mode 100644 index 00000000..d36ff587 --- /dev/null +++ b/internal/mirror/rpm.go @@ -0,0 +1,236 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "maps" + "net/url" + "path" + "regexp" + "slices" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// rpmIndexTTL is how long a repository's primary metadata is reused, so +// mirroring the packages of an image reads it once. +const rpmIndexTTL = 5 * time.Minute + +var ( + // rpmDistroPattern takes the release from the distro qualifier syft + // writes from os-release's ID and VERSION_ID, such as fedora-40, + // rocky-9.4 or opensuse-leap-15.6. + rpmDistroPattern = regexp.MustCompile(`^[a-z][a-z0-9-]*-(\d+(?:\.\d+)*)$`) + rpmArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) + rpmEpochPattern = regexp.MustCompile(`^\d+$`) + + // rpmDefaultRepositoryPaths are Fedora's repositories of a release on + // dl.fedoraproject.org, the default upstream.rpm: its updates, then the + // release. + rpmDefaultRepositoryPaths = []string{ + "updates/{release}/Everything/{arch}", + "releases/{release}/Everything/{arch}/os", + } +) + +// rpmIndexes keeps one registries client per repository, which holds the +// parsed primary metadata. +type rpmIndexes struct { + mu sync.Mutex + regs map[string]rpmIndex +} + +type rpmIndex struct { + // reg is nil for a repository upstream answered 404 for. + reg registries.Registry + fetched time.Time +} + +// mirrorRPM mirrors an RPM package under the name the RPM handler caches it +// at, its path below upstream.rpm. The distro qualifier gives the release +// and arch the architecture, and the package is looked up in the +// repositories upstream.rpm_repository_paths names for them, Fedora's by +// default. +func (m *Mirror) mirrorRPM(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + release, major, ok := rpmRelease(pv.Qualifiers["distro"]) + if !ok { + fail(fmt.Errorf("rpm packages need a distro qualifier with the release, such as distro=fedora-40, not %q", pv.Qualifiers["distro"])) + return + } + arch := pv.Qualifiers["arch"] + if !rpmArchPattern.MatchString(arch) { + fail(fmt.Errorf("rpm packages need an arch qualifier, such as arch=x86_64, not %q", arch)) + return + } + epoch := pv.Qualifiers["epoch"] + if epoch != "" && !rpmEpochPattern.MatchString(epoch) { + fail(fmt.Errorf("invalid epoch %q", epoch)) + return + } + // noarch packages are in every architecture's repository. + repoArch := arch + if arch == "noarch" { + repoArch = "x86_64" + } + replacer := strings.NewReplacer("{release}", release, "{major}", major, "{arch}", repoArch) + paths := m.RPMRepositoryPaths + if len(paths) == 0 { + paths = rpmDefaultRepositoryPaths + } + upstreamURL := handler.RPMUpstreamURL(m.RPMUpstream) + + dirs := make([]string, len(paths)) + for i, p := range paths { + dirs[i] = path.Clean(replacer.Replace(p)) + } + artifact, err := m.rpmArtifact(ctx, upstreamURL, dirs, pv.Qualifiers["distro"], arch, pv.Name, pv.Version, epoch) + if err != nil { + fail(err) + return + } + filePath := path.Clean(strings.TrimPrefix(artifact.URL, upstreamURL+"/")) + name, version, _ := handler.OSPackageIdentity("rpm", filePath) + if name == "" { + fail(fmt.Errorf("%s isn't a package file the RPM handler serves", filePath)) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filePath, artifact.URL, fileDigest(artifact.Integrity)) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// rpmRelease returns the release a distro qualifier names, and its major +// version. +func rpmRelease(distro string) (release, major string, ok bool) { + match := rpmDistroPattern.FindStringSubmatch(distro) + if match == nil { + return "", "", false + } + major, _, _ = strings.Cut(match[1], ".") + return match[1], major, true +} + +// rpmArtifact returns the file of name for arch whose version-release is +// version, from the first of the repositories dirs below upstreamURL that +// lists it. epoch, when given, picks the epoch; +// without it, builds that differ only in their epoch are refused rather +// than one picked. +func (m *Mirror) rpmArtifact(ctx context.Context, upstreamURL string, dirs []string, distro, arch, name, version, epoch string) (registries.Artifact, error) { + for _, dir := range dirs { + repoURL := upstreamURL + "/" + dir + reg, err := m.rpmRegistry(repoURL, distro) + if err != nil { + return registries.Artifact{}, err + } + if reg == nil { + continue + } + versions, err := reg.FetchVersions(ctx, name) + var httpErr *registries.HTTPError + if errors.As(err, &httpErr) && httpErr.IsNotFound() { + // The repository isn't there, which is kept for rpmIndexTTL + // rather than asked again for each package. + m.rpm.missing(repoURL) + continue + } + if isNotFound(err) { + continue + } + if err != nil { + m.rpm.forget(repoURL) + return registries.Artifact{}, fmt.Errorf("reading %s/repodata: %w", repoURL, err) + } + found, epochs := rpmMatch(versions, arch, version, epoch) + switch { + case len(epochs) > 1: + return registries.Artifact{}, fmt.Errorf("%s %s has epochs %s in %s; give one in the epoch qualifier", + name, version, strings.Join(epochs, " and "), repoURL) + case len(epochs) == 1: + return found, nil + } + } + return registries.Artifact{}, fmt.Errorf("%s %s isn't in %s below %s for %s", name, version, strings.Join(dirs, " or "), upstreamURL, arch) +} + +// rpmMatch returns a file of versions for arch whose version-release is +// version, and the epochs of those files, of epoch alone when it is given. +func rpmMatch(versions []registries.Version, arch, version, epoch string) (registries.Artifact, []string) { + var found registries.Artifact + var epochs []string + for _, v := range versions { + if v.Number != version { + continue + } + for _, a := range v.Artifacts { + e := rpmEpoch(a) + if (a.Metadata["arch"] == arch || a.Metadata["arch"] == "noarch") && (epoch == "" || e == epoch) { + if !slices.Contains(epochs, e) { + epochs = append(epochs, e) + } + found = a + } + } + } + return found, epochs +} + +// rpmEpoch returns the epoch of a, 0 when it records none. +func rpmEpoch(a registries.Artifact) string { + if e, _ := a.Metadata["epoch"].(string); e != "" { + return e + } + return "0" +} + +// rpmRegistry returns the client for the repository at repoURL, reusing it, +// and the metadata it read, for rpmIndexTTL. It returns nil for a +// repository recorded as missing. +func (m *Mirror) rpmRegistry(repoURL, distro string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface + m.rpm.mu.Lock() + defer m.rpm.mu.Unlock() + if idx, ok := m.rpm.regs[repoURL]; ok && time.Since(idx.fetched) < rpmIndexTTL { + return idx.reg, nil + } + r, _ := m.Versions.(RegistryVersions) + reg, err := registries.New("rpm", repoURL+"?"+url.Values{"distro": {distro}}.Encode(), r.client()) + if err != nil { + return nil, err + } + m.rpm.store(repoURL, reg) + return reg, nil +} + +// store keeps reg for repoURL, dropping the clients past rpmIndexTTL, so +// indexes no longer asked for aren't kept for the Mirror's life. The +// caller holds r.mu. +func (r *rpmIndexes) store(repoURL string, reg registries.Registry) { + if r.regs == nil { + r.regs = map[string]rpmIndex{} + } + maps.DeleteFunc(r.regs, func(_ string, idx rpmIndex) bool { return time.Since(idx.fetched) >= rpmIndexTTL }) + r.regs[repoURL] = rpmIndex{reg: reg, fetched: time.Now()} +} + +// missing records that the repository at repoURL isn't there. +func (r *rpmIndexes) missing(repoURL string) { + r.mu.Lock() + defer r.mu.Unlock() + r.store(repoURL, nil) +} + +// forget drops the client for repoURL, so metadata that failed to load is +// read again next time. +func (r *rpmIndexes) forget(repoURL string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.regs, repoURL) +} diff --git a/internal/mirror/rpm_test.go b/internal/mirror/rpm_test.go new file mode 100644 index 00000000..335f6ee2 --- /dev/null +++ b/internal/mirror/rpm_test.go @@ -0,0 +1,55 @@ +package mirror + +import ( + "testing" + "time" +) + +// Storing a repository's client drops those past rpmIndexTTL, so indexes no +// longer asked for aren't kept for the Mirror's life. +func TestRPMRegistryDropsExpiredIndexes(t *testing.T) { + m := &Mirror{} + for _, repo := range []string{"https://old.example/os", "https://fresh.example/os"} { + if _, err := m.rpmRegistry(repo, "fedora-40"); err != nil { + t.Fatal(err) + } + } + idx := m.rpm.regs["https://old.example/os"] + idx.fetched = time.Now().Add(-rpmIndexTTL) + m.rpm.regs["https://old.example/os"] = idx + + if _, err := m.rpmRegistry("https://new.example/os", "fedora-40"); err != nil { + t.Fatal(err) + } + if _, ok := m.rpm.regs["https://old.example/os"]; ok { + t.Error("expired index kept") + } + if _, ok := m.rpm.regs["https://fresh.example/os"]; !ok { + t.Error("index within its TTL dropped") + } +} + +// The release comes from the end of the distro qualifier syft writes from +// os-release, whose ID may contain hyphens. +func TestRPMRelease(t *testing.T) { + tests := []struct{ distro, release, major string }{ + {"fedora-40", "40", "40"}, + {"rocky-9.4", "9.4", "9"}, + {"opensuse-leap-15.6", "15.6", "15"}, + {"rhel-8.10", "8.10", "8"}, + {"fedora", "", ""}, + {"fedora-", "", ""}, + {"-40", "", ""}, + {"Fedora-40", "", ""}, + {"fedora-40/../x", "", ""}, + {"fedora-40.x", "", ""}, + } + for _, tt := range tests { + t.Run(tt.distro, func(t *testing.T) { + release, major, ok := rpmRelease(tt.distro) + if release != tt.release || major != tt.major || ok != (tt.release != "") { + t.Errorf("rpmRelease = %q, %q, %v; want %q, %q", release, major, ok, tt.release, tt.major) + } + }) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index bbfddf0b..52a2846f 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -151,6 +151,8 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} case "alpine": qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} + case "rpm": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro"), "epoch": p.Qualifier("epoch")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 3e2cebb6..56afad63 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -65,6 +65,8 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia m.APKRepositories = cfg.Upstream.APK + m.RPMUpstream = cfg.Upstream.RPM + m.RPMRepositoryPaths = cfg.Upstream.RPMRepositoryPaths if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_rpm_test.go b/internal/server/mirror_rpm_test.go new file mode 100644 index 00000000..115cac12 --- /dev/null +++ b/internal/server/mirror_rpm_test.go @@ -0,0 +1,246 @@ +package server + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +// rpmPackage is one package of a primary file, with the file it locates. +type rpmPackage struct { + name, epoch, ver, rel, arch, body string + // file overrides the file name built from the name and version. + file string +} + +func (p rpmPackage) filename() string { + if p.file != "" { + return p.file + } + return fmt.Sprintf("%s-%s-%s.%s.rpm", p.name, p.ver, p.rel, p.arch) +} + +// rpmUpstream serves RPM repositories below its root and counts requests +// by path. +type rpmUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newRPMUpstream(t *testing.T) *rpmUpstream { + t.Helper() + u := &rpmUpstream{files: map[string]string{}, hits: map[string]int{}} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *rpmUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func rpmSHA256(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// addRepository serves the repository at dir, below the root, with pkgs. +func (u *rpmUpstream) addRepository(t *testing.T, dir string, pkgs ...rpmPackage) { + t.Helper() + var primary strings.Builder + primary.WriteString(``) + for _, p := range pkgs { + href := "Packages/" + p.name[:1] + "/" + p.filename() + fmt.Fprintf(&primary, `%s%s`+ + `%s`, + p.name, p.arch, p.epoch, p.ver, p.rel, rpmSHA256(p.body), len(p.body), href) + u.files["/"+dir+"/"+href] = p.body + } + primary.WriteString(``) + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = io.WriteString(zw, primary.String()) + _ = zw.Close() + compressed := gz.String() + u.files["/"+dir+"/repodata/primary.xml.gz"] = compressed + u.files["/"+dir+"/repodata/repomd.xml"] = fmt.Sprintf(``+ + `%s%d`, + rpmSHA256(compressed), len(compressed)) +} + +const ( + rpmMirrorCurlPath = "/updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" + rpmMirrorBashPath = "/releases/40/Everything/x86_64/os/Packages/b/bash-5.2.26-3.fc40.x86_64.rpm" + rpmMirrorTzdataPath = "/releases/40/Everything/x86_64/os/Packages/t/tzdata-2024a-5.fc40.noarch.rpm" +) + +// newFedoraTree serves Fedora 40's release and updates repositories for +// x86_64, laid out as on dl.fedoraproject.org. +func newFedoraTree(t *testing.T) *rpmUpstream { + t.Helper() + u := newRPMUpstream(t) + u.addRepository(t, "releases/40/Everything/x86_64/os", + rpmPackage{"curl", "0", "8.6.0", "7.fc40", "x86_64", "curl old", ""}, + rpmPackage{"bash", "0", "5.2.26", "3.fc40", "x86_64", "bash", ""}, + rpmPackage{"tzdata", "0", "2024a", "5.fc40", "noarch", "tzdata", ""}, + rpmPackage{"dual", "1", "2.0", "1.fc40", "x86_64", "dual one", ""}, + rpmPackage{"odd", "0", "1.0", "1.fc40", "x86_64", "odd", "odd.rpm"}, + ) + u.addRepository(t, "updates/40/Everything/x86_64", + rpmPackage{"curl", "0", "8.6.0", "10.fc40", "x86_64", "curl new", ""}, + rpmPackage{"dual", "2", "2.0", "1.fc40", "x86_64", "dual two", ""}, + rpmPackage{"dual", "3", "2.0", "1.fc40", "x86_64", "dual three", ""}, + ) + return u +} + +func rpmMirrorConfig(upstream *rpmUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.RPM = upstream.URL + return cfg +} + +func getRPM(proxy *handler.Proxy, upstream *rpmUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewRPMHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestMirrorRPMPackagesAreServedFromCache(t *testing.T) { + upstream := newFedoraTree(t) + + progress, proxy := mirrorWithConfig(t, rpmMirrorConfig(upstream), + // syft writes the epoch as a qualifier, 0 included. + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40", + "pkg:rpm/fedora/bash@5.2.26-3.fc40?arch=x86_64&distro=fedora-40&epoch=0", + "pkg:rpm/fedora/tzdata@2024a-5.fc40?arch=noarch&distro=fedora-40", + // A noarch package asked for by the machine's architecture. + "pkg:rpm/fedora/tzdata@2024a-5.fc40?arch=x86_64&distro=fedora-40") + + if progress.Completed+progress.Skipped != 4 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{rpmMirrorCurlPath, rpmMirrorBashPath, rpmMirrorTzdataPath, + "/updates/40/Everything/x86_64/repodata/primary.xml.gz", "/releases/40/Everything/x86_64/os/repodata/primary.xml.gz"} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // dnf gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + rpmMirrorCurlPath: "curl new", + rpmMirrorBashPath: "bash", + rpmMirrorTzdataPath: "tzdata", + } { + if rec := getRPM(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +// upstream.rpm_repository_paths lays out another distribution's tree, with +// the major release in the path. +func TestMirrorRPMFollowsConfiguredRepositoryPaths(t *testing.T) { + upstream := newRPMUpstream(t) + upstream.addRepository(t, "9/BaseOS/x86_64/os", rpmPackage{"bash", "0", "5.1.8", "9.el9", "x86_64", "rocky bash", ""}) + cfg := rpmMirrorConfig(upstream) + cfg.Upstream.RPMRepositoryPaths = []string{"{major}/AppStream/{arch}/os", "{major}/BaseOS/{arch}/os"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:rpm/rocky/bash@5.1.8-9.el9?arch=x86_64&distro=rocky-9.4") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm"); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +// An epoch picks one of the builds that differ only in it, from the first +// repository that has it. +func TestMirrorRPMEpochSelectsTheBuild(t *testing.T) { + upstream := newFedoraTree(t) + + progress, _ := mirrorWithConfig(t, rpmMirrorConfig(upstream), + "pkg:rpm/fedora/dual@2.0-1.fc40?arch=x86_64&distro=fedora-40&epoch=1") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/releases/40/Everything/x86_64/os/Packages/d/dual-2.0-1.fc40.x86_64.rpm"); got != 1 { + t.Errorf("epoch 1 build fetched %d times, want 1", got) + } +} + +func TestMirrorRPMRefusesWhatItCantFind(t *testing.T) { + upstream := newFedoraTree(t) + + tests := map[string]string{ + "pkg:rpm/fedora/curl@8.6.0-10.fc40?distro=fedora-40": "arch", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=../x&distro=fedora-40": "arch", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64": "distro", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora": "distro", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40/../x": "distro", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=x": "epoch", + "pkg:rpm/fedora/curl@9.9.9-1.fc40?arch=x86_64&distro=fedora-40": "isn't in", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=aarch64&distro=fedora-40": "isn't in", + "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=3": "isn't in", + "pkg:rpm/fedora/dual@2.0-1.fc40?arch=x86_64&distro=fedora-40": "epoch", + "pkg:rpm/fedora/odd@1.0-1.fc40?arch=x86_64&distro=fedora-40": "isn't a package file", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, rpmMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} + +// A repository that isn't there is asked for once, not once per package. +func TestMirrorRPMRemembersAMissingRepository(t *testing.T) { + upstream := newRPMUpstream(t) + upstream.addRepository(t, "9/BaseOS/x86_64/os", + rpmPackage{"bash", "0", "5.1.8", "9.el9", "x86_64", "rocky bash", ""}, + rpmPackage{"curl", "0", "7.76.1", "29.el9", "x86_64", "rocky curl", ""}, + ) + cfg := rpmMirrorConfig(upstream) + cfg.Upstream.RPMRepositoryPaths = []string{"{major}/AppStream/{arch}/os", "{major}/BaseOS/{arch}/os"} + + progress, _ := mirrorWithConfig(t, cfg, + "pkg:rpm/rocky/bash@5.1.8-9.el9?arch=x86_64&distro=rocky-9.4", + "pkg:rpm/rocky/curl@7.76.1-29.el9?arch=x86_64&distro=rocky-9.4") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/9/AppStream/x86_64/os/repodata/repomd.xml"); got != 1 { + t.Errorf("missing repository asked for %d times, want once", got) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 81491d95..73f580f5 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -41,11 +41,14 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { // UseMirrorUpstreams points m at the configured upstreams: it lists // versions from them, through client, fetches Julia packages from -// upstream.julia and Alpine packages from upstream.apk. +// upstream.julia, Alpine packages from upstream.apk and RPM packages from +// upstream.rpm. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia m.APKRepositories = upstream.APK + m.RPMUpstream = upstream.RPM + m.RPMRepositoryPaths = upstream.RPMRepositoryPaths } // mirrorVersionsURL returns the registry base URL to list a package's