diff --git a/docs/configuration.md b/docs/configuration.md
index cb70bf5a..b74a64c9 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -348,6 +348,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under
| `upstream.oci_default` | `PROXY_UPSTREAM_OCI_DEFAULT` | `https://registry-1.docker.io` |
| `upstream.debian` | `PROXY_UPSTREAM_DEBIAN` | `http://deb.debian.org/debian` |
| `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` |
+| `upstream.rpm_repository_paths` | - | `["updates/{release}/Everything/{arch}", "releases/{release}/Everything/{arch}/os"]` |
| `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` |
| `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` |
| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) |
@@ -2006,6 +2007,8 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp
An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one.
+An RPM package is mirrored from the repositories below `upstream.rpm` that `upstream.rpm_repository_paths` names, in order, as syft writes it in image SBOMs: `pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=0`. The `distro` qualifier gives the release, which replaces `{release}` (`40`, `9.4` for `rocky-9.4`, or `15.6` for `opensuse-leap-15.6`) and `{major}` (`9`, `15`), and `arch` the architecture, which replaces `{arch}`; packages without either are refused. A `noarch` package is looked up in the `x86_64` repositories. By default the paths are Fedora's on dl.fedoraproject.org, its updates and then its release; another distribution's tree needs its own, such as `["{major}/BaseOS/{arch}/os", "{major}/AppStream/{arch}/os"]` for Rocky Linux under `https://dl.rockylinux.org/pub/rocky`. The package is taken from the first repository whose primary metadata lists it, with the `epoch` qualifier picking among builds that differ only in their epoch, checked against its SHA-256 where the metadata gives one, and cached under its path below `upstream.rpm`, which `/rpm/` serves it from. The metadata isn't mirrored. Give a version: Fedora's update repositories keep only the newest build.
+
| Flag | Default | Description |
|------|---------|-------------|
| `--sbom` | | Path to CycloneDX or SPDX SBOM file |
diff --git a/go.mod b/go.mod
index c3409e24..c552f50e 100644
--- a/go.mod
+++ b/go.mod
@@ -18,7 +18,7 @@ require (
github.com/git-pkgs/integrity v0.2.0
github.com/git-pkgs/magic v0.6.0
github.com/git-pkgs/purl v0.2.1
- github.com/git-pkgs/registries v0.13.2
+ github.com/git-pkgs/registries v0.14.0
github.com/git-pkgs/sbom v0.1.9
github.com/git-pkgs/spdx v0.3.6
github.com/git-pkgs/vers v0.7.2
@@ -75,7 +75,7 @@ require (
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
- github.com/git-pkgs/pom v0.1.8 // indirect
+ github.com/git-pkgs/pom v0.1.9 // indirect
github.com/git-pkgs/vulns v0.2.4 // indirect
github.com/github/go-spdx/v2 v2.8.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
diff --git a/go.sum b/go.sum
index 4a2270eb..93e42071 100644
--- a/go.sum
+++ b/go.sum
@@ -131,12 +131,12 @@ github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZ
github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk=
github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ=
github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4=
-github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8=
-github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw=
+github.com/git-pkgs/pom v0.1.9 h1:gP7nqAS+UXITeRXjHhpyrMHRl9K3sRRBzRLV3+TeqHU=
+github.com/git-pkgs/pom v0.1.9/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw=
github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0=
github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw=
-github.com/git-pkgs/registries v0.13.2 h1:3fWinubUZbBEJ+kDii3O5kzqHJMJBmJK1X9FefAow+I=
-github.com/git-pkgs/registries v0.13.2/go.mod h1:UlX6jcAd4ftRBVVNzB4mQg32tfSNCk/L2pO4Wtplnho=
+github.com/git-pkgs/registries v0.14.0 h1:PQbfRFghM6cN2xu1FcDHS2F73yOMATVIBaq+zY/6uSI=
+github.com/git-pkgs/registries v0.14.0/go.mod h1:zgtz1lneDi8olIzAJxnSvSMT7PzChp9dsbFxJVbfyfQ=
github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc=
github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M=
github.com/git-pkgs/spdx v0.3.6 h1:tbkqeYLKgAcDV1qPZNAkNxpNMTx9EfoNQQhcVEHApzs=
diff --git a/internal/config/config.go b/internal/config/config.go
index e5106a75..62dea7af 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -63,6 +63,7 @@ import (
_ "embed"
"encoding/base64"
"encoding/json"
+ "errors"
"fmt"
"maps"
"math"
@@ -831,6 +832,13 @@ type UpstreamConfig struct {
// Default: https://dl.fedoraproject.org/pub/fedora/linux
RPM string `json:"rpm" yaml:"rpm"`
+ // RPMRepositoryPaths are the repository directories below upstream.rpm
+ // that proxy mirror looks an RPM package up in, in order, with
+ // {release}, {major} and {arch} taken from the package's distro and
+ // arch qualifiers. Default: Fedora's updates/{release}/Everything/{arch}
+ // and releases/{release}/Everything/{arch}/os.
+ RPMRepositoryPaths []string `json:"rpm_repository_paths" yaml:"rpm_repository_paths"`
+
// HomebrewAPI is the upstream Homebrew JSON API URL.
// Default: https://formulae.brew.sh/api
HomebrewAPI string `json:"homebrew_api" yaml:"homebrew_api"`
@@ -1027,6 +1035,39 @@ func (u *UpstreamConfig) Validate() error {
)
}
}
+ return validateRPMRepositoryPaths(u.RPMRepositoryPaths)
+}
+
+func validateRPMRepositoryPaths(paths []string) error {
+ for _, p := range paths {
+ if err := validateRPMRepositoryPath(p); err != nil {
+ return fmt.Errorf("invalid upstream.rpm_repository_paths entry %q: %w", p, err)
+ }
+ }
+ return nil
+}
+
+// rpmPathPlaceholder matches the placeholders of an RPM repository path.
+var rpmPathPlaceholder = regexp.MustCompile(`\{(release|major|arch)\}`)
+
+// validateRPMRepositoryPath checks a path below upstream.rpm: relative,
+// without dot segments or a query, and with no placeholders but {release},
+// {major} and {arch}.
+func validateRPMRepositoryPath(p string) error {
+ rest := rpmPathPlaceholder.ReplaceAllString(p, "x")
+ switch {
+ case p == "":
+ return errors.New("empty")
+ case strings.HasPrefix(p, "/") || strings.Contains(p, "://"):
+ return errors.New("not a path relative to upstream.rpm")
+ case strings.ContainsAny(rest, "{}?#\\"):
+ return errors.New("only {release}, {major} and {arch} can be replaced, and it can't have a query")
+ }
+ for _, segment := range strings.Split(p, "/") {
+ if segment == "." || segment == ".." {
+ return errors.New("dot segments aren't allowed")
+ }
+ }
return nil
}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 6ec28ae3..48ce5d31 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -1975,6 +1975,34 @@ func TestGenericUpstreamNamedHosted(t *testing.T) {
}
}
+func TestValidateRPMRepositoryPaths(t *testing.T) {
+ tests := []struct {
+ name string
+ paths []string
+ wantErr bool
+ }{
+ {"Fedora", []string{"updates/{release}/Everything/{arch}", "releases/{release}/Everything/{arch}/os"}, false},
+ {"major release", []string{"{major}/BaseOS/{arch}/os"}, false},
+ {"empty path", []string{""}, true},
+ {"absolute path", []string{"/updates/{release}"}, true},
+ {"URL", []string{"https://example.com/{release}"}, true},
+ {"dot dot segment", []string{"updates/../{release}"}, true},
+ {"dot segment", []string{"./{release}"}, true},
+ {"unknown placeholder", []string{"{version}/{arch}"}, true},
+ {"unclosed placeholder", []string{"{release/{arch}"}, true},
+ {"query", []string{"{release}?x=1"}, true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cfg := Default()
+ cfg.Upstream.RPMRepositoryPaths = tt.paths
+ if err := cfg.Validate(); (err != nil) != tt.wantErr {
+ t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
func TestValidateFreeBSDUpstreams(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go
index 7363cca1..3a177408 100644
--- a/internal/handler/rpm.go
+++ b/internal/handler/rpm.go
@@ -52,10 +52,16 @@ func NewRPMHandler(proxy *Proxy, proxyURL string) *RPMHandler {
}
}
+// RPMUpstreamURL returns upstream.rpm, the default Fedora tree when none is
+// configured. Package files are cached under their path below it.
+func RPMUpstreamURL(configured string) string {
+ return configuredUpstreamURL(configured, defaultRPMUpstream)
+}
+
// NewRPMHandlerWithUpstream creates an RPM handler with a custom upstream.
func NewRPMHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *RPMHandler {
h := NewRPMHandler(proxy, proxyURL)
- h.upstreamURL = configuredUpstreamURL(upstreamURL, defaultRPMUpstream)
+ h.upstreamURL = RPMUpstreamURL(upstreamURL)
return h
}
diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go
index 1dded24d..dc23f207 100644
--- a/internal/mirror/mirror.go
+++ b/internal/mirror/mirror.go
@@ -53,12 +53,18 @@ type Mirror struct {
// the official Alpine mirror as "alpine".
APKRepositories map[string]string
+ // RPMUpstream is upstream.rpm, and RPMRepositoryPaths are
+ // upstream.rpm_repository_paths. Empty ones use Fedora's tree.
+ RPMUpstream string
+ RPMRepositoryPaths []string
+
// RefusePackages, when set, fails every item Images does not take with
// this reason instead of fetching it, so nothing is cached.
RefusePackages string
cran cranIndex
apk apkIndexes
+ rpm rpmIndexes
}
// New creates a new Mirror with the given dependencies.
@@ -297,6 +303,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
m.mirrorAPK(ctx, pv, tracker)
return
}
+ if pv.Ecosystem == "rpm" {
+ m.mirrorRPM(ctx, pv, tracker)
+ return
+ }
if pv.Ecosystem == "git" {
m.mirrorGit(ctx, pv, tracker)
return
diff --git a/internal/mirror/rpm.go b/internal/mirror/rpm.go
new file mode 100644
index 00000000..d36ff587
--- /dev/null
+++ b/internal/mirror/rpm.go
@@ -0,0 +1,236 @@
+package mirror
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "maps"
+ "net/url"
+ "path"
+ "regexp"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/git-pkgs/proxy/internal/handler"
+ "github.com/git-pkgs/registries"
+)
+
+// rpmIndexTTL is how long a repository's primary metadata is reused, so
+// mirroring the packages of an image reads it once.
+const rpmIndexTTL = 5 * time.Minute
+
+var (
+ // rpmDistroPattern takes the release from the distro qualifier syft
+ // writes from os-release's ID and VERSION_ID, such as fedora-40,
+ // rocky-9.4 or opensuse-leap-15.6.
+ rpmDistroPattern = regexp.MustCompile(`^[a-z][a-z0-9-]*-(\d+(?:\.\d+)*)$`)
+ rpmArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`)
+ rpmEpochPattern = regexp.MustCompile(`^\d+$`)
+
+ // rpmDefaultRepositoryPaths are Fedora's repositories of a release on
+ // dl.fedoraproject.org, the default upstream.rpm: its updates, then the
+ // release.
+ rpmDefaultRepositoryPaths = []string{
+ "updates/{release}/Everything/{arch}",
+ "releases/{release}/Everything/{arch}/os",
+ }
+)
+
+// rpmIndexes keeps one registries client per repository, which holds the
+// parsed primary metadata.
+type rpmIndexes struct {
+ mu sync.Mutex
+ regs map[string]rpmIndex
+}
+
+type rpmIndex struct {
+ // reg is nil for a repository upstream answered 404 for.
+ reg registries.Registry
+ fetched time.Time
+}
+
+// mirrorRPM mirrors an RPM package under the name the RPM handler caches it
+// at, its path below upstream.rpm. The distro qualifier gives the release
+// and arch the architecture, and the package is looked up in the
+// repositories upstream.rpm_repository_paths names for them, Fedora's by
+// default.
+func (m *Mirror) mirrorRPM(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
+ fail := func(err error) { m.recordFailure(tracker, pv, err) }
+ release, major, ok := rpmRelease(pv.Qualifiers["distro"])
+ if !ok {
+ fail(fmt.Errorf("rpm packages need a distro qualifier with the release, such as distro=fedora-40, not %q", pv.Qualifiers["distro"]))
+ return
+ }
+ arch := pv.Qualifiers["arch"]
+ if !rpmArchPattern.MatchString(arch) {
+ fail(fmt.Errorf("rpm packages need an arch qualifier, such as arch=x86_64, not %q", arch))
+ return
+ }
+ epoch := pv.Qualifiers["epoch"]
+ if epoch != "" && !rpmEpochPattern.MatchString(epoch) {
+ fail(fmt.Errorf("invalid epoch %q", epoch))
+ return
+ }
+ // noarch packages are in every architecture's repository.
+ repoArch := arch
+ if arch == "noarch" {
+ repoArch = "x86_64"
+ }
+ replacer := strings.NewReplacer("{release}", release, "{major}", major, "{arch}", repoArch)
+ paths := m.RPMRepositoryPaths
+ if len(paths) == 0 {
+ paths = rpmDefaultRepositoryPaths
+ }
+ upstreamURL := handler.RPMUpstreamURL(m.RPMUpstream)
+
+ dirs := make([]string, len(paths))
+ for i, p := range paths {
+ dirs[i] = path.Clean(replacer.Replace(p))
+ }
+ artifact, err := m.rpmArtifact(ctx, upstreamURL, dirs, pv.Qualifiers["distro"], arch, pv.Name, pv.Version, epoch)
+ if err != nil {
+ fail(err)
+ return
+ }
+ filePath := path.Clean(strings.TrimPrefix(artifact.URL, upstreamURL+"/"))
+ name, version, _ := handler.OSPackageIdentity("rpm", filePath)
+ if name == "" {
+ fail(fmt.Errorf("%s isn't a package file the RPM handler serves", filePath))
+ return
+ }
+ result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filePath, artifact.URL, fileDigest(artifact.Integrity))
+ if err != nil {
+ fail(err)
+ return
+ }
+ m.recordResult(tracker, pv, result)
+}
+
+// rpmRelease returns the release a distro qualifier names, and its major
+// version.
+func rpmRelease(distro string) (release, major string, ok bool) {
+ match := rpmDistroPattern.FindStringSubmatch(distro)
+ if match == nil {
+ return "", "", false
+ }
+ major, _, _ = strings.Cut(match[1], ".")
+ return match[1], major, true
+}
+
+// rpmArtifact returns the file of name for arch whose version-release is
+// version, from the first of the repositories dirs below upstreamURL that
+// lists it. epoch, when given, picks the epoch;
+// without it, builds that differ only in their epoch are refused rather
+// than one picked.
+func (m *Mirror) rpmArtifact(ctx context.Context, upstreamURL string, dirs []string, distro, arch, name, version, epoch string) (registries.Artifact, error) {
+ for _, dir := range dirs {
+ repoURL := upstreamURL + "/" + dir
+ reg, err := m.rpmRegistry(repoURL, distro)
+ if err != nil {
+ return registries.Artifact{}, err
+ }
+ if reg == nil {
+ continue
+ }
+ versions, err := reg.FetchVersions(ctx, name)
+ var httpErr *registries.HTTPError
+ if errors.As(err, &httpErr) && httpErr.IsNotFound() {
+ // The repository isn't there, which is kept for rpmIndexTTL
+ // rather than asked again for each package.
+ m.rpm.missing(repoURL)
+ continue
+ }
+ if isNotFound(err) {
+ continue
+ }
+ if err != nil {
+ m.rpm.forget(repoURL)
+ return registries.Artifact{}, fmt.Errorf("reading %s/repodata: %w", repoURL, err)
+ }
+ found, epochs := rpmMatch(versions, arch, version, epoch)
+ switch {
+ case len(epochs) > 1:
+ return registries.Artifact{}, fmt.Errorf("%s %s has epochs %s in %s; give one in the epoch qualifier",
+ name, version, strings.Join(epochs, " and "), repoURL)
+ case len(epochs) == 1:
+ return found, nil
+ }
+ }
+ return registries.Artifact{}, fmt.Errorf("%s %s isn't in %s below %s for %s", name, version, strings.Join(dirs, " or "), upstreamURL, arch)
+}
+
+// rpmMatch returns a file of versions for arch whose version-release is
+// version, and the epochs of those files, of epoch alone when it is given.
+func rpmMatch(versions []registries.Version, arch, version, epoch string) (registries.Artifact, []string) {
+ var found registries.Artifact
+ var epochs []string
+ for _, v := range versions {
+ if v.Number != version {
+ continue
+ }
+ for _, a := range v.Artifacts {
+ e := rpmEpoch(a)
+ if (a.Metadata["arch"] == arch || a.Metadata["arch"] == "noarch") && (epoch == "" || e == epoch) {
+ if !slices.Contains(epochs, e) {
+ epochs = append(epochs, e)
+ }
+ found = a
+ }
+ }
+ }
+ return found, epochs
+}
+
+// rpmEpoch returns the epoch of a, 0 when it records none.
+func rpmEpoch(a registries.Artifact) string {
+ if e, _ := a.Metadata["epoch"].(string); e != "" {
+ return e
+ }
+ return "0"
+}
+
+// rpmRegistry returns the client for the repository at repoURL, reusing it,
+// and the metadata it read, for rpmIndexTTL. It returns nil for a
+// repository recorded as missing.
+func (m *Mirror) rpmRegistry(repoURL, distro string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface
+ m.rpm.mu.Lock()
+ defer m.rpm.mu.Unlock()
+ if idx, ok := m.rpm.regs[repoURL]; ok && time.Since(idx.fetched) < rpmIndexTTL {
+ return idx.reg, nil
+ }
+ r, _ := m.Versions.(RegistryVersions)
+ reg, err := registries.New("rpm", repoURL+"?"+url.Values{"distro": {distro}}.Encode(), r.client())
+ if err != nil {
+ return nil, err
+ }
+ m.rpm.store(repoURL, reg)
+ return reg, nil
+}
+
+// store keeps reg for repoURL, dropping the clients past rpmIndexTTL, so
+// indexes no longer asked for aren't kept for the Mirror's life. The
+// caller holds r.mu.
+func (r *rpmIndexes) store(repoURL string, reg registries.Registry) {
+ if r.regs == nil {
+ r.regs = map[string]rpmIndex{}
+ }
+ maps.DeleteFunc(r.regs, func(_ string, idx rpmIndex) bool { return time.Since(idx.fetched) >= rpmIndexTTL })
+ r.regs[repoURL] = rpmIndex{reg: reg, fetched: time.Now()}
+}
+
+// missing records that the repository at repoURL isn't there.
+func (r *rpmIndexes) missing(repoURL string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.store(repoURL, nil)
+}
+
+// forget drops the client for repoURL, so metadata that failed to load is
+// read again next time.
+func (r *rpmIndexes) forget(repoURL string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ delete(r.regs, repoURL)
+}
diff --git a/internal/mirror/rpm_test.go b/internal/mirror/rpm_test.go
new file mode 100644
index 00000000..335f6ee2
--- /dev/null
+++ b/internal/mirror/rpm_test.go
@@ -0,0 +1,55 @@
+package mirror
+
+import (
+ "testing"
+ "time"
+)
+
+// Storing a repository's client drops those past rpmIndexTTL, so indexes no
+// longer asked for aren't kept for the Mirror's life.
+func TestRPMRegistryDropsExpiredIndexes(t *testing.T) {
+ m := &Mirror{}
+ for _, repo := range []string{"https://old.example/os", "https://fresh.example/os"} {
+ if _, err := m.rpmRegistry(repo, "fedora-40"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ idx := m.rpm.regs["https://old.example/os"]
+ idx.fetched = time.Now().Add(-rpmIndexTTL)
+ m.rpm.regs["https://old.example/os"] = idx
+
+ if _, err := m.rpmRegistry("https://new.example/os", "fedora-40"); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := m.rpm.regs["https://old.example/os"]; ok {
+ t.Error("expired index kept")
+ }
+ if _, ok := m.rpm.regs["https://fresh.example/os"]; !ok {
+ t.Error("index within its TTL dropped")
+ }
+}
+
+// The release comes from the end of the distro qualifier syft writes from
+// os-release, whose ID may contain hyphens.
+func TestRPMRelease(t *testing.T) {
+ tests := []struct{ distro, release, major string }{
+ {"fedora-40", "40", "40"},
+ {"rocky-9.4", "9.4", "9"},
+ {"opensuse-leap-15.6", "15.6", "15"},
+ {"rhel-8.10", "8.10", "8"},
+ {"fedora", "", ""},
+ {"fedora-", "", ""},
+ {"-40", "", ""},
+ {"Fedora-40", "", ""},
+ {"fedora-40/../x", "", ""},
+ {"fedora-40.x", "", ""},
+ }
+ for _, tt := range tests {
+ t.Run(tt.distro, func(t *testing.T) {
+ release, major, ok := rpmRelease(tt.distro)
+ if release != tt.release || major != tt.major || ok != (tt.release != "") {
+ t.Errorf("rpmRelease = %q, %q, %v; want %q, %q", release, major, ok, tt.release, tt.major)
+ }
+ })
+ }
+}
diff --git a/internal/mirror/source.go b/internal/mirror/source.go
index bbfddf0b..52a2846f 100644
--- a/internal/mirror/source.go
+++ b/internal/mirror/source.go
@@ -151,6 +151,8 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s
qualifiers = map[string]string{"uuid": p.Qualifier("uuid")}
case "alpine":
qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")}
+ case "rpm":
+ qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro"), "epoch": p.Qualifier("epoch")}
}
return ecosystem, name, qualifiers
}
diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go
index 3e2cebb6..56afad63 100644
--- a/internal/server/mirror_fetch_test.go
+++ b/internal/server/mirror_fetch_test.go
@@ -65,6 +65,8 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio
m.Versions = versions
m.JuliaUpstream = cfg.Upstream.Julia
m.APKRepositories = cfg.Upstream.APK
+ m.RPMUpstream = cfg.Upstream.RPM
+ m.RPMRepositoryPaths = cfg.Upstream.RPMRepositoryPaths
if versions == nil {
m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
}
diff --git a/internal/server/mirror_rpm_test.go b/internal/server/mirror_rpm_test.go
new file mode 100644
index 00000000..115cac12
--- /dev/null
+++ b/internal/server/mirror_rpm_test.go
@@ -0,0 +1,246 @@
+package server
+
+import (
+ "bytes"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/git-pkgs/proxy/internal/config"
+ "github.com/git-pkgs/proxy/internal/handler"
+)
+
+// rpmPackage is one package of a primary file, with the file it locates.
+type rpmPackage struct {
+ name, epoch, ver, rel, arch, body string
+ // file overrides the file name built from the name and version.
+ file string
+}
+
+func (p rpmPackage) filename() string {
+ if p.file != "" {
+ return p.file
+ }
+ return fmt.Sprintf("%s-%s-%s.%s.rpm", p.name, p.ver, p.rel, p.arch)
+}
+
+// rpmUpstream serves RPM repositories below its root and counts requests
+// by path.
+type rpmUpstream struct {
+ *httptest.Server
+ mu sync.Mutex
+ files map[string]string
+ hits map[string]int
+}
+
+func newRPMUpstream(t *testing.T) *rpmUpstream {
+ t.Helper()
+ u := &rpmUpstream{files: map[string]string{}, hits: map[string]int{}}
+ u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ u.mu.Lock()
+ body, ok := u.files[r.URL.Path]
+ u.hits[r.URL.Path]++
+ u.mu.Unlock()
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ _, _ = io.WriteString(w, body)
+ }))
+ t.Cleanup(u.Close)
+ return u
+}
+
+func (u *rpmUpstream) hitsFor(path string) int {
+ u.mu.Lock()
+ defer u.mu.Unlock()
+ return u.hits[path]
+}
+
+func rpmSHA256(s string) string {
+ sum := sha256.Sum256([]byte(s))
+ return hex.EncodeToString(sum[:])
+}
+
+// addRepository serves the repository at dir, below the root, with pkgs.
+func (u *rpmUpstream) addRepository(t *testing.T, dir string, pkgs ...rpmPackage) {
+ t.Helper()
+ var primary strings.Builder
+ primary.WriteString(``)
+ for _, p := range pkgs {
+ href := "Packages/" + p.name[:1] + "/" + p.filename()
+ fmt.Fprintf(&primary, `%s%s`+
+ `%s`,
+ p.name, p.arch, p.epoch, p.ver, p.rel, rpmSHA256(p.body), len(p.body), href)
+ u.files["/"+dir+"/"+href] = p.body
+ }
+ primary.WriteString(``)
+ var gz bytes.Buffer
+ zw := gzip.NewWriter(&gz)
+ _, _ = io.WriteString(zw, primary.String())
+ _ = zw.Close()
+ compressed := gz.String()
+ u.files["/"+dir+"/repodata/primary.xml.gz"] = compressed
+ u.files["/"+dir+"/repodata/repomd.xml"] = fmt.Sprintf(``+
+ `%s%d`,
+ rpmSHA256(compressed), len(compressed))
+}
+
+const (
+ rpmMirrorCurlPath = "/updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm"
+ rpmMirrorBashPath = "/releases/40/Everything/x86_64/os/Packages/b/bash-5.2.26-3.fc40.x86_64.rpm"
+ rpmMirrorTzdataPath = "/releases/40/Everything/x86_64/os/Packages/t/tzdata-2024a-5.fc40.noarch.rpm"
+)
+
+// newFedoraTree serves Fedora 40's release and updates repositories for
+// x86_64, laid out as on dl.fedoraproject.org.
+func newFedoraTree(t *testing.T) *rpmUpstream {
+ t.Helper()
+ u := newRPMUpstream(t)
+ u.addRepository(t, "releases/40/Everything/x86_64/os",
+ rpmPackage{"curl", "0", "8.6.0", "7.fc40", "x86_64", "curl old", ""},
+ rpmPackage{"bash", "0", "5.2.26", "3.fc40", "x86_64", "bash", ""},
+ rpmPackage{"tzdata", "0", "2024a", "5.fc40", "noarch", "tzdata", ""},
+ rpmPackage{"dual", "1", "2.0", "1.fc40", "x86_64", "dual one", ""},
+ rpmPackage{"odd", "0", "1.0", "1.fc40", "x86_64", "odd", "odd.rpm"},
+ )
+ u.addRepository(t, "updates/40/Everything/x86_64",
+ rpmPackage{"curl", "0", "8.6.0", "10.fc40", "x86_64", "curl new", ""},
+ rpmPackage{"dual", "2", "2.0", "1.fc40", "x86_64", "dual two", ""},
+ rpmPackage{"dual", "3", "2.0", "1.fc40", "x86_64", "dual three", ""},
+ )
+ return u
+}
+
+func rpmMirrorConfig(upstream *rpmUpstream) *config.Config {
+ cfg := config.Default()
+ cfg.Upstream.RPM = upstream.URL
+ return cfg
+}
+
+func getRPM(proxy *handler.Proxy, upstream *rpmUpstream, path string) *httptest.ResponseRecorder {
+ rec := httptest.NewRecorder()
+ h := handler.NewRPMHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL)
+ h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
+ return rec
+}
+
+func TestMirrorRPMPackagesAreServedFromCache(t *testing.T) {
+ upstream := newFedoraTree(t)
+
+ progress, proxy := mirrorWithConfig(t, rpmMirrorConfig(upstream),
+ // syft writes the epoch as a qualifier, 0 included.
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40",
+ "pkg:rpm/fedora/bash@5.2.26-3.fc40?arch=x86_64&distro=fedora-40&epoch=0",
+ "pkg:rpm/fedora/tzdata@2024a-5.fc40?arch=noarch&distro=fedora-40",
+ // A noarch package asked for by the machine's architecture.
+ "pkg:rpm/fedora/tzdata@2024a-5.fc40?arch=x86_64&distro=fedora-40")
+
+ if progress.Completed+progress.Skipped != 4 || progress.Failed != 0 {
+ t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors)
+ }
+ for _, path := range []string{rpmMirrorCurlPath, rpmMirrorBashPath, rpmMirrorTzdataPath,
+ "/updates/40/Everything/x86_64/repodata/primary.xml.gz", "/releases/40/Everything/x86_64/os/repodata/primary.xml.gz"} {
+ if got := upstream.hitsFor(path); got != 1 {
+ t.Errorf("%s fetched %d times, want 1", path, got)
+ }
+ }
+
+ // dnf gets the packages from the cache once the upstream is down.
+ upstream.Close()
+ for path, want := range map[string]string{
+ rpmMirrorCurlPath: "curl new",
+ rpmMirrorBashPath: "bash",
+ rpmMirrorTzdataPath: "tzdata",
+ } {
+ if rec := getRPM(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want {
+ t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want)
+ }
+ }
+}
+
+// upstream.rpm_repository_paths lays out another distribution's tree, with
+// the major release in the path.
+func TestMirrorRPMFollowsConfiguredRepositoryPaths(t *testing.T) {
+ upstream := newRPMUpstream(t)
+ upstream.addRepository(t, "9/BaseOS/x86_64/os", rpmPackage{"bash", "0", "5.1.8", "9.el9", "x86_64", "rocky bash", ""})
+ cfg := rpmMirrorConfig(upstream)
+ cfg.Upstream.RPMRepositoryPaths = []string{"{major}/AppStream/{arch}/os", "{major}/BaseOS/{arch}/os"}
+
+ progress, _ := mirrorWithConfig(t, cfg, "pkg:rpm/rocky/bash@5.1.8-9.el9?arch=x86_64&distro=rocky-9.4")
+ if progress.Completed != 1 || progress.Failed != 0 {
+ t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors)
+ }
+ if got := upstream.hitsFor("/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm"); got != 1 {
+ t.Errorf("package fetched %d times, want 1", got)
+ }
+}
+
+// An epoch picks one of the builds that differ only in it, from the first
+// repository that has it.
+func TestMirrorRPMEpochSelectsTheBuild(t *testing.T) {
+ upstream := newFedoraTree(t)
+
+ progress, _ := mirrorWithConfig(t, rpmMirrorConfig(upstream),
+ "pkg:rpm/fedora/dual@2.0-1.fc40?arch=x86_64&distro=fedora-40&epoch=1")
+ if progress.Completed != 1 || progress.Failed != 0 {
+ t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors)
+ }
+ if got := upstream.hitsFor("/releases/40/Everything/x86_64/os/Packages/d/dual-2.0-1.fc40.x86_64.rpm"); got != 1 {
+ t.Errorf("epoch 1 build fetched %d times, want 1", got)
+ }
+}
+
+func TestMirrorRPMRefusesWhatItCantFind(t *testing.T) {
+ upstream := newFedoraTree(t)
+
+ tests := map[string]string{
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?distro=fedora-40": "arch",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=../x&distro=fedora-40": "arch",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64": "distro",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora": "distro",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40/../x": "distro",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=x": "epoch",
+ "pkg:rpm/fedora/curl@9.9.9-1.fc40?arch=x86_64&distro=fedora-40": "isn't in",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=aarch64&distro=fedora-40": "isn't in",
+ "pkg:rpm/fedora/curl@8.6.0-10.fc40?arch=x86_64&distro=fedora-40&epoch=3": "isn't in",
+ "pkg:rpm/fedora/dual@2.0-1.fc40?arch=x86_64&distro=fedora-40": "epoch",
+ "pkg:rpm/fedora/odd@1.0-1.fc40?arch=x86_64&distro=fedora-40": "isn't a package file",
+ }
+ for p, want := range tests {
+ t.Run(p, func(t *testing.T) {
+ progress, _ := mirrorWithConfig(t, rpmMirrorConfig(upstream), p)
+ if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) {
+ t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want)
+ }
+ })
+ }
+}
+
+// A repository that isn't there is asked for once, not once per package.
+func TestMirrorRPMRemembersAMissingRepository(t *testing.T) {
+ upstream := newRPMUpstream(t)
+ upstream.addRepository(t, "9/BaseOS/x86_64/os",
+ rpmPackage{"bash", "0", "5.1.8", "9.el9", "x86_64", "rocky bash", ""},
+ rpmPackage{"curl", "0", "7.76.1", "29.el9", "x86_64", "rocky curl", ""},
+ )
+ cfg := rpmMirrorConfig(upstream)
+ cfg.Upstream.RPMRepositoryPaths = []string{"{major}/AppStream/{arch}/os", "{major}/BaseOS/{arch}/os"}
+
+ progress, _ := mirrorWithConfig(t, cfg,
+ "pkg:rpm/rocky/bash@5.1.8-9.el9?arch=x86_64&distro=rocky-9.4",
+ "pkg:rpm/rocky/curl@7.76.1-29.el9?arch=x86_64&distro=rocky-9.4")
+ if progress.Completed != 2 || progress.Failed != 0 {
+ t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors)
+ }
+ if got := upstream.hitsFor("/9/AppStream/x86_64/os/repodata/repomd.xml"); got != 1 {
+ t.Errorf("missing repository asked for %d times, want once", got)
+ }
+}
diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go
index 81491d95..73f580f5 100644
--- a/internal/server/mirror_versions.go
+++ b/internal/server/mirror_versions.go
@@ -41,11 +41,14 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror {
// UseMirrorUpstreams points m at the configured upstreams: it lists
// versions from them, through client, fetches Julia packages from
-// upstream.julia and Alpine packages from upstream.apk.
+// upstream.julia, Alpine packages from upstream.apk and RPM packages from
+// upstream.rpm.
func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) {
m.Versions = NewMirrorVersions(upstream, client)
m.JuliaUpstream = upstream.Julia
m.APKRepositories = upstream.APK
+ m.RPMUpstream = upstream.RPM
+ m.RPMRepositoryPaths = upstream.RPMRepositoryPaths
}
// mirrorVersionsURL returns the registry base URL to list a package's