From bcf72b69d5d01feb3a55152d3b588e5b39be7d8b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 16 Nov 2026 18:17:24 +0100 Subject: [PATCH 1/4] Mirror Debian packages Part of #414, now that the Debian client is in registries v0.14.0. The PURL namespace names the archive: an upstream.debian_repositories one by that name, or upstream.debian for debian or none. The distro qualifier gives the release, as syft and trivy write it (debian-12, debian-12.4, ubuntu-22.04) or as a codename, and arch the architecture. The package is looked up in the release's suites the archive serves, its epoch matched from the version or the epoch qualifier, checked against the index's SHA256 and cached under the path the Debian handler serves it from. Bumps registries to v0.14.0. --- docs/configuration.md | 2 + go.mod | 6 +- go.sum | 12 +- internal/handler/debian.go | 21 +- internal/mirror/deb.go | 280 ++++++++++++++++++++++++++ internal/mirror/mirror.go | 11 + internal/mirror/source.go | 9 +- internal/server/mirror_debian_test.go | 250 +++++++++++++++++++++++ internal/server/mirror_fetch_test.go | 2 + internal/server/mirror_versions.go | 5 +- 10 files changed, 581 insertions(+), 17 deletions(-) create mode 100644 internal/mirror/deb.go create mode 100644 internal/server/mirror_debian_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 26ba3714..6fd70f7d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1976,6 +1976,8 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +A Debian package is mirrored from the archive the namespace names: the one in `upstream.debian_repositories` by that name, or else `upstream.debian` for `debian` or no namespace, as syft and trivy write it in image SBOMs: `pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12`. The `distro` qualifier gives the release, as `debian-12`, a point release such as `debian-12.4`, `ubuntu-22.04` or a codename such as `bookworm`, and `arch` the architecture; packages without either are refused. The package is looked up in the release's suites the archive serves, of `{codename}`, `{codename}-updates` and `{codename}-security`, through their `Packages` indexes, which are checked against the `Release`. Debian keeps `{codename}-security` in an archive of its own, `deb.debian.org/debian-security`, which can be added to `upstream.debian_repositories` and named in the namespace. An epoch is matched from the version (`1:9.2p1-2+deb12u3`) or the `epoch` qualifier; without one, a version listed with two epochs is refused. The file is checked against the index's SHA256 and cached under the path `/debian/` serves it from, so apt gets it with the upstream down. The indexes aren't mirrored. Give a version: an archive keeps only the current one of each suite. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/go.mod b/go.mod index 12b05c61..c552f50e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 - github.com/git-pkgs/registries v0.13.0 + github.com/git-pkgs/registries v0.14.0 github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 @@ -75,7 +75,7 @@ require ( github.com/dustin/go-humanize v1.1.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/pom v0.1.9 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.8.0 // indirect github.com/go-logr/logr v1.4.4 // indirect @@ -114,7 +114,7 @@ require ( go.opentelemetry.io/otel/sdk v1.46.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect go.opentelemetry.io/otel/trace v1.46.0 // indirect - go.yaml.in/yaml/v3 v3.0.5 // indirect + go.yaml.in/yaml/v3 v3.0.6 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/go.sum b/go.sum index f993ecc9..93e42071 100644 --- a/go.sum +++ b/go.sum @@ -131,12 +131,12 @@ github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZ github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= -github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/pom v0.1.9 h1:gP7nqAS+UXITeRXjHhpyrMHRl9K3sRRBzRLV3+TeqHU= +github.com/git-pkgs/pom v0.1.9/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= -github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= +github.com/git-pkgs/registries v0.14.0 h1:PQbfRFghM6cN2xu1FcDHS2F73yOMATVIBaq+zY/6uSI= +github.com/git-pkgs/registries v0.14.0/go.mod h1:zgtz1lneDi8olIzAJxnSvSMT7PzChp9dsbFxJVbfyfQ= github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.6 h1:tbkqeYLKgAcDV1qPZNAkNxpNMTx9EfoNQQhcVEHApzs= @@ -303,8 +303,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= +go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA= gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= diff --git a/internal/handler/debian.go b/internal/handler/debian.go index cb68afdd..28b9e50a 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -58,12 +58,9 @@ func NewDebianHandler( upstreamURL string, repositories map[string]string, ) *DebianHandler { - if upstreamURL == "" { - upstreamURL = debianUpstream - } h := &DebianHandler{ proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + upstreamURL: DebianUpstreamURL(upstreamURL), proxyURL: strings.TrimSuffix(proxyURL, "/"), repositories: make(map[string]string, len(repositories)), } @@ -238,11 +235,27 @@ func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL, return nil, "" } +// DebianUpstreamURL returns the upstream.debian archive, the default one when +// none is configured. +func DebianUpstreamURL(configured string) string { + if configured == "" { + configured = debianUpstream + } + return strings.TrimSuffix(configured, "/") +} + // artifactCacheFilename names a package in the cache by its repository and // its path in it, since the same file name can hold different bytes in // different archives, and in different directories of one archive: Docker's // repository has a pool under each suite. func (h *DebianHandler) artifactCacheFilename(repository, path string) string { + return DebianCacheFilename(repository, path) +} + +// DebianCacheFilename is the name the Debian handler caches the package at +// path in the upstream.debian_repositories archive repository under, or in +// the upstream.debian archive when repository is empty. +func DebianCacheFilename(repository, path string) string { if repository == "" { return path } diff --git a/internal/mirror/deb.go b/internal/mirror/deb.go new file mode 100644 index 00000000..8d9534bd --- /dev/null +++ b/internal/mirror/deb.go @@ -0,0 +1,280 @@ +package mirror + +import ( + "context" + "fmt" + "net/http" + "net/url" + "regexp" + "slices" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// debIndexTTL is how long an archive's indexes are reused, so mirroring the +// packages of an image reads each index once. +const debIndexTTL = 5 * time.Minute + +var ( + // debReleasePattern takes the release from the distro qualifier syft and + // trivy write, such as debian-12, trivy's debian-12.4 or ubuntu-22.04. + debReleasePattern = regexp.MustCompile(`^(debian-\d+|ubuntu-\d+\.\d+)(?:\.\d+)?$`) + // debCodenamePattern matches a codename given as the distro qualifier. + debCodenamePattern = regexp.MustCompile(`^[a-z]+$`) + debArchPattern = regexp.MustCompile(`^[a-z0-9]+$`) + debEpochPattern = regexp.MustCompile(`^\d+$`) + + // debCodenames are the codenames of the Debian and Ubuntu releases, + // which their suites are named after. Other releases can be given by + // codename. + debCodenames = map[string]string{ + "debian-8": "jessie", "debian-9": "stretch", "debian-10": "buster", + "debian-11": "bullseye", "debian-12": "bookworm", "debian-13": "trixie", + "debian-14": "forky", + "ubuntu-16.04": "xenial", "ubuntu-18.04": "bionic", "ubuntu-20.04": "focal", + "ubuntu-22.04": "jammy", "ubuntu-24.04": "noble", "ubuntu-24.10": "oracular", + "ubuntu-25.04": "plucky", "ubuntu-25.10": "questing", + } + + // debSuffixes name the suites of a release that are read: the release, + // its updates and its security updates. An archive serves the ones it + // has; Debian keeps security updates in an archive of its own. + debSuffixes = []string{"", "-updates", "-security"} + + // debComponents are read where a suite's Release lists them. + debComponents = []string{"main", "contrib", "non-free", "non-free-firmware", "restricted", "universe", "multiverse"} +) + +// debIndexes keeps one registries client per archive, release and +// architecture, which holds the parsed indexes. +type debIndexes struct { + mu sync.Mutex + regs map[string]debIndex +} + +type debIndex struct { + reg registries.Registry + // suites are the suites of the release the archive serves. + suites []string + fetched time.Time +} + +// mirrorDeb mirrors a Debian package under the name the Debian handler +// caches it at, its path in the archive. The PURL namespace names the +// archive: an upstream.debian_repositories one by that name, or else +// upstream.debian for debian or none. The distro qualifier gives the release +// and arch the architecture; the package is looked up in the release's +// suites the archive serves. +func (m *Mirror) mirrorDeb(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + repository, archiveURL, err := m.debArchive(pv.Namespace) + if err != nil { + fail(err) + return + } + codename, err := debCodename(pv.Qualifiers["distro"]) + if err != nil { + fail(err) + return + } + arch := pv.Qualifiers["arch"] + if !debArchPattern.MatchString(arch) { + fail(fmt.Errorf("deb packages need an arch qualifier, such as arch=amd64, not %q", arch)) + return + } + epoch, version := debSplitEpoch(pv.Version) + if e := pv.Qualifiers["epoch"]; e != "" { + epoch = e + } + if epoch != "" && !debEpochPattern.MatchString(epoch) { + fail(fmt.Errorf("invalid epoch %q", epoch)) + return + } + + artifact, err := m.debArtifact(ctx, archiveURL, codename, arch, pv.Name, version, epoch) + if err != nil { + fail(err) + return + } + path := strings.TrimPrefix(artifact.URL, archiveURL+"/") + name, fileVersion, _ := handler.OSPackageIdentity("deb", path) + if name == "" { + fail(fmt.Errorf("%s isn't a package file the Debian handler serves", path)) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, fileVersion, + handler.DebianCacheFilename(repository, path), artifact.URL, fileDigest(artifact.Integrity)) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// debArchive returns the repository name and URL of the archive namespace +// names, with an empty name for upstream.debian. +func (m *Mirror) debArchive(namespace string) (string, string, error) { + if archiveURL, ok := m.DebianRepositories[namespace]; ok && namespace != "" { + return namespace, strings.TrimSuffix(archiveURL, "/"), nil + } + if namespace == "" || namespace == "debian" { + return "", handler.DebianUpstreamURL(m.DebianUpstream), nil + } + return "", "", fmt.Errorf("unknown Debian archive %q: it isn't in upstream.debian_repositories", namespace) +} + +// debCodename returns the codename of the release distro names. +func debCodename(distro string) (string, error) { + if match := debReleasePattern.FindStringSubmatch(distro); match != nil { + if codename, ok := debCodenames[match[1]]; ok { + return codename, nil + } + } else if debCodenamePattern.MatchString(distro) { + return distro, nil + } + return "", fmt.Errorf("deb packages need a distro qualifier with a known release, such as distro=debian-12, or its codename, such as distro=bookworm, not %q", distro) +} + +// debSplitEpoch splits the epoch off a Debian version, returning "" for +// one without. +func debSplitEpoch(version string) (epoch, rest string) { + if e, r, ok := strings.Cut(version, ":"); ok && debEpochPattern.MatchString(e) { + return e, r + } + return "", version +} + +// debArtifact returns the file of name for arch in the release codename of +// the archive whose version, without its epoch, is version. epoch, when +// given, picks the epoch; without it, versions that differ only in their +// epoch are refused rather than one picked. +func (m *Mirror) debArtifact(ctx context.Context, archiveURL, codename, arch, name, version, epoch string) (registries.Artifact, error) { + // Architecture: all packages are in every binary- index. + indexArch := arch + if arch == "all" { + indexArch = "amd64" + } + reg, suites, err := m.debRegistry(ctx, archiveURL, codename, indexArch) + if err != nil { + return registries.Artifact{}, err + } + notIn := fmt.Errorf("%s %s isn't in %s of %s for %s", name, version, strings.Join(suites, ", "), archiveURL, arch) + versions, err := reg.FetchVersions(ctx, name) + if isNotFound(err) { + return registries.Artifact{}, notIn + } + if err != nil { + m.deb.forget(debIndexKey(archiveURL, codename, indexArch)) + return registries.Artifact{}, fmt.Errorf("reading the indexes of %s: %w", archiveURL, err) + } + var found registries.Artifact + var epochs []string + for _, v := range versions { + e, rest := debSplitEpoch(v.Number) + if rest != version || (epoch != "" && debEpoch(e) != debEpoch(epoch)) { + continue + } + // The indexes are those of codename alone. + for _, a := range v.Artifacts { + if a.Metadata["arch"] == arch || a.Metadata["arch"] == "all" { + if !slices.Contains(epochs, debEpoch(e)) { + epochs = append(epochs, debEpoch(e)) + } + found = a + break + } + } + } + switch { + case len(epochs) == 0: + return registries.Artifact{}, notIn + case len(epochs) > 1: + return registries.Artifact{}, fmt.Errorf("%s %s has epochs %s in %s; give one in the version or the epoch qualifier", + name, version, strings.Join(epochs, " and "), archiveURL) + } + return found, nil +} + +// debEpoch is epoch with the implied 0 of a version without one. +func debEpoch(epoch string) string { + if epoch == "" { + return "0" + } + return epoch +} + +func debIndexKey(archiveURL, codename, arch string) string { + return archiveURL + "\x00" + codename + "\x00" + arch +} + +// debRegistry returns the client for the binary-arch indexes of the suites +// of release codename that the archive serves, reusing it, and the indexes it read, for +// debIndexTTL. It also returns the suites. +func (m *Mirror) debRegistry(ctx context.Context, archiveURL, codename, arch string) (registries.Registry, []string, error) { //nolint:ireturn // registries.New returns the interface + key := debIndexKey(archiveURL, codename, arch) + m.deb.mu.Lock() + defer m.deb.mu.Unlock() + if idx, ok := m.deb.regs[key]; ok && time.Since(idx.fetched) < debIndexTTL { + return idx.reg, idx.suites, nil + } + r, _ := m.Versions.(RegistryVersions) + client := r.client() + var suites []string + for _, suffix := range debSuffixes { + suite := codename + suffix + served, err := debSuiteServed(ctx, client, archiveURL, suite) + if err != nil { + return nil, nil, fmt.Errorf("checking %s for suite %s: %w", archiveURL, suite, err) + } + if served { + suites = append(suites, suite) + } + } + if len(suites) == 0 { + return nil, nil, fmt.Errorf("%s serves no suite of %s", archiveURL, codename) + } + query := url.Values{ + "suites": {strings.Join(suites, ",")}, + "components": {strings.Join(debComponents, ",")}, + "architectures": {arch}, + } + reg, err := registries.New("deb", archiveURL+"?"+query.Encode(), client) + if err != nil { + return nil, nil, err + } + if m.deb.regs == nil { + m.deb.regs = map[string]debIndex{} + } + m.deb.regs[key] = debIndex{reg: reg, suites: suites, fetched: time.Now()} + return reg, suites, nil +} + +// debSuiteServed reports whether the archive serves suite, by its InRelease +// or Release file. An answer other than the file or 404 is an error, so a +// failing archive doesn't leave a suite out for debIndexTTL. +func debSuiteServed(ctx context.Context, client *registries.Client, archiveURL, suite string) (bool, error) { + for _, file := range []string{"InRelease", "Release"} { + status, err := client.Head(ctx, archiveURL+"/dists/"+suite+"/"+file) + switch { + case err != nil: + return false, err + case status == http.StatusOK: + return true, nil + case status != http.StatusNotFound: + return false, fmt.Errorf("%s/dists/%s/%s: status %d", archiveURL, suite, file, status) + } + } + return false, nil +} + +// forget drops the client for key, so indexes that failed to load are read +// again next time. +func (d *debIndexes) forget(key string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.regs, key) +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 1dded24d..7f0d46ba 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -53,12 +53,19 @@ type Mirror struct { // the official Alpine mirror as "alpine". APKRepositories map[string]string + // DebianUpstream and DebianRepositories are upstream.debian and + // upstream.debian_repositories. An empty DebianUpstream uses + // deb.debian.org. + DebianUpstream string + DebianRepositories map[string]string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex apk apkIndexes + deb debIndexes } // New creates a new Mirror with the given dependencies. @@ -297,6 +304,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "deb" { + m.mirrorDeb(ctx, pv, tracker) + return + } if pv.Ecosystem == "git" { m.mirrorGit(ctx, pv, tracker) return diff --git a/internal/mirror/source.go b/internal/mirror/source.go index bbfddf0b..bc41f47d 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,8 +16,9 @@ type PackageVersion struct { Ecosystem string Name string Version string - // Namespace is the PURL namespace of apk packages, which Name leaves - // out. It picks the upstream.apk repository. + // Namespace is the PURL namespace of apk and deb packages, which Name + // leaves out. It picks the upstream.apk repository, or the Debian + // archive. Namespace string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select @@ -72,7 +73,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro ecosystem, name, qualifiers := mirrorTarget(p) var namespace string - if ecosystem == "alpine" { + if ecosystem == "alpine" || ecosystem == "deb" { namespace = p.Namespace } @@ -151,6 +152,8 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} case "alpine": qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} + case "deb": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro"), "epoch": p.Qualifier("epoch")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_debian_test.go b/internal/server/mirror_debian_test.go new file mode 100644 index 00000000..83876252 --- /dev/null +++ b/internal/server/mirror_debian_test.go @@ -0,0 +1,250 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +// debPackage is one stanza of a Packages index, with the pool file it names. +type debPackage struct { + name, version, arch, filename, body string +} + +// debUpstream serves Debian archives: the main one at its root and named +// ones under /{name}. It counts requests by path. +type debUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int + // fail answers the paths it holds with that status. + fail map[string]int +} + +func newDebUpstream(t *testing.T) *debUpstream { + t.Helper() + u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + status := u.fail[r.URL.Path] + u.mu.Unlock() + if status != 0 { + w.WriteHeader(status) + return + } + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// addSuite serves suite of the archive under root, with codename in its +// Release and pkgs in component main for each package's architecture. +func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) { + indexes := map[string]*strings.Builder{} + for _, p := range pkgs { + arch := p.arch + if arch == "all" { + arch = "amd64" + } + dir := "main/binary-" + arch + if indexes[dir] == nil { + indexes[dir] = &strings.Builder{} + } + fmt.Fprintf(indexes[dir], "Package: %s\nVersion: %s\nArchitecture: %s\nFilename: %s\nSize: %d\nSHA256: %s\n\n", + p.name, p.version, p.arch, p.filename, len(p.body), sha256Hex(p.body)) + u.files[root+"/"+p.filename] = p.body + } + dirs := make([]string, 0, len(indexes)) + for dir := range indexes { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + release := "Codename: " + codename + "\nSHA256:\n" + for _, dir := range dirs { + index := indexes[dir].String() + release += fmt.Sprintf(" %s %d %s/Packages\n", sha256Hex(index), len(index), dir) + u.files[root+"/dists/"+suite+"/"+dir+"/Packages"] = index + } + u.files[root+"/dists/"+suite+"/Release"] = release +} + +func (u *debUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +const ( + debMirrorCurlPath = "/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb" + debMirrorOpenSSHPath = "/pool/main/o/openssh/openssh-client_9.2p1-2+deb12u3_amd64.deb" + debMirrorTzdataPath = "/pool/main/t/tzdata/tzdata_2024a-0+deb12u1_all.deb" + debMirrorZlibPath = "/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" +) + +// newDebianArchives serves a Debian archive with bookworm and +// bookworm-updates, without bookworm-security, which Debian keeps in an +// archive of its own, and an Ubuntu one with jammy under /ubuntu. +func newDebianArchives(t *testing.T) *debUpstream { + t.Helper() + u := newDebUpstream(t) + u.addSuite("", "bookworm", "bookworm", + debPackage{"curl", "7.88.1-10+deb12u5", "amd64", "pool/main/c/curl/curl_7.88.1-10+deb12u5_amd64.deb", "curl old"}, + debPackage{"openssh-client", "1:9.2p1-2+deb12u3", "amd64", strings.TrimPrefix(debMirrorOpenSSHPath, "/"), "openssh-client"}, + debPackage{"tzdata", "2024a-0+deb12u1", "all", strings.TrimPrefix(debMirrorTzdataPath, "/"), "tzdata"}, + debPackage{"dual", "1:2.0-1", "amd64", "pool/main/d/dual/dual_2.0-1_amd64.deb", "dual one"}, + debPackage{"dual", "2:2.0-1", "amd64", "pool/main/d/dual/dual_2.0-1+e2_amd64.deb", "dual two"}, + debPackage{"odd", "1.0-1", "amd64", "pool/main/o/odd/odd.deb", "odd"}, + ) + u.addSuite("", "bookworm-updates", "bookworm", + debPackage{"curl", "7.88.1-10+deb12u8", "amd64", strings.TrimPrefix(debMirrorCurlPath, "/"), "curl new"}, + ) + u.addSuite("/ubuntu", "jammy", "jammy", + debPackage{"zlib1g", "1:1.2.11.dfsg-2ubuntu9.2", "amd64", strings.TrimPrefix(debMirrorZlibPath, "/"), "zlib1g"}, + ) + return u +} + +func debMirrorConfig(upstream *debUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Debian = upstream.URL + cfg.Upstream.DebianRepositories = map[string]string{"ubuntu": upstream.URL + "/ubuntu"} + return cfg +} + +func getDeb(proxy *handler.Proxy, upstream *debUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewDebianHandler(proxy, "http://proxy.test", upstream.URL, map[string]string{"ubuntu": upstream.URL + "/ubuntu"}) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestMirrorDebianPackagesAreServedFromCache(t *testing.T) { + upstream := newDebianArchives(t) + + progress, proxy := mirrorWithConfig(t, debMirrorConfig(upstream), + // syft: the release as distro, the epoch in the version. + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12", + "pkg:deb/debian/openssh-client@1%3A9.2p1-2+deb12u3?arch=amd64&distro=debian-12", + // trivy: the point release, and the epoch as a qualifier. + "pkg:deb/ubuntu/zlib1g@1.2.11.dfsg-2ubuntu9.2?arch=amd64&distro=ubuntu-22.04&epoch=1", + "pkg:deb/debian/curl@7.88.1-10+deb12u5?arch=amd64&distro=debian-12.4", + // A codename, and an Architecture: all package. + "pkg:deb/debian/tzdata@2024a-0+deb12u1?arch=all&distro=bookworm") + + if progress.Completed != 5 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{debMirrorCurlPath, debMirrorOpenSSHPath, debMirrorTzdataPath, "/ubuntu" + debMirrorZlibPath, "/dists/bookworm/main/binary-amd64/Packages"} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // apt gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + debMirrorCurlPath: "curl new", + debMirrorOpenSSHPath: "openssh-client", + debMirrorTzdataPath: "tzdata", + "/ubuntu" + debMirrorZlibPath: "zlib1g", + } { + if rec := getDeb(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorDebianRefusesWhatItCantFind(t *testing.T) { + upstream := newDebianArchives(t) + + tests := map[string]string{ + "pkg:deb/debian/curl@7.88.1-10+deb12u8?distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=../../etc&distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64": "distro", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-99": "distro", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=Bookworm/../x": "distro", + "pkg:deb/other/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12": "upstream.debian_repositories", + "pkg:deb/debian/curl@9.9.9-1?arch=amd64&distro=debian-12": "isn't in", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=arm64&distro=debian-12": "no Debian Packages index", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-13": "trixie", + "pkg:deb/debian/openssh-client@9.2p1-2+deb12u3?arch=amd64&distro=debian-12&epoch=2": "isn't in", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12&epoch=x": "epoch", + "pkg:deb/debian/odd@1.0-1?arch=amd64&distro=debian-12": "isn't a package file", + "pkg:deb/debian/dual@2.0-1?arch=amd64&distro=debian-12": "epoch", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} + +// An epoch picks one of two versions that differ only in it, given in the +// version or as a qualifier. +func TestMirrorDebianEpochSelectsTheVersion(t *testing.T) { + upstream := newDebianArchives(t) + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/dual@2.0-1?arch=amd64&distro=debian-12&epoch=2", + "pkg:deb/debian/dual@1%3A2.0-1?arch=amd64&distro=debian-12") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{"/pool/main/d/dual/dual_2.0-1+e2_amd64.deb", "/pool/main/d/dual/dual_2.0-1_amd64.deb"} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } +} + +// An archive failing for a suite fails the package, rather than leaving the +// suite out of the indexes that are reused. +func TestMirrorDebianFailsOnAFailingArchive(t *testing.T) { + upstream := newDebianArchives(t) + upstream.fail["/dists/bookworm-security/InRelease"] = http.StatusInternalServerError + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "bookworm-security") { + t.Errorf("failed = %d, errors = %v; want one error naming bookworm-security", progress.Failed, progress.Errors) + } +} + +// An Architecture: all package is found for any architecture, and epoch=0 +// matches a version without an epoch, as trivy writes it. +func TestMirrorDebianMatchesArchitectureAllAndEpochZero(t *testing.T) { + upstream := newDebianArchives(t) + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/tzdata@2024a-0+deb12u1?arch=amd64&distro=debian-12", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12&epoch=0") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 3e2cebb6..54d8c021 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -65,6 +65,8 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia m.APKRepositories = cfg.Upstream.APK + m.DebianUpstream = cfg.Upstream.Debian + m.DebianRepositories = cfg.Upstream.DebianRepositories if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 81491d95..e0518f0e 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -41,11 +41,14 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { // UseMirrorUpstreams points m at the configured upstreams: it lists // versions from them, through client, fetches Julia packages from -// upstream.julia and Alpine packages from upstream.apk. +// upstream.julia, Alpine packages from upstream.apk and Debian packages +// from upstream.debian and upstream.debian_repositories. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia m.APKRepositories = upstream.APK + m.DebianUpstream = upstream.Debian + m.DebianRepositories = upstream.DebianRepositories } // mirrorVersionsURL returns the registry base URL to list a package's From ab98f47bc0f44719f9c3bbc5c81cea00a6009c1d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 17 Nov 2026 12:02:34 +0100 Subject: [PATCH 2/4] Mirror Debian ports packages, and drop expired indexes An architecture can have a hyphen, such as hurd-amd64. An Architecture: all package is looked up in amd64's index where the Release lists amd64 or no architectures, and otherwise in the first it lists, as a ports archive has no amd64. Loading an archive's indexes drops those past their TTL. --- internal/mirror/deb.go | 76 +++++++++++++++++++++- internal/mirror/deb_test.go | 94 +++++++++++++++++++++++++++ internal/server/mirror_debian_test.go | 55 +++++++++++++++- 3 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 internal/mirror/deb_test.go diff --git a/internal/mirror/deb.go b/internal/mirror/deb.go index 8d9534bd..077028c3 100644 --- a/internal/mirror/deb.go +++ b/internal/mirror/deb.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "maps" "net/http" "net/url" "regexp" @@ -25,8 +26,10 @@ var ( debReleasePattern = regexp.MustCompile(`^(debian-\d+|ubuntu-\d+\.\d+)(?:\.\d+)?$`) // debCodenamePattern matches a codename given as the distro qualifier. debCodenamePattern = regexp.MustCompile(`^[a-z]+$`) - debArchPattern = regexp.MustCompile(`^[a-z0-9]+$`) - debEpochPattern = regexp.MustCompile(`^\d+$`) + // debArchPattern matches an architecture, such as amd64 or the ports' + // hurd-amd64. + debArchPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + debEpochPattern = regexp.MustCompile(`^\d+$`) // debCodenames are the codenames of the Debian and Ubuntu releases, // which their suites are named after. Other releases can be given by @@ -54,6 +57,14 @@ var ( type debIndexes struct { mu sync.Mutex regs map[string]debIndex + // allArchs are the architectures whose index is read for Architecture: + // all packages, by archive and release. + allArchs map[string]debAllArch +} + +type debAllArch struct { + arch string + fetched time.Time } type debIndex struct { @@ -156,7 +167,10 @@ func (m *Mirror) debArtifact(ctx context.Context, archiveURL, codename, arch, na // Architecture: all packages are in every binary- index. indexArch := arch if arch == "all" { - indexArch = "amd64" + var err error + if indexArch, err = m.debAllArch(ctx, archiveURL, codename); err != nil { + return registries.Artifact{}, err + } } reg, suites, err := m.debRegistry(ctx, archiveURL, codename, indexArch) if err != nil { @@ -207,6 +221,59 @@ func debEpoch(epoch string) string { return epoch } +// debAllArch returns the architecture whose index is read for +// Architecture: all packages of release codename: amd64 where the archive's +// Release lists it or lists none, and otherwise the first it lists, as a +// ports archive has no amd64. +func (m *Mirror) debAllArch(ctx context.Context, archiveURL, codename string) (string, error) { + key := archiveURL + "\x00" + codename + m.deb.mu.Lock() + defer m.deb.mu.Unlock() + if a, ok := m.deb.allArchs[key]; ok && time.Since(a.fetched) < debIndexTTL { + return a.arch, nil + } + r, _ := m.Versions.(RegistryVersions) + client := r.client() + arch := "amd64" + for _, file := range []string{"InRelease", "Release"} { + body, err := client.GetBody(ctx, archiveURL+"/dists/"+codename+"/"+file) + if isNotFound(err) { + continue + } + if err != nil { + return "", fmt.Errorf("reading the %s of %s: %w", file, codename, err) + } + arch = debReleaseAllArch(string(body)) + break + } + if m.deb.allArchs == nil { + m.deb.allArchs = map[string]debAllArch{} + } + m.deb.allArchs[key] = debAllArch{arch: arch, fetched: time.Now()} + return arch, nil +} + +// debReleaseAllArch picks the architecture for Architecture: all packages +// from a Release's Architectures field. +func debReleaseAllArch(release string) string { + for line := range strings.Lines(release) { + value, ok := strings.CutPrefix(line, "Architectures:") + if !ok { + continue + } + var listed []string + for _, a := range strings.Fields(value) { + if a != "all" && debArchPattern.MatchString(a) { + listed = append(listed, a) + } + } + if len(listed) > 0 && !slices.Contains(listed, "amd64") { + return listed[0] + } + } + return "amd64" +} + func debIndexKey(archiveURL, codename, arch string) string { return archiveURL + "\x00" + codename + "\x00" + arch } @@ -249,6 +316,9 @@ func (m *Mirror) debRegistry(ctx context.Context, archiveURL, codename, arch str if m.deb.regs == nil { m.deb.regs = map[string]debIndex{} } + // Indexes no longer asked for are dropped, rather than kept for the + // Mirror's life. + maps.DeleteFunc(m.deb.regs, func(_ string, idx debIndex) bool { return time.Since(idx.fetched) >= debIndexTTL }) m.deb.regs[key] = debIndex{reg: reg, suites: suites, fetched: time.Now()} return reg, suites, nil } diff --git a/internal/mirror/deb_test.go b/internal/mirror/deb_test.go new file mode 100644 index 00000000..9f9005aa --- /dev/null +++ b/internal/mirror/deb_test.go @@ -0,0 +1,94 @@ +package mirror + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +// Loading an archive's indexes drops those past debIndexTTL, so indexes no +// longer asked for aren't kept for the Mirror's life. +func TestDebRegistryDropsExpiredIndexes(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(srv.Close) + m := &Mirror{} + ctx := context.Background() + + if _, _, err := m.debRegistry(ctx, srv.URL+"/old", "bookworm", "amd64"); err != nil { + t.Fatal(err) + } + if _, _, err := m.debRegistry(ctx, srv.URL+"/fresh", "bookworm", "amd64"); err != nil { + t.Fatal(err) + } + old := debIndexKey(srv.URL+"/old", "bookworm", "amd64") + idx := m.deb.regs[old] + idx.fetched = time.Now().Add(-debIndexTTL) + m.deb.regs[old] = idx + + if _, _, err := m.debRegistry(ctx, srv.URL+"/new", "bookworm", "amd64"); err != nil { + t.Fatal(err) + } + if _, ok := m.deb.regs[old]; ok { + t.Error("expired index kept") + } + if _, ok := m.deb.regs[debIndexKey(srv.URL+"/fresh", "bookworm", "amd64")]; !ok { + t.Error("index within its TTL dropped") + } +} + +func TestDebReleaseAllArch(t *testing.T) { + tests := map[string]string{ + "Codename: jammy\n": "amd64", + "Architectures: amd64 arm64 i386\n": "amd64", + "Architectures: arm64 amd64\n": "amd64", + "Codename: jammy\nArchitectures: all arm64 armhf\n": "arm64", + "Architectures: armhf\nComponents: main\n": "armhf", + "Architectures: all\n": "amd64", + "Architectures: ../x arm64\n": "arm64", + } + for release, want := range tests { + if got := debReleaseAllArch(release); got != want { + t.Errorf("debReleaseAllArch(%q) = %q, want %q", release, got, want) + } + } +} + +// The architecture for Architecture: all packages is read from the Release +// once per debIndexTTL. +func TestDebAllArchIsReadOncePerTTL(t *testing.T) { + var reads atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/jammy/InRelease" { + http.NotFound(w, r) + return + } + reads.Add(1) + _, _ = io.WriteString(w, "Architectures: arm64\n") + })) + t.Cleanup(srv.Close) + m := &Mirror{} + ctx := context.Background() + + for range 2 { + if arch, err := m.debAllArch(ctx, srv.URL, "jammy"); err != nil || arch != "arm64" { + t.Fatalf("debAllArch = %q, %v; want arm64", arch, err) + } + } + if got := reads.Load(); got != 1 { + t.Errorf("InRelease read %d times, want once", got) + } + key := srv.URL + "\x00jammy" + a := m.deb.allArchs[key] + a.fetched = time.Now().Add(-debIndexTTL) + m.deb.allArchs[key] = a + if _, err := m.debAllArch(ctx, srv.URL, "jammy"); err != nil { + t.Fatal(err) + } + if got := reads.Load(); got != 2 { + t.Errorf("InRelease read %d times after the TTL, want twice", got) + } +} diff --git a/internal/server/mirror_debian_test.go b/internal/server/mirror_debian_test.go index 83876252..1981e3a8 100644 --- a/internal/server/mirror_debian_test.go +++ b/internal/server/mirror_debian_test.go @@ -30,11 +30,15 @@ type debUpstream struct { hits map[string]int // fail answers the paths it holds with that status. fail map[string]int + // architectures, by archive root, are the architectures its Releases + // list, the first of which holds Architecture: all packages. An + // archive not in it lists none and holds them in binary-amd64. + architectures map[string][]string } func newDebUpstream(t *testing.T) *debUpstream { t.Helper() - u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}} + u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}, architectures: map[string][]string{}} u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.mu.Lock() body, ok := u.files[r.URL.Path] @@ -68,6 +72,9 @@ func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) arch := p.arch if arch == "all" { arch = "amd64" + if archs := u.architectures[root]; len(archs) > 0 { + arch = archs[0] + } } dir := "main/binary-" + arch if indexes[dir] == nil { @@ -82,7 +89,11 @@ func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) dirs = append(dirs, dir) } sort.Strings(dirs) - release := "Codename: " + codename + "\nSHA256:\n" + release := "Codename: " + codename + "\n" + if archs := u.architectures[root]; len(archs) > 0 { + release += "Architectures: " + strings.Join(archs, " ") + "\n" + } + release += "SHA256:\n" for _, dir := range dirs { index := indexes[dir].String() release += fmt.Sprintf(" %s %d %s/Packages\n", sha256Hex(index), len(index), dir) @@ -183,6 +194,7 @@ func TestMirrorDebianRefusesWhatItCantFind(t *testing.T) { tests := map[string]string{ "pkg:deb/debian/curl@7.88.1-10+deb12u8?distro=debian-12": "arch", "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=../../etc&distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=-amd64&distro=debian-12": "arch", "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64": "distro", "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-99": "distro", "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=Bookworm/../x": "distro", @@ -248,3 +260,42 @@ func TestMirrorDebianMatchesArchitectureAllAndEpochZero(t *testing.T) { t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) } } + +// An architecture name can have a hyphen, as Debian's ports have. +func TestMirrorDebianArchitectureWithHyphen(t *testing.T) { + upstream := newDebUpstream(t) + upstream.addSuite("/ports", "sid", "sid", + debPackage{"hello", "2.10-3", "hurd-amd64", "pool/main/h/hello/hello_2.10-3_hurd-amd64.deb", "hello"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/ports/hello@2.10-3?arch=hurd-amd64&distro=sid") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/pool/main/h/hello/hello_2.10-3_hurd-amd64.deb"); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +// An Architecture: all package is looked up in the index of an architecture +// the archive's Release lists, as a ports archive has no amd64. +func TestMirrorDebianArchitectureAllOnAPortsArchive(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/ports"] = []string{"arm64", "armhf"} + upstream.addSuite("/ports", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, + "pkg:deb/ports/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/dists/jammy/main/binary-amd64/Packages"); got != 0 { + t.Errorf("binary-amd64 read %d times, want none", got) + } +} From 4fd548320516c89fe37620bb9ce56a63090a55e6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 17 Nov 2026 17:48:31 +0100 Subject: [PATCH 3/4] Read the components a Debian suite's Release lists The mirror read only Debian's and Ubuntu's usual components, so a package of Docker's stable or Raspbian's rpi wasn't found. Read each suite's InRelease or Release once for the suites served, the components they list and the architecture for Architecture: all packages, instead of a HEAD for each suite and a second read for the architecture. --- docs/configuration.md | 2 +- internal/mirror/deb.go | 183 +++++++++++++++----------- internal/mirror/deb_test.go | 120 +++++++++++++++-- internal/server/mirror_debian_test.go | 56 +++++++- 4 files changed, 267 insertions(+), 94 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c30accea..bb1ea5fb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2006,7 +2006,7 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. -A Debian package is mirrored from the archive the namespace names: the one in `upstream.debian_repositories` by that name, or else `upstream.debian` for `debian` or no namespace, as syft and trivy write it in image SBOMs: `pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12`. The `distro` qualifier gives the release, as `debian-12`, a point release such as `debian-12.4`, `ubuntu-22.04` or a codename such as `bookworm`, and `arch` the architecture; packages without either are refused. The package is looked up in the release's suites the archive serves, of `{codename}`, `{codename}-updates` and `{codename}-security`, through their `Packages` indexes, which are checked against the `Release`. Debian keeps `{codename}-security` in an archive of its own, `deb.debian.org/debian-security`, which can be added to `upstream.debian_repositories` and named in the namespace. An epoch is matched from the version (`1:9.2p1-2+deb12u3`) or the `epoch` qualifier; without one, a version listed with two epochs is refused. The file is checked against the index's SHA256 and cached under the path `/debian/` serves it from, so apt gets it with the upstream down. The indexes aren't mirrored. Give a version: an archive keeps only the current one of each suite. +A Debian package is mirrored from the archive the namespace names: the one in `upstream.debian_repositories` by that name, or else `upstream.debian` for `debian` or no namespace, as syft and trivy write it in image SBOMs: `pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12`. The `distro` qualifier gives the release, as `debian-12`, a point release such as `debian-12.4`, `ubuntu-22.04` or a codename such as `bookworm`, and `arch` the architecture; packages without either are refused. The package is looked up in the release's suites the archive serves, of `{codename}`, `{codename}-updates` and `{codename}-security`, through the `Packages` indexes of the components their `Release` lists, such as `stable` in Docker's apt repository, which are checked against the `Release`. A suite whose `Release` lists none is read for `main`, `contrib`, `non-free`, `non-free-firmware`, `restricted`, `universe` and `multiverse`. Debian keeps `{codename}-security` in an archive of its own, `deb.debian.org/debian-security`, which can be added to `upstream.debian_repositories` and named in the namespace. An epoch is matched from the version (`1:9.2p1-2+deb12u3`) or the `epoch` qualifier; without one, a version listed with two epochs is refused. The file is checked against the index's SHA256 and cached under the path `/debian/` serves it from, so apt gets it with the upstream down. The indexes aren't mirrored. Give a version: an archive keeps only the current one of each suite. | Flag | Default | Description | |------|---------|-------------| diff --git a/internal/mirror/deb.go b/internal/mirror/deb.go index 077028c3..c5fb1012 100644 --- a/internal/mirror/deb.go +++ b/internal/mirror/deb.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "maps" - "net/http" "net/url" "regexp" "slices" @@ -28,8 +27,10 @@ var ( debCodenamePattern = regexp.MustCompile(`^[a-z]+$`) // debArchPattern matches an architecture, such as amd64 or the ports' // hurd-amd64. - debArchPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) - debEpochPattern = regexp.MustCompile(`^\d+$`) + debArchPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + // debComponentPattern matches the components read from a Release. + debComponentPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.+_-]*$`) + debEpochPattern = regexp.MustCompile(`^\d+$`) // debCodenames are the codenames of the Debian and Ubuntu releases, // which their suites are named after. Other releases can be given by @@ -48,7 +49,8 @@ var ( // has; Debian keeps security updates in an archive of its own. debSuffixes = []string{"", "-updates", "-security"} - // debComponents are read where a suite's Release lists them. + // debComponents are read where no Release of a release lists its + // components. debComponents = []string{"main", "contrib", "non-free", "non-free-firmware", "restricted", "universe", "multiverse"} ) @@ -57,20 +59,22 @@ var ( type debIndexes struct { mu sync.Mutex regs map[string]debIndex - // allArchs are the architectures whose index is read for Architecture: - // all packages, by archive and release. - allArchs map[string]debAllArch + // releases are what the Releases of a release's suites give, by + // archive and release. + releases map[string]debReleaseInfo } -type debAllArch struct { - arch string - fetched time.Time +// debReleaseInfo is what the Releases of a release's suites give: the suites +// the archive serves, the components they list, and the architecture whose +// index is read for Architecture: all packages. +type debReleaseInfo struct { + suites, components []string + allArch string + fetched time.Time } type debIndex struct { - reg registries.Registry - // suites are the suites of the release the archive serves. - suites []string + reg registries.Registry fetched time.Time } @@ -164,19 +168,20 @@ func debSplitEpoch(version string) (epoch, rest string) { // given, picks the epoch; without it, versions that differ only in their // epoch are refused rather than one picked. func (m *Mirror) debArtifact(ctx context.Context, archiveURL, codename, arch, name, version, epoch string) (registries.Artifact, error) { + rel, err := m.debReleaseInfo(ctx, archiveURL, codename) + if err != nil { + return registries.Artifact{}, err + } // Architecture: all packages are in every binary- index. indexArch := arch if arch == "all" { - var err error - if indexArch, err = m.debAllArch(ctx, archiveURL, codename); err != nil { - return registries.Artifact{}, err - } + indexArch = rel.allArch } - reg, suites, err := m.debRegistry(ctx, archiveURL, codename, indexArch) + reg, err := m.debRegistry(archiveURL, codename, indexArch, rel) if err != nil { return registries.Artifact{}, err } - notIn := fmt.Errorf("%s %s isn't in %s of %s for %s", name, version, strings.Join(suites, ", "), archiveURL, arch) + notIn := fmt.Errorf("%s %s isn't in %s of %s for %s", name, version, strings.Join(rel.suites, ", "), archiveURL, arch) versions, err := reg.FetchVersions(ctx, name) if isNotFound(err) { return registries.Artifact{}, notIn @@ -221,36 +226,76 @@ func debEpoch(epoch string) string { return epoch } -// debAllArch returns the architecture whose index is read for -// Architecture: all packages of release codename: amd64 where the archive's -// Release lists it or lists none, and otherwise the first it lists, as a -// ports archive has no amd64. -func (m *Mirror) debAllArch(ctx context.Context, archiveURL, codename string) (string, error) { +// debReleaseInfo returns what the Releases of the suites of release codename +// give, reading each suite's once per debIndexTTL: the suites the archive +// serves, the components they list, or debComponents where none lists any, +// and the architecture for Architecture: all packages, from the first +// served suite's Release. +func (m *Mirror) debReleaseInfo(ctx context.Context, archiveURL, codename string) (debReleaseInfo, error) { key := archiveURL + "\x00" + codename m.deb.mu.Lock() defer m.deb.mu.Unlock() - if a, ok := m.deb.allArchs[key]; ok && time.Since(a.fetched) < debIndexTTL { - return a.arch, nil + if rel, ok := m.deb.releases[key]; ok && time.Since(rel.fetched) < debIndexTTL { + return rel, nil } r, _ := m.Versions.(RegistryVersions) client := r.client() - arch := "amd64" - for _, file := range []string{"InRelease", "Release"} { - body, err := client.GetBody(ctx, archiveURL+"/dists/"+codename+"/"+file) - if isNotFound(err) { + var rel debReleaseInfo + for _, suffix := range debSuffixes { + suite := codename + suffix + body, served, err := debReleaseFile(ctx, client, archiveURL, suite) + if err != nil { + return debReleaseInfo{}, fmt.Errorf("checking %s for suite %s: %w", archiveURL, suite, err) + } + if !served { continue } - if err != nil { - return "", fmt.Errorf("reading the %s of %s: %w", file, codename, err) + if rel.suites == nil { + rel.allArch = debReleaseAllArch(body) } - arch = debReleaseAllArch(string(body)) - break + rel.suites = append(rel.suites, suite) + for _, c := range debReleaseComponents(body) { + if !slices.Contains(rel.components, c) { + rel.components = append(rel.components, c) + } + } + } + if len(rel.suites) == 0 { + return debReleaseInfo{}, fmt.Errorf("%s serves no suite of %s", archiveURL, codename) } - if m.deb.allArchs == nil { - m.deb.allArchs = map[string]debAllArch{} + if rel.components == nil { + rel.components = debComponents + } + rel.fetched = time.Now() + if m.deb.releases == nil { + m.deb.releases = map[string]debReleaseInfo{} + } + maps.DeleteFunc(m.deb.releases, func(_ string, rel debReleaseInfo) bool { return time.Since(rel.fetched) >= debIndexTTL }) + m.deb.releases[key] = rel + return rel, nil +} + +// debReleaseComponents returns the components a Release lists. A +// component below a directory, as debian-security lists updates/main, is +// read where its last part names it, as apt reads it. +func debReleaseComponents(release string) []string { + var components []string + for line := range strings.Lines(release) { + value, ok := strings.CutPrefix(line, "Components:") + if !ok { + continue + } + for _, c := range strings.Fields(value) { + parts := strings.Split(c, "/") + if slices.ContainsFunc(parts, func(p string) bool { return !debComponentPattern.MatchString(p) }) { + continue + } + if c = parts[len(parts)-1]; !slices.Contains(components, c) { + components = append(components, c) + } + } } - m.deb.allArchs[key] = debAllArch{arch: arch, fetched: time.Now()} - return arch, nil + return components } // debReleaseAllArch picks the architecture for Architecture: all packages @@ -278,40 +323,25 @@ func debIndexKey(archiveURL, codename, arch string) string { return archiveURL + "\x00" + codename + "\x00" + arch } -// debRegistry returns the client for the binary-arch indexes of the suites -// of release codename that the archive serves, reusing it, and the indexes it read, for -// debIndexTTL. It also returns the suites. -func (m *Mirror) debRegistry(ctx context.Context, archiveURL, codename, arch string) (registries.Registry, []string, error) { //nolint:ireturn // registries.New returns the interface +// debRegistry returns the client for the binary-arch indexes of the +// components of the suites of release codename that rel names, reusing it, +// and the indexes it read, for debIndexTTL. +func (m *Mirror) debRegistry(archiveURL, codename, arch string, rel debReleaseInfo) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface key := debIndexKey(archiveURL, codename, arch) m.deb.mu.Lock() defer m.deb.mu.Unlock() if idx, ok := m.deb.regs[key]; ok && time.Since(idx.fetched) < debIndexTTL { - return idx.reg, idx.suites, nil + return idx.reg, nil } r, _ := m.Versions.(RegistryVersions) - client := r.client() - var suites []string - for _, suffix := range debSuffixes { - suite := codename + suffix - served, err := debSuiteServed(ctx, client, archiveURL, suite) - if err != nil { - return nil, nil, fmt.Errorf("checking %s for suite %s: %w", archiveURL, suite, err) - } - if served { - suites = append(suites, suite) - } - } - if len(suites) == 0 { - return nil, nil, fmt.Errorf("%s serves no suite of %s", archiveURL, codename) - } query := url.Values{ - "suites": {strings.Join(suites, ",")}, - "components": {strings.Join(debComponents, ",")}, + "suites": {strings.Join(rel.suites, ",")}, + "components": {strings.Join(rel.components, ",")}, "architectures": {arch}, } - reg, err := registries.New("deb", archiveURL+"?"+query.Encode(), client) + reg, err := registries.New("deb", archiveURL+"?"+query.Encode(), r.client()) if err != nil { - return nil, nil, err + return nil, err } if m.deb.regs == nil { m.deb.regs = map[string]debIndex{} @@ -319,26 +349,25 @@ func (m *Mirror) debRegistry(ctx context.Context, archiveURL, codename, arch str // Indexes no longer asked for are dropped, rather than kept for the // Mirror's life. maps.DeleteFunc(m.deb.regs, func(_ string, idx debIndex) bool { return time.Since(idx.fetched) >= debIndexTTL }) - m.deb.regs[key] = debIndex{reg: reg, suites: suites, fetched: time.Now()} - return reg, suites, nil + m.deb.regs[key] = debIndex{reg: reg, fetched: time.Now()} + return reg, nil } -// debSuiteServed reports whether the archive serves suite, by its InRelease -// or Release file. An answer other than the file or 404 is an error, so a -// failing archive doesn't leave a suite out for debIndexTTL. -func debSuiteServed(ctx context.Context, client *registries.Client, archiveURL, suite string) (bool, error) { +// debReleaseFile returns the InRelease or Release of suite, and whether +// the archive serves one. An answer other than the file or 404 is an error, +// so a failing archive doesn't leave a suite out for debIndexTTL. +func debReleaseFile(ctx context.Context, client *registries.Client, archiveURL, suite string) (string, bool, error) { for _, file := range []string{"InRelease", "Release"} { - status, err := client.Head(ctx, archiveURL+"/dists/"+suite+"/"+file) - switch { - case err != nil: - return false, err - case status == http.StatusOK: - return true, nil - case status != http.StatusNotFound: - return false, fmt.Errorf("%s/dists/%s/%s: status %d", archiveURL, suite, file, status) + body, err := client.GetBody(ctx, archiveURL+"/dists/"+suite+"/"+file) + if isNotFound(err) { + continue + } + if err != nil { + return "", false, err } + return string(body), true, nil } - return false, nil + return "", false, nil } // forget drops the client for key, so indexes that failed to load are read diff --git a/internal/mirror/deb_test.go b/internal/mirror/deb_test.go index 9f9005aa..8f233db0 100644 --- a/internal/mirror/deb_test.go +++ b/internal/mirror/deb_test.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "sync/atomic" "testing" "time" @@ -16,12 +17,12 @@ func TestDebRegistryDropsExpiredIndexes(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) t.Cleanup(srv.Close) m := &Mirror{} - ctx := context.Background() + rel := debReleaseInfo{suites: []string{"bookworm"}, components: []string{"main"}} - if _, _, err := m.debRegistry(ctx, srv.URL+"/old", "bookworm", "amd64"); err != nil { + if _, err := m.debRegistry(srv.URL+"/old", "bookworm", "amd64", rel); err != nil { t.Fatal(err) } - if _, _, err := m.debRegistry(ctx, srv.URL+"/fresh", "bookworm", "amd64"); err != nil { + if _, err := m.debRegistry(srv.URL+"/fresh", "bookworm", "amd64", rel); err != nil { t.Fatal(err) } old := debIndexKey(srv.URL+"/old", "bookworm", "amd64") @@ -29,7 +30,7 @@ func TestDebRegistryDropsExpiredIndexes(t *testing.T) { idx.fetched = time.Now().Add(-debIndexTTL) m.deb.regs[old] = idx - if _, _, err := m.debRegistry(ctx, srv.URL+"/new", "bookworm", "amd64"); err != nil { + if _, err := m.debRegistry(srv.URL+"/new", "bookworm", "amd64", rel); err != nil { t.Fatal(err) } if _, ok := m.deb.regs[old]; ok { @@ -57,9 +58,29 @@ func TestDebReleaseAllArch(t *testing.T) { } } -// The architecture for Architecture: all packages is read from the Release -// once per debIndexTTL. -func TestDebAllArchIsReadOncePerTTL(t *testing.T) { +func TestDebReleaseComponents(t *testing.T) { + tests := map[string][]string{ + "Codename: bookworm\n": nil, + "Components: main contrib non-free-firmware non-free\n": {"main", "contrib", "non-free-firmware", "non-free"}, + "Components: stable edge test nightly\n": {"stable", "edge", "test", "nightly"}, + // debian-security names its components below updates/, and keeps + // their indexes where the last part of that names them, as apt + // reads them. + "Components: updates/main updates/contrib updates/main\n": {"main", "contrib"}, + "Components: ../x . main%2f\n": nil, + "Codename: bookworm\nComponents: main\nArchitectures: all\n": {"main"}, + } + for release, want := range tests { + if got := debReleaseComponents(release); !slices.Equal(got, want) { + t.Errorf("debReleaseComponents(%q) = %q, want %q", release, got, want) + } + } +} + +// An archive's Releases are read once per debIndexTTL, for the suites it +// serves, their components and the architecture for Architecture: all +// packages. +func TestDebReleaseIsReadOncePerTTL(t *testing.T) { var reads atomic.Int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/dists/jammy/InRelease" { @@ -67,28 +88,99 @@ func TestDebAllArchIsReadOncePerTTL(t *testing.T) { return } reads.Add(1) - _, _ = io.WriteString(w, "Architectures: arm64\n") + _, _ = io.WriteString(w, "Architectures: arm64\nComponents: stable\n") })) t.Cleanup(srv.Close) m := &Mirror{} ctx := context.Background() + want := debReleaseInfo{suites: []string{"jammy"}, components: []string{"stable"}, allArch: "arm64"} for range 2 { - if arch, err := m.debAllArch(ctx, srv.URL, "jammy"); err != nil || arch != "arm64" { - t.Fatalf("debAllArch = %q, %v; want arm64", arch, err) + rel, err := m.debReleaseInfo(ctx, srv.URL, "jammy") + if err != nil || !slices.Equal(rel.suites, want.suites) || !slices.Equal(rel.components, want.components) || rel.allArch != want.allArch { + t.Fatalf("debReleaseInfo = %+v, %v; want %+v", rel, err, want) } } if got := reads.Load(); got != 1 { t.Errorf("InRelease read %d times, want once", got) } key := srv.URL + "\x00jammy" - a := m.deb.allArchs[key] - a.fetched = time.Now().Add(-debIndexTTL) - m.deb.allArchs[key] = a - if _, err := m.debAllArch(ctx, srv.URL, "jammy"); err != nil { + rel := m.deb.releases[key] + rel.fetched = time.Now().Add(-debIndexTTL) + m.deb.releases[key] = rel + if _, err := m.debReleaseInfo(ctx, srv.URL, "jammy"); err != nil { t.Fatal(err) } if got := reads.Load(); got != 2 { t.Errorf("InRelease read %d times after the TTL, want twice", got) } } + +// A suite whose Release lists no components is read for the usual ones, +// and one without amd64 among its architectures for the first it lists. +func TestDebReleaseDefaults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/jammy/Release" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "Codename: jammy\n") + })) + t.Cleanup(srv.Close) + rel, err := (&Mirror{}).debReleaseInfo(context.Background(), srv.URL, "jammy") + if err != nil || !slices.Equal(rel.components, debComponents) || rel.allArch != "amd64" { + t.Errorf("debReleaseInfo = %+v, %v; want the default components and amd64", rel, err) + } +} + +// The components of all served suites are read, each once, and the +// architecture for Architecture: all packages is the release's own. +func TestDebReleaseOfSeveralSuites(t *testing.T) { + releases := map[string]string{ + "/dists/jammy/InRelease": "Architectures: arm64 armhf\nComponents: stable\n", + "/dists/jammy-updates/InRelease": "Architectures: armhf\nComponents: stable test\n", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := releases[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + rel, err := (&Mirror{}).debReleaseInfo(context.Background(), srv.URL, "jammy") + if err != nil { + t.Fatal(err) + } + if !slices.Equal(rel.suites, []string{"jammy", "jammy-updates"}) || !slices.Equal(rel.components, []string{"stable", "test"}) || rel.allArch != "arm64" { + t.Errorf("debReleaseInfo = %+v; want jammy and jammy-updates, stable and test, arm64", rel) + } +} + +// Reading a release's Releases drops those read past debIndexTTL. +func TestDebReleaseInfoDropsExpiredEntries(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(srv.Close) + m := &Mirror{} + ctx := context.Background() + for _, root := range []string{"/old", "/fresh"} { + if _, err := m.debReleaseInfo(ctx, srv.URL+root, "bookworm"); err != nil { + t.Fatal(err) + } + } + old := srv.URL + "/old\x00bookworm" + rel := m.deb.releases[old] + rel.fetched = time.Now().Add(-debIndexTTL) + m.deb.releases[old] = rel + + if _, err := m.debReleaseInfo(ctx, srv.URL+"/new", "bookworm"); err != nil { + t.Fatal(err) + } + if _, ok := m.deb.releases[old]; ok { + t.Error("expired entry kept") + } + if _, ok := m.deb.releases[srv.URL+"/fresh\x00bookworm"]; !ok { + t.Error("entry within its TTL dropped") + } +} diff --git a/internal/server/mirror_debian_test.go b/internal/server/mirror_debian_test.go index 1981e3a8..c9fdfc1d 100644 --- a/internal/server/mirror_debian_test.go +++ b/internal/server/mirror_debian_test.go @@ -34,11 +34,15 @@ type debUpstream struct { // list, the first of which holds Architecture: all packages. An // archive not in it lists none and holds them in binary-amd64. architectures map[string][]string + // components, by archive root, is the component its packages are in, + // which its Releases list. An archive not in it keeps them in main and + // lists no components. + components map[string]string } func newDebUpstream(t *testing.T) *debUpstream { t.Helper() - u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}, architectures: map[string][]string{}} + u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}, architectures: map[string][]string{}, components: map[string]string{}} u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.mu.Lock() body, ok := u.files[r.URL.Path] @@ -76,7 +80,11 @@ func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) arch = archs[0] } } - dir := "main/binary-" + arch + component := "main" + if c := u.components[root]; c != "" { + component = c + } + dir := component + "/binary-" + arch if indexes[dir] == nil { indexes[dir] = &strings.Builder{} } @@ -93,6 +101,9 @@ func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) if archs := u.architectures[root]; len(archs) > 0 { release += "Architectures: " + strings.Join(archs, " ") + "\n" } + if c := u.components[root]; c != "" { + release += "Components: " + c + "\n" + } release += "SHA256:\n" for _, dir := range dirs { index := indexes[dir].String() @@ -299,3 +310,44 @@ func TestMirrorDebianArchitectureAllOnAPortsArchive(t *testing.T) { t.Errorf("binary-amd64 read %d times, want none", got) } } + +// A package in a component the archive's Release lists, such as the stable +// of Docker's apt repository, is found there. +func TestMirrorDebianReadsTheComponentsTheReleaseLists(t *testing.T) { + upstream := newDebUpstream(t) + upstream.components["/docker"] = "stable" + upstream.addSuite("/docker", "bookworm", "bookworm", + debPackage{"docker-ce", "5:27.3.1-1~debian.12~bookworm", "amd64", "pool/stable/d/docker-ce/docker-ce_27.3.1-1~debian.12~bookworm_amd64.deb", "docker-ce"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"docker": upstream.URL + "/docker"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/docker/docker-ce@5:27.3.1-1~debian.12~bookworm?arch=amd64&distro=debian-12") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/docker/dists/bookworm/main/binary-amd64/Packages"); got != 0 { + t.Errorf("main read %d times, want none: the Release doesn't list it", got) + } +} + +// The Release of a suite is read once for the suites, components and the +// architecture of Architecture: all packages, and once more by the index +// reader that checks the indexes against it. +func TestMirrorDebianReadsEachReleaseOnce(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/ports"] = []string{"arm64", "armhf"} + upstream.addSuite("/ports", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/ports/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/dists/jammy/Release"); got != 2 { + t.Errorf("Release read %d times, want 2", got) + } +} From 82e5cf7f63b5620270a56fafd4a09d525dd10e6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 17 Nov 2026 18:32:29 +0100 Subject: [PATCH 4/4] Give every Debian suite an index the registries client can read The client needs an index in each suite it is given. Read the usual components for a suite whose Release lists none, even when another suite lists its own, and read Architecture: all packages from the architecture each suite's Release picks rather than the first suite's alone. --- internal/mirror/deb.go | 46 ++++++++++++--------- internal/mirror/deb_test.go | 22 +++++----- internal/server/mirror_debian_test.go | 58 +++++++++++++++++++++------ 3 files changed, 85 insertions(+), 41 deletions(-) diff --git a/internal/mirror/deb.go b/internal/mirror/deb.go index c5fb1012..1861616e 100644 --- a/internal/mirror/deb.go +++ b/internal/mirror/deb.go @@ -65,12 +65,11 @@ type debIndexes struct { } // debReleaseInfo is what the Releases of a release's suites give: the suites -// the archive serves, the components they list, and the architecture whose -// index is read for Architecture: all packages. +// the archive serves, the components they list, and the architectures whose +// indexes are read for Architecture: all packages, one for each suite. type debReleaseInfo struct { - suites, components []string - allArch string - fetched time.Time + suites, components, allArchs []string + fetched time.Time } type debIndex struct { @@ -175,7 +174,7 @@ func (m *Mirror) debArtifact(ctx context.Context, archiveURL, codename, arch, na // Architecture: all packages are in every binary- index. indexArch := arch if arch == "all" { - indexArch = rel.allArch + indexArch = strings.Join(rel.allArchs, ",") } reg, err := m.debRegistry(archiveURL, codename, indexArch, rel) if err != nil { @@ -228,9 +227,10 @@ func debEpoch(epoch string) string { // debReleaseInfo returns what the Releases of the suites of release codename // give, reading each suite's once per debIndexTTL: the suites the archive -// serves, the components they list, or debComponents where none lists any, -// and the architecture for Architecture: all packages, from the first -// served suite's Release. +// serves, the components they list, with debComponents for a suite that +// lists none, and the architecture for Architecture: all packages of each. +// The registries client needs an index of every suite among those it is +// given. func (m *Mirror) debReleaseInfo(ctx context.Context, archiveURL, codename string) (debReleaseInfo, error) { key := archiveURL + "\x00" + codename m.deb.mu.Lock() @@ -250,22 +250,17 @@ func (m *Mirror) debReleaseInfo(ctx context.Context, archiveURL, codename string if !served { continue } - if rel.suites == nil { - rel.allArch = debReleaseAllArch(body) - } rel.suites = append(rel.suites, suite) - for _, c := range debReleaseComponents(body) { - if !slices.Contains(rel.components, c) { - rel.components = append(rel.components, c) - } + components := debReleaseComponents(body) + if components == nil { + components = debComponents } + rel.components = appendNew(rel.components, components...) + rel.allArchs = appendNew(rel.allArchs, debReleaseAllArch(body)) } if len(rel.suites) == 0 { return debReleaseInfo{}, fmt.Errorf("%s serves no suite of %s", archiveURL, codename) } - if rel.components == nil { - rel.components = debComponents - } rel.fetched = time.Now() if m.deb.releases == nil { m.deb.releases = map[string]debReleaseInfo{} @@ -319,13 +314,24 @@ func debReleaseAllArch(release string) string { return "amd64" } +// appendNew appends the items of add that list doesn't hold yet. +func appendNew(list []string, add ...string) []string { + for _, item := range add { + if !slices.Contains(list, item) { + list = append(list, item) + } + } + return list +} + func debIndexKey(archiveURL, codename, arch string) string { return archiveURL + "\x00" + codename + "\x00" + arch } // debRegistry returns the client for the binary-arch indexes of the // components of the suites of release codename that rel names, reusing it, -// and the indexes it read, for debIndexTTL. +// and the indexes it read, for debIndexTTL. arch may list several +// architectures, separated by commas. func (m *Mirror) debRegistry(archiveURL, codename, arch string, rel debReleaseInfo) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface key := debIndexKey(archiveURL, codename, arch) m.deb.mu.Lock() diff --git a/internal/mirror/deb_test.go b/internal/mirror/deb_test.go index 8f233db0..cc4f0a1a 100644 --- a/internal/mirror/deb_test.go +++ b/internal/mirror/deb_test.go @@ -93,11 +93,11 @@ func TestDebReleaseIsReadOncePerTTL(t *testing.T) { t.Cleanup(srv.Close) m := &Mirror{} ctx := context.Background() - want := debReleaseInfo{suites: []string{"jammy"}, components: []string{"stable"}, allArch: "arm64"} + want := debReleaseInfo{suites: []string{"jammy"}, components: []string{"stable"}, allArchs: []string{"arm64"}} for range 2 { rel, err := m.debReleaseInfo(ctx, srv.URL, "jammy") - if err != nil || !slices.Equal(rel.suites, want.suites) || !slices.Equal(rel.components, want.components) || rel.allArch != want.allArch { + if err != nil || !slices.Equal(rel.suites, want.suites) || !slices.Equal(rel.components, want.components) || !slices.Equal(rel.allArchs, want.allArchs) { t.Fatalf("debReleaseInfo = %+v, %v; want %+v", rel, err, want) } } @@ -128,17 +128,20 @@ func TestDebReleaseDefaults(t *testing.T) { })) t.Cleanup(srv.Close) rel, err := (&Mirror{}).debReleaseInfo(context.Background(), srv.URL, "jammy") - if err != nil || !slices.Equal(rel.components, debComponents) || rel.allArch != "amd64" { + if err != nil || !slices.Equal(rel.components, debComponents) || !slices.Equal(rel.allArchs, []string{"amd64"}) { t.Errorf("debReleaseInfo = %+v, %v; want the default components and amd64", rel, err) } } -// The components of all served suites are read, each once, and the -// architecture for Architecture: all packages is the release's own. +// The components of all served suites are read, each once, with the usual +// ones for a suite that lists none, and the architectures for Architecture: +// all packages are each suite's: the registries client needs an index in +// every suite. func TestDebReleaseOfSeveralSuites(t *testing.T) { releases := map[string]string{ - "/dists/jammy/InRelease": "Architectures: arm64 armhf\nComponents: stable\n", - "/dists/jammy-updates/InRelease": "Architectures: armhf\nComponents: stable test\n", + "/dists/jammy/InRelease": "Architectures: arm64 armhf\nComponents: stable\n", + "/dists/jammy-updates/InRelease": "Architectures: armhf\nComponents: stable test\n", + "/dists/jammy-security/InRelease": "Architectures: arm64\n", } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, ok := releases[r.URL.Path] @@ -153,8 +156,9 @@ func TestDebReleaseOfSeveralSuites(t *testing.T) { if err != nil { t.Fatal(err) } - if !slices.Equal(rel.suites, []string{"jammy", "jammy-updates"}) || !slices.Equal(rel.components, []string{"stable", "test"}) || rel.allArch != "arm64" { - t.Errorf("debReleaseInfo = %+v; want jammy and jammy-updates, stable and test, arm64", rel) + wantComponents := append([]string{"stable", "test"}, debComponents...) + if !slices.Equal(rel.suites, []string{"jammy", "jammy-updates", "jammy-security"}) || !slices.Equal(rel.components, wantComponents) || !slices.Equal(rel.allArchs, []string{"arm64", "armhf"}) { + t.Errorf("debReleaseInfo = %+v; want the three suites, components %v, arm64 and armhf", rel, wantComponents) } } diff --git a/internal/server/mirror_debian_test.go b/internal/server/mirror_debian_test.go index c9fdfc1d..fa80a4c2 100644 --- a/internal/server/mirror_debian_test.go +++ b/internal/server/mirror_debian_test.go @@ -30,13 +30,14 @@ type debUpstream struct { hits map[string]int // fail answers the paths it holds with that status. fail map[string]int - // architectures, by archive root, are the architectures its Releases - // list, the first of which holds Architecture: all packages. An - // archive not in it lists none and holds them in binary-amd64. + // architectures, by archive root, or by root and suite as + // "{root}#{suite}", are the architectures its Releases list, the first + // of which holds Architecture: all packages. An archive not in it lists + // none and holds them in binary-amd64. architectures map[string][]string - // components, by archive root, is the component its packages are in, - // which its Releases list. An archive not in it keeps them in main and - // lists no components. + // components, keyed as architectures, is the component its packages + // are in, which its Releases list. An archive not in it keeps them in + // main and lists no components. components map[string]string } @@ -71,18 +72,26 @@ func sha256Hex(s string) string { // addSuite serves suite of the archive under root, with codename in its // Release and pkgs in component main for each package's architecture. func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) { + archs, ok := u.architectures[root+"#"+suite] + if !ok { + archs = u.architectures[root] + } + comp, ok := u.components[root+"#"+suite] + if !ok { + comp = u.components[root] + } indexes := map[string]*strings.Builder{} for _, p := range pkgs { arch := p.arch if arch == "all" { arch = "amd64" - if archs := u.architectures[root]; len(archs) > 0 { + if len(archs) > 0 { arch = archs[0] } } component := "main" - if c := u.components[root]; c != "" { - component = c + if comp != "" { + component = comp } dir := component + "/binary-" + arch if indexes[dir] == nil { @@ -98,11 +107,11 @@ func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) } sort.Strings(dirs) release := "Codename: " + codename + "\n" - if archs := u.architectures[root]; len(archs) > 0 { + if len(archs) > 0 { release += "Architectures: " + strings.Join(archs, " ") + "\n" } - if c := u.components[root]; c != "" { - release += "Components: " + c + "\n" + if comp != "" { + release += "Components: " + comp + "\n" } release += "SHA256:\n" for _, dir := range dirs { @@ -351,3 +360,28 @@ func TestMirrorDebianReadsEachReleaseOnce(t *testing.T) { t.Errorf("Release read %d times, want 2", got) } } + +// Suites of a release that list other components, or none, are each read +// for theirs, and an Architecture: all package is found where the suites +// hold such packages in the indexes of different architectures. +func TestMirrorDebianSuitesThatDiffer(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/vendor#jammy"] = []string{"arm64", "armhf"} + upstream.architectures["/vendor#jammy-updates"] = []string{"armhf"} + upstream.components["/vendor#jammy-updates"] = "stable" + upstream.addSuite("/vendor", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + upstream.addSuite("/vendor", "jammy-updates", "jammy", + debPackage{"agent", "1.2-1", "all", "pool/stable/a/agent/agent_1.2-1_all.deb", "agent"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"vendor": upstream.URL + "/vendor"} + + progress, _ := mirrorWithConfig(t, cfg, + "pkg:deb/vendor/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04", + "pkg:deb/vendor/agent@1.2-1?arch=all&distro=ubuntu-22.04") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +}