diff --git a/docs/configuration.md b/docs/configuration.md index cb70bf5a..bb1ea5fb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2006,6 +2006,8 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +A Debian package is mirrored from the archive the namespace names: the one in `upstream.debian_repositories` by that name, or else `upstream.debian` for `debian` or no namespace, as syft and trivy write it in image SBOMs: `pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12`. The `distro` qualifier gives the release, as `debian-12`, a point release such as `debian-12.4`, `ubuntu-22.04` or a codename such as `bookworm`, and `arch` the architecture; packages without either are refused. The package is looked up in the release's suites the archive serves, of `{codename}`, `{codename}-updates` and `{codename}-security`, through the `Packages` indexes of the components their `Release` lists, such as `stable` in Docker's apt repository, which are checked against the `Release`. A suite whose `Release` lists none is read for `main`, `contrib`, `non-free`, `non-free-firmware`, `restricted`, `universe` and `multiverse`. Debian keeps `{codename}-security` in an archive of its own, `deb.debian.org/debian-security`, which can be added to `upstream.debian_repositories` and named in the namespace. An epoch is matched from the version (`1:9.2p1-2+deb12u3`) or the `epoch` qualifier; without one, a version listed with two epochs is refused. The file is checked against the index's SHA256 and cached under the path `/debian/` serves it from, so apt gets it with the upstream down. The indexes aren't mirrored. Give a version: an archive keeps only the current one of each suite. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/go.mod b/go.mod index c3409e24..c552f50e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 - github.com/git-pkgs/registries v0.13.2 + github.com/git-pkgs/registries v0.14.0 github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 @@ -75,7 +75,7 @@ require ( github.com/dustin/go-humanize v1.1.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/pom v0.1.9 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.8.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index 4a2270eb..93e42071 100644 --- a/go.sum +++ b/go.sum @@ -131,12 +131,12 @@ github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZ github.com/git-pkgs/integrity v0.2.0/go.mod h1:vPLb2RB+hR/iCMIgqph0PZbQ7iktXDTK+uupoHQikKk= github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= -github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/pom v0.1.9 h1:gP7nqAS+UXITeRXjHhpyrMHRl9K3sRRBzRLV3+TeqHU= +github.com/git-pkgs/pom v0.1.9/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.13.2 h1:3fWinubUZbBEJ+kDii3O5kzqHJMJBmJK1X9FefAow+I= -github.com/git-pkgs/registries v0.13.2/go.mod h1:UlX6jcAd4ftRBVVNzB4mQg32tfSNCk/L2pO4Wtplnho= +github.com/git-pkgs/registries v0.14.0 h1:PQbfRFghM6cN2xu1FcDHS2F73yOMATVIBaq+zY/6uSI= +github.com/git-pkgs/registries v0.14.0/go.mod h1:zgtz1lneDi8olIzAJxnSvSMT7PzChp9dsbFxJVbfyfQ= github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.6 h1:tbkqeYLKgAcDV1qPZNAkNxpNMTx9EfoNQQhcVEHApzs= diff --git a/internal/handler/debian.go b/internal/handler/debian.go index cb68afdd..28b9e50a 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -58,12 +58,9 @@ func NewDebianHandler( upstreamURL string, repositories map[string]string, ) *DebianHandler { - if upstreamURL == "" { - upstreamURL = debianUpstream - } h := &DebianHandler{ proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + upstreamURL: DebianUpstreamURL(upstreamURL), proxyURL: strings.TrimSuffix(proxyURL, "/"), repositories: make(map[string]string, len(repositories)), } @@ -238,11 +235,27 @@ func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL, return nil, "" } +// DebianUpstreamURL returns the upstream.debian archive, the default one when +// none is configured. +func DebianUpstreamURL(configured string) string { + if configured == "" { + configured = debianUpstream + } + return strings.TrimSuffix(configured, "/") +} + // artifactCacheFilename names a package in the cache by its repository and // its path in it, since the same file name can hold different bytes in // different archives, and in different directories of one archive: Docker's // repository has a pool under each suite. func (h *DebianHandler) artifactCacheFilename(repository, path string) string { + return DebianCacheFilename(repository, path) +} + +// DebianCacheFilename is the name the Debian handler caches the package at +// path in the upstream.debian_repositories archive repository under, or in +// the upstream.debian archive when repository is empty. +func DebianCacheFilename(repository, path string) string { if repository == "" { return path } diff --git a/internal/mirror/deb.go b/internal/mirror/deb.go new file mode 100644 index 00000000..1861616e --- /dev/null +++ b/internal/mirror/deb.go @@ -0,0 +1,385 @@ +package mirror + +import ( + "context" + "fmt" + "maps" + "net/url" + "regexp" + "slices" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// debIndexTTL is how long an archive's indexes are reused, so mirroring the +// packages of an image reads each index once. +const debIndexTTL = 5 * time.Minute + +var ( + // debReleasePattern takes the release from the distro qualifier syft and + // trivy write, such as debian-12, trivy's debian-12.4 or ubuntu-22.04. + debReleasePattern = regexp.MustCompile(`^(debian-\d+|ubuntu-\d+\.\d+)(?:\.\d+)?$`) + // debCodenamePattern matches a codename given as the distro qualifier. + debCodenamePattern = regexp.MustCompile(`^[a-z]+$`) + // debArchPattern matches an architecture, such as amd64 or the ports' + // hurd-amd64. + debArchPattern = regexp.MustCompile(`^[a-z0-9]+(?:-[a-z0-9]+)*$`) + // debComponentPattern matches the components read from a Release. + debComponentPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.+_-]*$`) + debEpochPattern = regexp.MustCompile(`^\d+$`) + + // debCodenames are the codenames of the Debian and Ubuntu releases, + // which their suites are named after. Other releases can be given by + // codename. + debCodenames = map[string]string{ + "debian-8": "jessie", "debian-9": "stretch", "debian-10": "buster", + "debian-11": "bullseye", "debian-12": "bookworm", "debian-13": "trixie", + "debian-14": "forky", + "ubuntu-16.04": "xenial", "ubuntu-18.04": "bionic", "ubuntu-20.04": "focal", + "ubuntu-22.04": "jammy", "ubuntu-24.04": "noble", "ubuntu-24.10": "oracular", + "ubuntu-25.04": "plucky", "ubuntu-25.10": "questing", + } + + // debSuffixes name the suites of a release that are read: the release, + // its updates and its security updates. An archive serves the ones it + // has; Debian keeps security updates in an archive of its own. + debSuffixes = []string{"", "-updates", "-security"} + + // debComponents are read where no Release of a release lists its + // components. + debComponents = []string{"main", "contrib", "non-free", "non-free-firmware", "restricted", "universe", "multiverse"} +) + +// debIndexes keeps one registries client per archive, release and +// architecture, which holds the parsed indexes. +type debIndexes struct { + mu sync.Mutex + regs map[string]debIndex + // releases are what the Releases of a release's suites give, by + // archive and release. + releases map[string]debReleaseInfo +} + +// debReleaseInfo is what the Releases of a release's suites give: the suites +// the archive serves, the components they list, and the architectures whose +// indexes are read for Architecture: all packages, one for each suite. +type debReleaseInfo struct { + suites, components, allArchs []string + fetched time.Time +} + +type debIndex struct { + reg registries.Registry + fetched time.Time +} + +// mirrorDeb mirrors a Debian package under the name the Debian handler +// caches it at, its path in the archive. The PURL namespace names the +// archive: an upstream.debian_repositories one by that name, or else +// upstream.debian for debian or none. The distro qualifier gives the release +// and arch the architecture; the package is looked up in the release's +// suites the archive serves. +func (m *Mirror) mirrorDeb(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + repository, archiveURL, err := m.debArchive(pv.Namespace) + if err != nil { + fail(err) + return + } + codename, err := debCodename(pv.Qualifiers["distro"]) + if err != nil { + fail(err) + return + } + arch := pv.Qualifiers["arch"] + if !debArchPattern.MatchString(arch) { + fail(fmt.Errorf("deb packages need an arch qualifier, such as arch=amd64, not %q", arch)) + return + } + epoch, version := debSplitEpoch(pv.Version) + if e := pv.Qualifiers["epoch"]; e != "" { + epoch = e + } + if epoch != "" && !debEpochPattern.MatchString(epoch) { + fail(fmt.Errorf("invalid epoch %q", epoch)) + return + } + + artifact, err := m.debArtifact(ctx, archiveURL, codename, arch, pv.Name, version, epoch) + if err != nil { + fail(err) + return + } + path := strings.TrimPrefix(artifact.URL, archiveURL+"/") + name, fileVersion, _ := handler.OSPackageIdentity("deb", path) + if name == "" { + fail(fmt.Errorf("%s isn't a package file the Debian handler serves", path)) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, fileVersion, + handler.DebianCacheFilename(repository, path), artifact.URL, fileDigest(artifact.Integrity)) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// debArchive returns the repository name and URL of the archive namespace +// names, with an empty name for upstream.debian. +func (m *Mirror) debArchive(namespace string) (string, string, error) { + if archiveURL, ok := m.DebianRepositories[namespace]; ok && namespace != "" { + return namespace, strings.TrimSuffix(archiveURL, "/"), nil + } + if namespace == "" || namespace == "debian" { + return "", handler.DebianUpstreamURL(m.DebianUpstream), nil + } + return "", "", fmt.Errorf("unknown Debian archive %q: it isn't in upstream.debian_repositories", namespace) +} + +// debCodename returns the codename of the release distro names. +func debCodename(distro string) (string, error) { + if match := debReleasePattern.FindStringSubmatch(distro); match != nil { + if codename, ok := debCodenames[match[1]]; ok { + return codename, nil + } + } else if debCodenamePattern.MatchString(distro) { + return distro, nil + } + return "", fmt.Errorf("deb packages need a distro qualifier with a known release, such as distro=debian-12, or its codename, such as distro=bookworm, not %q", distro) +} + +// debSplitEpoch splits the epoch off a Debian version, returning "" for +// one without. +func debSplitEpoch(version string) (epoch, rest string) { + if e, r, ok := strings.Cut(version, ":"); ok && debEpochPattern.MatchString(e) { + return e, r + } + return "", version +} + +// debArtifact returns the file of name for arch in the release codename of +// the archive whose version, without its epoch, is version. epoch, when +// given, picks the epoch; without it, versions that differ only in their +// epoch are refused rather than one picked. +func (m *Mirror) debArtifact(ctx context.Context, archiveURL, codename, arch, name, version, epoch string) (registries.Artifact, error) { + rel, err := m.debReleaseInfo(ctx, archiveURL, codename) + if err != nil { + return registries.Artifact{}, err + } + // Architecture: all packages are in every binary- index. + indexArch := arch + if arch == "all" { + indexArch = strings.Join(rel.allArchs, ",") + } + reg, err := m.debRegistry(archiveURL, codename, indexArch, rel) + if err != nil { + return registries.Artifact{}, err + } + notIn := fmt.Errorf("%s %s isn't in %s of %s for %s", name, version, strings.Join(rel.suites, ", "), archiveURL, arch) + versions, err := reg.FetchVersions(ctx, name) + if isNotFound(err) { + return registries.Artifact{}, notIn + } + if err != nil { + m.deb.forget(debIndexKey(archiveURL, codename, indexArch)) + return registries.Artifact{}, fmt.Errorf("reading the indexes of %s: %w", archiveURL, err) + } + var found registries.Artifact + var epochs []string + for _, v := range versions { + e, rest := debSplitEpoch(v.Number) + if rest != version || (epoch != "" && debEpoch(e) != debEpoch(epoch)) { + continue + } + // The indexes are those of codename alone. + for _, a := range v.Artifacts { + if a.Metadata["arch"] == arch || a.Metadata["arch"] == "all" { + if !slices.Contains(epochs, debEpoch(e)) { + epochs = append(epochs, debEpoch(e)) + } + found = a + break + } + } + } + switch { + case len(epochs) == 0: + return registries.Artifact{}, notIn + case len(epochs) > 1: + return registries.Artifact{}, fmt.Errorf("%s %s has epochs %s in %s; give one in the version or the epoch qualifier", + name, version, strings.Join(epochs, " and "), archiveURL) + } + return found, nil +} + +// debEpoch is epoch with the implied 0 of a version without one. +func debEpoch(epoch string) string { + if epoch == "" { + return "0" + } + return epoch +} + +// debReleaseInfo returns what the Releases of the suites of release codename +// give, reading each suite's once per debIndexTTL: the suites the archive +// serves, the components they list, with debComponents for a suite that +// lists none, and the architecture for Architecture: all packages of each. +// The registries client needs an index of every suite among those it is +// given. +func (m *Mirror) debReleaseInfo(ctx context.Context, archiveURL, codename string) (debReleaseInfo, error) { + key := archiveURL + "\x00" + codename + m.deb.mu.Lock() + defer m.deb.mu.Unlock() + if rel, ok := m.deb.releases[key]; ok && time.Since(rel.fetched) < debIndexTTL { + return rel, nil + } + r, _ := m.Versions.(RegistryVersions) + client := r.client() + var rel debReleaseInfo + for _, suffix := range debSuffixes { + suite := codename + suffix + body, served, err := debReleaseFile(ctx, client, archiveURL, suite) + if err != nil { + return debReleaseInfo{}, fmt.Errorf("checking %s for suite %s: %w", archiveURL, suite, err) + } + if !served { + continue + } + rel.suites = append(rel.suites, suite) + components := debReleaseComponents(body) + if components == nil { + components = debComponents + } + rel.components = appendNew(rel.components, components...) + rel.allArchs = appendNew(rel.allArchs, debReleaseAllArch(body)) + } + if len(rel.suites) == 0 { + return debReleaseInfo{}, fmt.Errorf("%s serves no suite of %s", archiveURL, codename) + } + rel.fetched = time.Now() + if m.deb.releases == nil { + m.deb.releases = map[string]debReleaseInfo{} + } + maps.DeleteFunc(m.deb.releases, func(_ string, rel debReleaseInfo) bool { return time.Since(rel.fetched) >= debIndexTTL }) + m.deb.releases[key] = rel + return rel, nil +} + +// debReleaseComponents returns the components a Release lists. A +// component below a directory, as debian-security lists updates/main, is +// read where its last part names it, as apt reads it. +func debReleaseComponents(release string) []string { + var components []string + for line := range strings.Lines(release) { + value, ok := strings.CutPrefix(line, "Components:") + if !ok { + continue + } + for _, c := range strings.Fields(value) { + parts := strings.Split(c, "/") + if slices.ContainsFunc(parts, func(p string) bool { return !debComponentPattern.MatchString(p) }) { + continue + } + if c = parts[len(parts)-1]; !slices.Contains(components, c) { + components = append(components, c) + } + } + } + return components +} + +// debReleaseAllArch picks the architecture for Architecture: all packages +// from a Release's Architectures field. +func debReleaseAllArch(release string) string { + for line := range strings.Lines(release) { + value, ok := strings.CutPrefix(line, "Architectures:") + if !ok { + continue + } + var listed []string + for _, a := range strings.Fields(value) { + if a != "all" && debArchPattern.MatchString(a) { + listed = append(listed, a) + } + } + if len(listed) > 0 && !slices.Contains(listed, "amd64") { + return listed[0] + } + } + return "amd64" +} + +// appendNew appends the items of add that list doesn't hold yet. +func appendNew(list []string, add ...string) []string { + for _, item := range add { + if !slices.Contains(list, item) { + list = append(list, item) + } + } + return list +} + +func debIndexKey(archiveURL, codename, arch string) string { + return archiveURL + "\x00" + codename + "\x00" + arch +} + +// debRegistry returns the client for the binary-arch indexes of the +// components of the suites of release codename that rel names, reusing it, +// and the indexes it read, for debIndexTTL. arch may list several +// architectures, separated by commas. +func (m *Mirror) debRegistry(archiveURL, codename, arch string, rel debReleaseInfo) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface + key := debIndexKey(archiveURL, codename, arch) + m.deb.mu.Lock() + defer m.deb.mu.Unlock() + if idx, ok := m.deb.regs[key]; ok && time.Since(idx.fetched) < debIndexTTL { + return idx.reg, nil + } + r, _ := m.Versions.(RegistryVersions) + query := url.Values{ + "suites": {strings.Join(rel.suites, ",")}, + "components": {strings.Join(rel.components, ",")}, + "architectures": {arch}, + } + reg, err := registries.New("deb", archiveURL+"?"+query.Encode(), r.client()) + if err != nil { + return nil, err + } + if m.deb.regs == nil { + m.deb.regs = map[string]debIndex{} + } + // Indexes no longer asked for are dropped, rather than kept for the + // Mirror's life. + maps.DeleteFunc(m.deb.regs, func(_ string, idx debIndex) bool { return time.Since(idx.fetched) >= debIndexTTL }) + m.deb.regs[key] = debIndex{reg: reg, fetched: time.Now()} + return reg, nil +} + +// debReleaseFile returns the InRelease or Release of suite, and whether +// the archive serves one. An answer other than the file or 404 is an error, +// so a failing archive doesn't leave a suite out for debIndexTTL. +func debReleaseFile(ctx context.Context, client *registries.Client, archiveURL, suite string) (string, bool, error) { + for _, file := range []string{"InRelease", "Release"} { + body, err := client.GetBody(ctx, archiveURL+"/dists/"+suite+"/"+file) + if isNotFound(err) { + continue + } + if err != nil { + return "", false, err + } + return string(body), true, nil + } + return "", false, nil +} + +// forget drops the client for key, so indexes that failed to load are read +// again next time. +func (d *debIndexes) forget(key string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.regs, key) +} diff --git a/internal/mirror/deb_test.go b/internal/mirror/deb_test.go new file mode 100644 index 00000000..cc4f0a1a --- /dev/null +++ b/internal/mirror/deb_test.go @@ -0,0 +1,190 @@ +package mirror + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "slices" + "sync/atomic" + "testing" + "time" +) + +// Loading an archive's indexes drops those past debIndexTTL, so indexes no +// longer asked for aren't kept for the Mirror's life. +func TestDebRegistryDropsExpiredIndexes(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(srv.Close) + m := &Mirror{} + rel := debReleaseInfo{suites: []string{"bookworm"}, components: []string{"main"}} + + if _, err := m.debRegistry(srv.URL+"/old", "bookworm", "amd64", rel); err != nil { + t.Fatal(err) + } + if _, err := m.debRegistry(srv.URL+"/fresh", "bookworm", "amd64", rel); err != nil { + t.Fatal(err) + } + old := debIndexKey(srv.URL+"/old", "bookworm", "amd64") + idx := m.deb.regs[old] + idx.fetched = time.Now().Add(-debIndexTTL) + m.deb.regs[old] = idx + + if _, err := m.debRegistry(srv.URL+"/new", "bookworm", "amd64", rel); err != nil { + t.Fatal(err) + } + if _, ok := m.deb.regs[old]; ok { + t.Error("expired index kept") + } + if _, ok := m.deb.regs[debIndexKey(srv.URL+"/fresh", "bookworm", "amd64")]; !ok { + t.Error("index within its TTL dropped") + } +} + +func TestDebReleaseAllArch(t *testing.T) { + tests := map[string]string{ + "Codename: jammy\n": "amd64", + "Architectures: amd64 arm64 i386\n": "amd64", + "Architectures: arm64 amd64\n": "amd64", + "Codename: jammy\nArchitectures: all arm64 armhf\n": "arm64", + "Architectures: armhf\nComponents: main\n": "armhf", + "Architectures: all\n": "amd64", + "Architectures: ../x arm64\n": "arm64", + } + for release, want := range tests { + if got := debReleaseAllArch(release); got != want { + t.Errorf("debReleaseAllArch(%q) = %q, want %q", release, got, want) + } + } +} + +func TestDebReleaseComponents(t *testing.T) { + tests := map[string][]string{ + "Codename: bookworm\n": nil, + "Components: main contrib non-free-firmware non-free\n": {"main", "contrib", "non-free-firmware", "non-free"}, + "Components: stable edge test nightly\n": {"stable", "edge", "test", "nightly"}, + // debian-security names its components below updates/, and keeps + // their indexes where the last part of that names them, as apt + // reads them. + "Components: updates/main updates/contrib updates/main\n": {"main", "contrib"}, + "Components: ../x . main%2f\n": nil, + "Codename: bookworm\nComponents: main\nArchitectures: all\n": {"main"}, + } + for release, want := range tests { + if got := debReleaseComponents(release); !slices.Equal(got, want) { + t.Errorf("debReleaseComponents(%q) = %q, want %q", release, got, want) + } + } +} + +// An archive's Releases are read once per debIndexTTL, for the suites it +// serves, their components and the architecture for Architecture: all +// packages. +func TestDebReleaseIsReadOncePerTTL(t *testing.T) { + var reads atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/jammy/InRelease" { + http.NotFound(w, r) + return + } + reads.Add(1) + _, _ = io.WriteString(w, "Architectures: arm64\nComponents: stable\n") + })) + t.Cleanup(srv.Close) + m := &Mirror{} + ctx := context.Background() + want := debReleaseInfo{suites: []string{"jammy"}, components: []string{"stable"}, allArchs: []string{"arm64"}} + + for range 2 { + rel, err := m.debReleaseInfo(ctx, srv.URL, "jammy") + if err != nil || !slices.Equal(rel.suites, want.suites) || !slices.Equal(rel.components, want.components) || !slices.Equal(rel.allArchs, want.allArchs) { + t.Fatalf("debReleaseInfo = %+v, %v; want %+v", rel, err, want) + } + } + if got := reads.Load(); got != 1 { + t.Errorf("InRelease read %d times, want once", got) + } + key := srv.URL + "\x00jammy" + rel := m.deb.releases[key] + rel.fetched = time.Now().Add(-debIndexTTL) + m.deb.releases[key] = rel + if _, err := m.debReleaseInfo(ctx, srv.URL, "jammy"); err != nil { + t.Fatal(err) + } + if got := reads.Load(); got != 2 { + t.Errorf("InRelease read %d times after the TTL, want twice", got) + } +} + +// A suite whose Release lists no components is read for the usual ones, +// and one without amd64 among its architectures for the first it lists. +func TestDebReleaseDefaults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/jammy/Release" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "Codename: jammy\n") + })) + t.Cleanup(srv.Close) + rel, err := (&Mirror{}).debReleaseInfo(context.Background(), srv.URL, "jammy") + if err != nil || !slices.Equal(rel.components, debComponents) || !slices.Equal(rel.allArchs, []string{"amd64"}) { + t.Errorf("debReleaseInfo = %+v, %v; want the default components and amd64", rel, err) + } +} + +// The components of all served suites are read, each once, with the usual +// ones for a suite that lists none, and the architectures for Architecture: +// all packages are each suite's: the registries client needs an index in +// every suite. +func TestDebReleaseOfSeveralSuites(t *testing.T) { + releases := map[string]string{ + "/dists/jammy/InRelease": "Architectures: arm64 armhf\nComponents: stable\n", + "/dists/jammy-updates/InRelease": "Architectures: armhf\nComponents: stable test\n", + "/dists/jammy-security/InRelease": "Architectures: arm64\n", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := releases[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + rel, err := (&Mirror{}).debReleaseInfo(context.Background(), srv.URL, "jammy") + if err != nil { + t.Fatal(err) + } + wantComponents := append([]string{"stable", "test"}, debComponents...) + if !slices.Equal(rel.suites, []string{"jammy", "jammy-updates", "jammy-security"}) || !slices.Equal(rel.components, wantComponents) || !slices.Equal(rel.allArchs, []string{"arm64", "armhf"}) { + t.Errorf("debReleaseInfo = %+v; want the three suites, components %v, arm64 and armhf", rel, wantComponents) + } +} + +// Reading a release's Releases drops those read past debIndexTTL. +func TestDebReleaseInfoDropsExpiredEntries(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(srv.Close) + m := &Mirror{} + ctx := context.Background() + for _, root := range []string{"/old", "/fresh"} { + if _, err := m.debReleaseInfo(ctx, srv.URL+root, "bookworm"); err != nil { + t.Fatal(err) + } + } + old := srv.URL + "/old\x00bookworm" + rel := m.deb.releases[old] + rel.fetched = time.Now().Add(-debIndexTTL) + m.deb.releases[old] = rel + + if _, err := m.debReleaseInfo(ctx, srv.URL+"/new", "bookworm"); err != nil { + t.Fatal(err) + } + if _, ok := m.deb.releases[old]; ok { + t.Error("expired entry kept") + } + if _, ok := m.deb.releases[srv.URL+"/fresh\x00bookworm"]; !ok { + t.Error("entry within its TTL dropped") + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 1dded24d..7f0d46ba 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -53,12 +53,19 @@ type Mirror struct { // the official Alpine mirror as "alpine". APKRepositories map[string]string + // DebianUpstream and DebianRepositories are upstream.debian and + // upstream.debian_repositories. An empty DebianUpstream uses + // deb.debian.org. + DebianUpstream string + DebianRepositories map[string]string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex apk apkIndexes + deb debIndexes } // New creates a new Mirror with the given dependencies. @@ -297,6 +304,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "deb" { + m.mirrorDeb(ctx, pv, tracker) + return + } if pv.Ecosystem == "git" { m.mirrorGit(ctx, pv, tracker) return diff --git a/internal/mirror/source.go b/internal/mirror/source.go index bbfddf0b..bc41f47d 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,8 +16,9 @@ type PackageVersion struct { Ecosystem string Name string Version string - // Namespace is the PURL namespace of apk packages, which Name leaves - // out. It picks the upstream.apk repository. + // Namespace is the PURL namespace of apk and deb packages, which Name + // leaves out. It picks the upstream.apk repository, or the Debian + // archive. Namespace string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select @@ -72,7 +73,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro ecosystem, name, qualifiers := mirrorTarget(p) var namespace string - if ecosystem == "alpine" { + if ecosystem == "alpine" || ecosystem == "deb" { namespace = p.Namespace } @@ -151,6 +152,8 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} case "alpine": qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} + case "deb": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro"), "epoch": p.Qualifier("epoch")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_debian_test.go b/internal/server/mirror_debian_test.go new file mode 100644 index 00000000..fa80a4c2 --- /dev/null +++ b/internal/server/mirror_debian_test.go @@ -0,0 +1,387 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sort" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +// debPackage is one stanza of a Packages index, with the pool file it names. +type debPackage struct { + name, version, arch, filename, body string +} + +// debUpstream serves Debian archives: the main one at its root and named +// ones under /{name}. It counts requests by path. +type debUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int + // fail answers the paths it holds with that status. + fail map[string]int + // architectures, by archive root, or by root and suite as + // "{root}#{suite}", are the architectures its Releases list, the first + // of which holds Architecture: all packages. An archive not in it lists + // none and holds them in binary-amd64. + architectures map[string][]string + // components, keyed as architectures, is the component its packages + // are in, which its Releases list. An archive not in it keeps them in + // main and lists no components. + components map[string]string +} + +func newDebUpstream(t *testing.T) *debUpstream { + t.Helper() + u := &debUpstream{files: map[string]string{}, hits: map[string]int{}, fail: map[string]int{}, architectures: map[string][]string{}, components: map[string]string{}} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + status := u.fail[r.URL.Path] + u.mu.Unlock() + if status != 0 { + w.WriteHeader(status) + return + } + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// addSuite serves suite of the archive under root, with codename in its +// Release and pkgs in component main for each package's architecture. +func (u *debUpstream) addSuite(root, suite, codename string, pkgs ...debPackage) { + archs, ok := u.architectures[root+"#"+suite] + if !ok { + archs = u.architectures[root] + } + comp, ok := u.components[root+"#"+suite] + if !ok { + comp = u.components[root] + } + indexes := map[string]*strings.Builder{} + for _, p := range pkgs { + arch := p.arch + if arch == "all" { + arch = "amd64" + if len(archs) > 0 { + arch = archs[0] + } + } + component := "main" + if comp != "" { + component = comp + } + dir := component + "/binary-" + arch + if indexes[dir] == nil { + indexes[dir] = &strings.Builder{} + } + fmt.Fprintf(indexes[dir], "Package: %s\nVersion: %s\nArchitecture: %s\nFilename: %s\nSize: %d\nSHA256: %s\n\n", + p.name, p.version, p.arch, p.filename, len(p.body), sha256Hex(p.body)) + u.files[root+"/"+p.filename] = p.body + } + dirs := make([]string, 0, len(indexes)) + for dir := range indexes { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + release := "Codename: " + codename + "\n" + if len(archs) > 0 { + release += "Architectures: " + strings.Join(archs, " ") + "\n" + } + if comp != "" { + release += "Components: " + comp + "\n" + } + release += "SHA256:\n" + for _, dir := range dirs { + index := indexes[dir].String() + release += fmt.Sprintf(" %s %d %s/Packages\n", sha256Hex(index), len(index), dir) + u.files[root+"/dists/"+suite+"/"+dir+"/Packages"] = index + } + u.files[root+"/dists/"+suite+"/Release"] = release +} + +func (u *debUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +const ( + debMirrorCurlPath = "/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb" + debMirrorOpenSSHPath = "/pool/main/o/openssh/openssh-client_9.2p1-2+deb12u3_amd64.deb" + debMirrorTzdataPath = "/pool/main/t/tzdata/tzdata_2024a-0+deb12u1_all.deb" + debMirrorZlibPath = "/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" +) + +// newDebianArchives serves a Debian archive with bookworm and +// bookworm-updates, without bookworm-security, which Debian keeps in an +// archive of its own, and an Ubuntu one with jammy under /ubuntu. +func newDebianArchives(t *testing.T) *debUpstream { + t.Helper() + u := newDebUpstream(t) + u.addSuite("", "bookworm", "bookworm", + debPackage{"curl", "7.88.1-10+deb12u5", "amd64", "pool/main/c/curl/curl_7.88.1-10+deb12u5_amd64.deb", "curl old"}, + debPackage{"openssh-client", "1:9.2p1-2+deb12u3", "amd64", strings.TrimPrefix(debMirrorOpenSSHPath, "/"), "openssh-client"}, + debPackage{"tzdata", "2024a-0+deb12u1", "all", strings.TrimPrefix(debMirrorTzdataPath, "/"), "tzdata"}, + debPackage{"dual", "1:2.0-1", "amd64", "pool/main/d/dual/dual_2.0-1_amd64.deb", "dual one"}, + debPackage{"dual", "2:2.0-1", "amd64", "pool/main/d/dual/dual_2.0-1+e2_amd64.deb", "dual two"}, + debPackage{"odd", "1.0-1", "amd64", "pool/main/o/odd/odd.deb", "odd"}, + ) + u.addSuite("", "bookworm-updates", "bookworm", + debPackage{"curl", "7.88.1-10+deb12u8", "amd64", strings.TrimPrefix(debMirrorCurlPath, "/"), "curl new"}, + ) + u.addSuite("/ubuntu", "jammy", "jammy", + debPackage{"zlib1g", "1:1.2.11.dfsg-2ubuntu9.2", "amd64", strings.TrimPrefix(debMirrorZlibPath, "/"), "zlib1g"}, + ) + return u +} + +func debMirrorConfig(upstream *debUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Debian = upstream.URL + cfg.Upstream.DebianRepositories = map[string]string{"ubuntu": upstream.URL + "/ubuntu"} + return cfg +} + +func getDeb(proxy *handler.Proxy, upstream *debUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewDebianHandler(proxy, "http://proxy.test", upstream.URL, map[string]string{"ubuntu": upstream.URL + "/ubuntu"}) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestMirrorDebianPackagesAreServedFromCache(t *testing.T) { + upstream := newDebianArchives(t) + + progress, proxy := mirrorWithConfig(t, debMirrorConfig(upstream), + // syft: the release as distro, the epoch in the version. + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12", + "pkg:deb/debian/openssh-client@1%3A9.2p1-2+deb12u3?arch=amd64&distro=debian-12", + // trivy: the point release, and the epoch as a qualifier. + "pkg:deb/ubuntu/zlib1g@1.2.11.dfsg-2ubuntu9.2?arch=amd64&distro=ubuntu-22.04&epoch=1", + "pkg:deb/debian/curl@7.88.1-10+deb12u5?arch=amd64&distro=debian-12.4", + // A codename, and an Architecture: all package. + "pkg:deb/debian/tzdata@2024a-0+deb12u1?arch=all&distro=bookworm") + + if progress.Completed != 5 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{debMirrorCurlPath, debMirrorOpenSSHPath, debMirrorTzdataPath, "/ubuntu" + debMirrorZlibPath, "/dists/bookworm/main/binary-amd64/Packages"} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // apt gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + debMirrorCurlPath: "curl new", + debMirrorOpenSSHPath: "openssh-client", + debMirrorTzdataPath: "tzdata", + "/ubuntu" + debMirrorZlibPath: "zlib1g", + } { + if rec := getDeb(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorDebianRefusesWhatItCantFind(t *testing.T) { + upstream := newDebianArchives(t) + + tests := map[string]string{ + "pkg:deb/debian/curl@7.88.1-10+deb12u8?distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=../../etc&distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=-amd64&distro=debian-12": "arch", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64": "distro", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-99": "distro", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=Bookworm/../x": "distro", + "pkg:deb/other/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12": "upstream.debian_repositories", + "pkg:deb/debian/curl@9.9.9-1?arch=amd64&distro=debian-12": "isn't in", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=arm64&distro=debian-12": "no Debian Packages index", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-13": "trixie", + "pkg:deb/debian/openssh-client@9.2p1-2+deb12u3?arch=amd64&distro=debian-12&epoch=2": "isn't in", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12&epoch=x": "epoch", + "pkg:deb/debian/odd@1.0-1?arch=amd64&distro=debian-12": "isn't a package file", + "pkg:deb/debian/dual@2.0-1?arch=amd64&distro=debian-12": "epoch", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} + +// An epoch picks one of two versions that differ only in it, given in the +// version or as a qualifier. +func TestMirrorDebianEpochSelectsTheVersion(t *testing.T) { + upstream := newDebianArchives(t) + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/dual@2.0-1?arch=amd64&distro=debian-12&epoch=2", + "pkg:deb/debian/dual@1%3A2.0-1?arch=amd64&distro=debian-12") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{"/pool/main/d/dual/dual_2.0-1+e2_amd64.deb", "/pool/main/d/dual/dual_2.0-1_amd64.deb"} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } +} + +// An archive failing for a suite fails the package, rather than leaving the +// suite out of the indexes that are reused. +func TestMirrorDebianFailsOnAFailingArchive(t *testing.T) { + upstream := newDebianArchives(t) + upstream.fail["/dists/bookworm-security/InRelease"] = http.StatusInternalServerError + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "bookworm-security") { + t.Errorf("failed = %d, errors = %v; want one error naming bookworm-security", progress.Failed, progress.Errors) + } +} + +// An Architecture: all package is found for any architecture, and epoch=0 +// matches a version without an epoch, as trivy writes it. +func TestMirrorDebianMatchesArchitectureAllAndEpochZero(t *testing.T) { + upstream := newDebianArchives(t) + + progress, _ := mirrorWithConfig(t, debMirrorConfig(upstream), + "pkg:deb/debian/tzdata@2024a-0+deb12u1?arch=amd64&distro=debian-12", + "pkg:deb/debian/curl@7.88.1-10+deb12u8?arch=amd64&distro=debian-12&epoch=0") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} + +// An architecture name can have a hyphen, as Debian's ports have. +func TestMirrorDebianArchitectureWithHyphen(t *testing.T) { + upstream := newDebUpstream(t) + upstream.addSuite("/ports", "sid", "sid", + debPackage{"hello", "2.10-3", "hurd-amd64", "pool/main/h/hello/hello_2.10-3_hurd-amd64.deb", "hello"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/ports/hello@2.10-3?arch=hurd-amd64&distro=sid") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/pool/main/h/hello/hello_2.10-3_hurd-amd64.deb"); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +// An Architecture: all package is looked up in the index of an architecture +// the archive's Release lists, as a ports archive has no amd64. +func TestMirrorDebianArchitectureAllOnAPortsArchive(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/ports"] = []string{"arm64", "armhf"} + upstream.addSuite("/ports", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, + "pkg:deb/ports/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/dists/jammy/main/binary-amd64/Packages"); got != 0 { + t.Errorf("binary-amd64 read %d times, want none", got) + } +} + +// A package in a component the archive's Release lists, such as the stable +// of Docker's apt repository, is found there. +func TestMirrorDebianReadsTheComponentsTheReleaseLists(t *testing.T) { + upstream := newDebUpstream(t) + upstream.components["/docker"] = "stable" + upstream.addSuite("/docker", "bookworm", "bookworm", + debPackage{"docker-ce", "5:27.3.1-1~debian.12~bookworm", "amd64", "pool/stable/d/docker-ce/docker-ce_27.3.1-1~debian.12~bookworm_amd64.deb", "docker-ce"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"docker": upstream.URL + "/docker"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/docker/docker-ce@5:27.3.1-1~debian.12~bookworm?arch=amd64&distro=debian-12") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/docker/dists/bookworm/main/binary-amd64/Packages"); got != 0 { + t.Errorf("main read %d times, want none: the Release doesn't list it", got) + } +} + +// The Release of a suite is read once for the suites, components and the +// architecture of Architecture: all packages, and once more by the index +// reader that checks the indexes against it. +func TestMirrorDebianReadsEachReleaseOnce(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/ports"] = []string{"arm64", "armhf"} + upstream.addSuite("/ports", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"ports": upstream.URL + "/ports"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:deb/ports/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04") + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor("/ports/dists/jammy/Release"); got != 2 { + t.Errorf("Release read %d times, want 2", got) + } +} + +// Suites of a release that list other components, or none, are each read +// for theirs, and an Architecture: all package is found where the suites +// hold such packages in the indexes of different architectures. +func TestMirrorDebianSuitesThatDiffer(t *testing.T) { + upstream := newDebUpstream(t) + upstream.architectures["/vendor#jammy"] = []string{"arm64", "armhf"} + upstream.architectures["/vendor#jammy-updates"] = []string{"armhf"} + upstream.components["/vendor#jammy-updates"] = "stable" + upstream.addSuite("/vendor", "jammy", "jammy", + debPackage{"tzdata", "2024a-0ubuntu0.22.04", "all", "pool/main/t/tzdata/tzdata_2024a-0ubuntu0.22.04_all.deb", "tzdata"}, + ) + upstream.addSuite("/vendor", "jammy-updates", "jammy", + debPackage{"agent", "1.2-1", "all", "pool/stable/a/agent/agent_1.2-1_all.deb", "agent"}, + ) + cfg := config.Default() + cfg.Upstream.DebianRepositories = map[string]string{"vendor": upstream.URL + "/vendor"} + + progress, _ := mirrorWithConfig(t, cfg, + "pkg:deb/vendor/tzdata@2024a-0ubuntu0.22.04?arch=all&distro=ubuntu-22.04", + "pkg:deb/vendor/agent@1.2-1?arch=all&distro=ubuntu-22.04") + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 3e2cebb6..54d8c021 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -65,6 +65,8 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia m.APKRepositories = cfg.Upstream.APK + m.DebianUpstream = cfg.Upstream.Debian + m.DebianRepositories = cfg.Upstream.DebianRepositories if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 81491d95..e0518f0e 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -41,11 +41,14 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { // UseMirrorUpstreams points m at the configured upstreams: it lists // versions from them, through client, fetches Julia packages from -// upstream.julia and Alpine packages from upstream.apk. +// upstream.julia, Alpine packages from upstream.apk and Debian packages +// from upstream.debian and upstream.debian_repositories. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia m.APKRepositories = upstream.APK + m.DebianUpstream = upstream.Debian + m.DebianRepositories = upstream.DebianRepositories } // mirrorVersionsURL returns the registry base URL to list a package's