diff --git a/README.md b/README.md index 87db762..8c65094 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | VS Code extension | `pkg:vscode-extension/redhat/vscode-yaml@1.15.0` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | | Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` | +| RPM | `pkg:rpm/fedora/curl@8.15.0-4.fc43?arch=x86_64&distro=fedora-43&repository_url=https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os%3Fdistro%3Dfedora-43` | ## Direct Registry Usage @@ -199,6 +200,7 @@ import _ "github.com/git-pkgs/registries/all" | Swift | `swift` | URL required | | Alpine | `apk` | URL required | | Debian | `deb` | URL required | +| RPM | `rpm` | URL required | | GitHub Actions | `githubactions` | https://api.github.com | | VS Code extension | `vscode-extension` | https://open-vsx.org | | APM | None | URL required | @@ -207,6 +209,8 @@ The Alpine URL is one repository architecture directory, such as `https://dl-cdn The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded. +The RPM URL is one repository, the directory that holds `repodata/`, such as `https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os`. The client doesn't pick a release, variant or architecture from a mirror root, and doesn't follow mirror lists or metalinks. It reads `repodata/repomd.xml`, follows its `primary` entry, checks that file against the checksum and size listed there, and decodes `.zst`, `.xz`, `.gz` or plain XML as a stream. Repository metadata doesn't name its distribution, so the `distro` to record on artifacts is given as a query parameter: `.../x86_64/os?distro=fedora-43`. Each artifact records `arch`, `epoch` and that `distro`, with the package's own checksum as integrity (`sha256-...`). The version number is `version-release`, as in the `rpm` PURL type, where the epoch is a qualifier; versions are ordered by epoch, version and release. Dependencies are the requirements that name packages, leaving out file paths, sonames and capabilities in parentheses such as `rpmlib(...)`; a constraint on a package is listed once per distinct constraint. A virtual capability such as `webserver` can't be told from a package name and is listed as one. In a PURL, `repository_url` is the whole URL, `distro` parameter included, so its `?` and `=` are percent-encoded. + GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. VS Code extensions are read from [Open VSX](https://open-vsx.org), or from a self-hosted Open VSX server given as the base URL. The purl spec makes the Visual Studio Marketplace the default for `vscode-extension`, but the client has no Marketplace support, so a PURL without `repository_url` also resolves to Open VSX. `URLs().PURL` always adds the registry as `repository_url`. Names are `publisher/name` or VS Code's `publisher.name`. An extension can publish a separate build for each target platform, such as `linux-x64` or `darwin-arm64`. Each version from `FetchVersions` and `FetchVersion` carries one artifact per build, with `platform`, `signature` (the `.sigzip`) and `sha256` (the URL of the checksum file) in its metadata. `URLs().Download` returns the universal build, which a version built only for specific platforms doesn't have; the `platform` qualifier, as in `pkg:vscode-extension/muhammad-sammy/csharp@2.160.9?platform=linux-x64`, makes `fetch.Resolver` pick the build for that platform, or the universal one when there is none. `FetchDependencies` lists both the extensions a version depends on and the ones it bundles as an extension pack. `FetchMaintainers` returns the user who published the latest version. diff --git a/all/all.go b/all/all.go index 6e64dba..82299a4 100644 --- a/all/all.go +++ b/all/all.go @@ -9,7 +9,7 @@ // // // Now all ecosystems are available // ecosystems := registries.SupportedEcosystems() -// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform", "vscode-extension"] +// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform", "vscode-extension"] package all import ( @@ -41,6 +41,7 @@ import ( _ "github.com/git-pkgs/registries/internal/packagist" _ "github.com/git-pkgs/registries/internal/pub" _ "github.com/git-pkgs/registries/internal/pypi" + _ "github.com/git-pkgs/registries/internal/rpm" _ "github.com/git-pkgs/registries/internal/rubygems" _ "github.com/git-pkgs/registries/internal/swift" _ "github.com/git-pkgs/registries/internal/terraform" diff --git a/fetch/rpm_resolver_test.go b/fetch/rpm_resolver_test.go new file mode 100644 index 0000000..6911bdc --- /dev/null +++ b/fetch/rpm_resolver_test.go @@ -0,0 +1,67 @@ +//go:build !tinygo + +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/rpm" + "github.com/git-pkgs/registries/safehttp" +) + +// rpmRepository serves a repository with one version of curl for two +// architectures, over TLS: the resolver only hands out https download URLs. +func rpmRepository(t *testing.T) (string, *registries.Client) { + t.Helper() + files := map[string]string{ + "/os/repodata/repomd.xml": ``, + "/os/repodata/primary.xml": ` +curlx86_64 + aa +curlaarch64 + bb +`, + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + client := registries.DefaultClient() + client.HTTPClient = server.Client() + return server.URL + "/os", client +} + +func TestResolveRPMByQualifiers(t *testing.T) { + base, client := rpmRepository(t) + reg, err := registries.New("rpm", base+"?distro=fedora-43", client) + if err != nil { + t.Fatal(err) + } + r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true})) + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64", "distro": "fedora-43"}}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if want := base + "/Packages/c/curl-8.15.0-4.fc43.aarch64.rpm"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 { + t.Errorf("info = %+v, want %s", info, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "x86_64", "distro": "fedora-42"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/go.mod b/go.mod index 9d83d32..9a14bd8 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 + github.com/klauspost/compress v1.20.3 github.com/opencontainers/go-digest v1.0.0 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rubyist/circuitbreaker v2.2.1+incompatible diff --git a/go.sum b/go.sum index 30e87b0..f6e12a2 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= github.com/github/go-spdx/v2 v2.8.0 h1:WKCnF2Ee/x+zwSIEeQdm6+be0H6PiCNLQmSZ5CFmxMI= github.com/github/go-spdx/v2 v2.8.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= diff --git a/internal/rpm/rpm.go b/internal/rpm/rpm.go new file mode 100644 index 0000000..7a392d2 --- /dev/null +++ b/internal/rpm/rpm.go @@ -0,0 +1,657 @@ +// Package rpm provides a registry client for RPM repositories, read from +// their repodata. +package rpm + +import ( + "compress/gzip" + "context" + "crypto/sha1" //nolint:gosec // repomd.xml checksums of old repositories + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/url" + "path" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzlimit" + "github.com/git-pkgs/vers" + "github.com/klauspost/compress/zstd" +) + +const ( + DefaultURL = "" + ecosystem = "rpm" + + repomdPath = "repodata/repomd.xml" + + // maxPrimarySize bounds the decompressed primary metadata. Fedora 44 + // Everything x86_64 is about 185 MB. + maxPrimarySize = 1 << 30 + // maxZstdWindow bounds the window a zstd frame may ask the decoder to + // allocate. zstd --ultra -22 uses 128 MiB. + maxZstdWindow = 128 << 20 +) + +var ( + // ErrInvalidConfig is returned when the repository URL is missing or + // is not a URL. + ErrInvalidConfig = errors.New("invalid RPM repository configuration") + // ErrChecksumMismatch is returned when the primary metadata doesn't + // match the checksum or size repomd.xml lists. + ErrChecksumMismatch = errors.New("primary metadata does not match its repomd.xml checksum") + // ErrMalformedMetadata is returned for repodata that cannot be parsed. + ErrMalformedMetadata = errors.New("malformed RPM repository metadata") + // ErrUnsupportedCompression is returned when the primary metadata is + // compressed in a format other than zstd, xz or gzip. + ErrUnsupportedCompression = errors.New("unsupported primary metadata compression") +) + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads the primary metadata of one repository: the directory +// that holds repodata/, such as +// https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os. +// The release, variant and architecture are never inferred from a mirror +// root, and mirror lists and metalinks are not followed. +// +// Repository metadata doesn't name its distribution, so the distro to +// record on artifacts, as in the PURL qualifier, is given as a query +// parameter, which repositories don't use: .../x86_64/os?distro=fedora-43. +// +// The metadata is downloaded once per Registry and kept for its lifetime, +// so callers looking up many packages should reuse one Registry. +type Registry struct { + repoURL string + distro string + configErr error + client *core.Client + urls *URLs + + maxPrimarySize int64 + + mu sync.Mutex + packages map[string][]record +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + r := &Registry{client: client, maxPrimarySize: maxPrimarySize} + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + r.configErr = fmt.Errorf("%w: %q is not a repository URL", ErrInvalidConfig, baseURL) + } else { + r.distro = u.Query().Get("distro") + u.RawQuery, u.Fragment = "", "" + r.repoURL = strings.TrimSuffix(u.String(), "/") + } + r.urls = &URLs{repositoryURL: r.repoURL} + // As a PURL's repository_url carries it: the repository with the + // distro it was configured with. + if r.distro != "" { + r.urls.repositoryURL += "?" + url.Values{"distro": {r.distro}}.Encode() + } + return r +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// record is one package of the primary metadata. +type record struct { + name string + arch string + epoch string + version string // ver-rel + checksumType string + checksum string + summary string + packager string + url string + license string + sourceRPM string + buildTime time.Time + size int64 + installedSize int64 + href string // "" when the file is not under the repository + requires []core.Dependency +} + +// evr is the form versions are compared in. +func (rec record) evr() string { + return rec.epoch + ":" + rec.version +} + +// index reads the repository's metadata, once. Nothing is kept unless it +// was read and checked whole, so a failed read is tried again next time. +func (r *Registry) index(ctx context.Context) (map[string][]record, error) { + if r.configErr != nil { + return nil, r.configErr + } + r.mu.Lock() + defer r.mu.Unlock() + if r.packages != nil { + return r.packages, nil + } + + body, err := r.client.GetBody(ctx, r.repoURL+"/"+repomdPath) + if err != nil { + return nil, err + } + primary, err := parseRepomd(body) + if err != nil { + return nil, err + } + + // Decided before the download, so an unsupported format costs none. + if !supportedPrimary(primary.href) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedCompression, primary.href) + } + + packages := map[string][]record{} + seen := map[string]bool{} + primaryURL := r.repoURL + "/" + primary.href + err = r.client.Stream(ctx, primaryURL, func(body io.Reader) error { + return r.readPrimary(body, primary, primaryURL, func(rec record) { + key := rec.name + "\x00" + rec.evr() + "\x00" + rec.arch + if seen[key] { + return + } + seen[key] = true + packages[rec.name] = append(packages[rec.name], rec) + }) + }) + if err != nil { + return nil, err + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].evr(), recs[j].evr(), ecosystem) > 0 + }) + } + r.packages = packages + return packages, nil +} + +// primaryData is repomd.xml's entry for the primary metadata. +type primaryData struct { + href string + checksumType string // "" when repomd.xml lists no checksum + checksum string + size int64 +} + +func parseRepomd(data []byte) (*primaryData, error) { + var repomd struct { + Data []struct { + Type string `xml:"type,attr"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + Size int64 `xml:"size"` + } `xml:"data"` + } + if err := xml.Unmarshal(data, &repomd); err != nil { + return nil, fmt.Errorf("%w: repomd.xml: %w", ErrMalformedMetadata, err) + } + for _, d := range repomd.Data { + if d.Type != "primary" { + continue + } + if !safeHref(d.Location.Href) { + return nil, fmt.Errorf("%w: repomd.xml: primary location %q", ErrMalformedMetadata, d.Location.Href) + } + return &primaryData{ + href: d.Location.Href, + checksumType: strings.ToLower(strings.TrimSpace(d.Checksum.Type)), + checksum: strings.ToLower(strings.TrimSpace(d.Checksum.Value)), + size: d.Size, + }, nil + } + return nil, fmt.Errorf("%w: repomd.xml lists no primary metadata", ErrMalformedMetadata) +} + +func newHash(checksumType string) (hash.Hash, bool) { + switch checksumType { + case "sha256": + return sha256.New(), true + case "sha512": + return sha512.New(), true + case "sha384": + return sha512.New384(), true + case "sha224": + return sha256.New224(), true + case "sha", "sha1": + return sha1.New(), true //nolint:gosec // as listed by the repository + } + return nil, false +} + +// readPrimary decompresses and parses the primary metadata, checking the +// compressed bytes against the checksum and size repomd.xml lists. +func (r *Registry) readPrimary(body io.Reader, primary *primaryData, primaryURL string, fn func(record)) error { + var hasher hash.Hash + counted := &countingReader{r: body} + if primary.checksumType != "" { + h, ok := newHash(primary.checksumType) + if !ok { + return fmt.Errorf("%w: repomd.xml: checksum type %q", ErrMalformedMetadata, primary.checksumType) + } + hasher = h + counted.r = io.TeeReader(body, hasher) + } + check := func() error { + if hasher == nil { + return nil + } + got := hex.EncodeToString(hasher.Sum(nil)) + if got != primary.checksum || (primary.size > 0 && counted.n != primary.size) { + return fmt.Errorf("%w: %s is %d bytes with %s %s, repomd.xml lists %d bytes with %s", + ErrChecksumMismatch, primaryURL, counted.n, primary.checksumType, got, primary.size, primary.checksum) + } + return nil + } + + var decoded io.Reader + switch { + case strings.HasSuffix(primary.href, ".zst"): + zr, err := zstd.NewReader(counted, zstd.WithDecoderMaxWindow(maxZstdWindow), zstd.WithDecoderConcurrency(1)) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer zr.Close() + decoded = zr + case strings.HasSuffix(primary.href, ".xz"): + // Bounded as it decodes: a block asking for more is an error. + decoded = xzlimit.NewReader(counted, xzlimit.MaxDict) + case strings.HasSuffix(primary.href, ".gz"): + gz, err := gzip.NewReader(counted) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer func() { _ = gz.Close() }() + decoded = gz + default: + decoded = counted + } + + limited := &io.LimitedReader{R: decoded, N: r.maxPrimarySize + 1} + if err := parsePrimary(limited, fn); err != nil { + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + return err + } + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + return check() +} + +// supportedPrimary reports whether href names primary metadata the client +// can decode: zstd, xz, gzip or plain XML. +func supportedPrimary(href string) bool { + for _, ext := range []string{".zst", ".xz", ".gz", ".xml"} { + if strings.HasSuffix(href, ext) { + return true + } + } + return false +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// xmlPackage is a of primary.xml, with the fields the client uses. +type xmlPackage struct { + Type string `xml:"type,attr"` + Name string `xml:"name"` + Arch string `xml:"arch"` + Version struct { + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"version"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Summary string `xml:"summary"` + Packager string `xml:"packager"` + URL string `xml:"url"` + Time struct { + Build int64 `xml:"build,attr"` + } `xml:"time"` + Size struct { + Package int64 `xml:"package,attr"` + Installed int64 `xml:"installed,attr"` + } `xml:"size"` + Location struct { + Href string `xml:"href,attr"` + Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"` + } `xml:"location"` + Format struct { + License string `xml:"license"` + SourceRPM string `xml:"sourcerpm"` + Requires []struct { + Name string `xml:"name,attr"` + Flags string `xml:"flags,attr"` + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"requires>entry"` + } `xml:"format"` +} + +// parsePrimary reads primary.xml as a stream, calling fn for each rpm +// package. +func parsePrimary(r io.Reader, fn func(record)) error { + dec := xml.NewDecoder(r) + // An empty response, or another document, is no empty repository. + root := false + for { + tok, err := dec.Token() + if err == io.EOF { + if !root { + return fmt.Errorf("%w: primary.xml: no metadata element", ErrMalformedMetadata) + } + return nil + } + if err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + start, ok := tok.(xml.StartElement) + if !ok { + continue + } + if !root { + if start.Name.Local != "metadata" { + return fmt.Errorf("%w: primary.xml: root element %q, want metadata", ErrMalformedMetadata, start.Name.Local) + } + root = true + } + if start.Name.Local != "package" { + continue + } + var pkg xmlPackage + if err := dec.DecodeElement(&pkg, &start); err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + if pkg.Type != "rpm" { + continue + } + if pkg.Name == "" || pkg.Version.Ver == "" { + return fmt.Errorf("%w: primary.xml: package without a name or version", ErrMalformedMetadata) + } + fn(pkg.record()) + } +} + +func (pkg *xmlPackage) record() record { + rec := record{ + name: pkg.Name, + arch: pkg.Arch, + epoch: pkg.Version.Epoch, + version: evr("", pkg.Version.Ver, pkg.Version.Rel), + checksumType: strings.ToLower(pkg.Checksum.Type), + checksum: strings.ToLower(strings.TrimSpace(pkg.Checksum.Value)), + summary: pkg.Summary, + packager: pkg.Packager, + url: pkg.URL, + license: pkg.Format.License, + sourceRPM: pkg.Format.SourceRPM, + size: pkg.Size.Package, + installedSize: pkg.Size.Installed, + } + if rec.epoch == "" { + rec.epoch = "0" + } + if pkg.Time.Build > 0 { + rec.buildTime = time.Unix(pkg.Time.Build, 0).UTC() + } + // xml:base points the file at another repository, which is not followed. + if pkg.Location.Base == "" && safeHref(pkg.Location.Href) { + rec.href = pkg.Location.Href + } + // Every constraint on a package holds, so each is kept; only an identical + // requirement is listed once. + seen := map[core.Dependency]bool{} + for _, req := range pkg.Format.Requires { + // File paths, sonames and capabilities such as rpmlib(...) name no + // package. A virtual capability such as webserver reads like a + // package name and is listed as one. + if req.Name == "" || strings.HasPrefix(req.Name, "/") || strings.Contains(req.Name, "(") || + sonamePattern.MatchString(req.Name) { + continue + } + dep := core.Dependency{ + Name: req.Name, + Requirements: requirement(req.Flags, req.Epoch, req.Ver, req.Rel), + Scope: core.Runtime, + } + if seen[dep] { + continue + } + seen[dep] = true + rec.requires = append(rec.requires, dep) + } + return rec +} + +// sonamePattern matches a soname written without its symbol version or +// word size, such as libfoo.so.1 in a 32-bit package. +var sonamePattern = regexp.MustCompile(`\.so(\.[0-9]+)*$`) + +// evr joins an epoch, version and release, leaving out an epoch of 0 and +// an empty release. +func evr(epoch, ver, rel string) string { + s := ver + if epoch != "" && epoch != "0" { + s = epoch + ":" + s + } + if rel != "" { + s += "-" + rel + } + return s +} + +// requirement renders a requires entry's version constraint. +func requirement(flags, epoch, ver, rel string) string { + ops := map[string]string{"EQ": "=", "GE": ">=", "LE": "<=", "GT": ">", "LT": "<"} + op, ok := ops[flags] + if !ok || ver == "" { + return "" + } + return op + " " + evr(epoch, ver, rel) +} + +// safeHref reports whether a location stays under the repository root: +// relative, without dot segments, escapes or URL syntax. +func safeHref(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.ContainsAny(p, `\?#%:`) { + return false + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + +// splitName strips a PURL vendor namespace: "fedora/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + packages, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.summary, + Homepage: latest.url, + Licenses: latest.license, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "epoch": latest.epoch, + "source_rpm": latest.sourceRPM, + }, + }, nil +} + +// FetchVersions returns one Version per epoch, version and release, newest +// first, with an artifact for each architecture. The version number is +// version-release, as in the rpm PURL type, where the epoch is a qualifier. +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + var versions []core.Version + last := "" + for _, rec := range recs { + if len(versions) == 0 || last != rec.evr() { + last = rec.evr() + versions = append(versions, core.Version{ + Number: rec.version, + PublishedAt: rec.buildTime, + Licenses: rec.license, + Metadata: map[string]any{ + "epoch": rec.epoch, + "source_rpm": rec.sourceRPM, + "packager": rec.packager, + "installed_size": rec.installedSize, + }, + }) + } + if artifact, ok := r.artifact(rec); ok { + v := &versions[len(versions)-1] + v.Artifacts = append(v.Artifacts, artifact) + } + } + return versions, nil +} + +func (r *Registry) artifact(rec record) (core.Artifact, bool) { + if rec.href == "" { + return core.Artifact{}, false + } + artifact := core.Artifact{ + URL: r.repoURL + "/" + rec.href, + Filename: path.Base(rec.href), + Size: rec.size, + Metadata: map[string]any{"arch": rec.arch, "epoch": rec.epoch}, + } + if r.distro != "" { + artifact.Metadata["distro"] = r.distro + } + if rec.checksum != "" && rec.checksumType != "" { + artifact.Integrity = rec.checksumType + "-" + rec.checksum + } + return artifact, true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version == version { + return rec.requires, nil + } + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].packager == "" { + return nil, nil + } + return []core.Maintainer{{Name: recs[0].packager}}, nil +} + +type URLs struct { + repositoryURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns "": a package's file is only known from the metadata. +func (u *URLs) Download(name, version string) string { + return "" +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + var qualifiers map[string]string + if u.repositoryURL != "" { + qualifiers = map[string]string{"repository_url": u.repositoryURL} + } + return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() +} diff --git a/internal/rpm/rpm_test.go b/internal/rpm/rpm_test.go new file mode 100644 index 0000000..c82ea8e --- /dev/null +++ b/internal/rpm/rpm_test.go @@ -0,0 +1,726 @@ +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/internal/core" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// Trimmed from Fedora 43 updates' primary.xml, with a second architecture, +// an older build and a package with an epoch added. +const primaryXML = ` + + + curl + x86_64 + + 235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a + A utility for getting files from remote servers + curl is a command line tool +for transferring data. + Fedora Project + https://curl.se/ + + + curl + i686 + + 335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b + + + + curl + x86_64 + + AB5b92fa + + + + bind + x86_64 + + 435b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5c + + + + bind + x86_64 + + 535b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5d + + + + tzdata + noarch + + 635b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5e + + + +` + +func compress(t *testing.T, ext, s string) []byte { + t.Helper() + var buf bytes.Buffer + var w interface { + Write([]byte) (int, error) + Close() error + } + var err error + switch ext { + case ".zst": + w, err = zstd.NewWriter(&buf) + case ".xz": + w, err = xz.NewWriter(&buf) + case ".gz": + w = gzip.NewWriter(&buf) + default: + return []byte(s) + } + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// repo is a fake RPM repository under /os/. +type repo struct { + mu sync.Mutex + files map[string][]byte + requests []string +} + +// newRepo serves primary, compressed as ext ("" for plain .xml), and a +// repomd.xml listing it with its sha256 and size. +func newRepo(t *testing.T, ext, primary string) (*repo, string) { + t.Helper() + r := &repo{files: map[string][]byte{}} + data := compress(t, ext, primary) + sum := sha256.Sum256(data) + href := "repodata/" + hex.EncodeToString(sum[:]) + "-primary.xml" + ext + r.files[href] = data + r.setRepomd("sha256", hex.EncodeToString(sum[:]), href, len(data)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + path := strings.TrimPrefix(req.URL.Path, "/os/") + r.mu.Lock() + r.requests = append(r.requests, path) + body, ok := r.files[path] + r.mu.Unlock() + if !ok { + http.NotFound(w, req) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return r, srv.URL + "/os" +} + +func (r *repo) setRepomd(sumType, sum, href string, size int) { + r.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(` + + 1794529563 + + 00 + + 1 + + + %s + + %d + 1 + + +`, sumType, sum, href, size)) +} + +func (r *repo) primaryHref() string { + for path := range r.files { + if strings.Contains(path, "-primary.xml") { + return path + } + } + return "" +} + +func numbers(versions []core.Version) []string { + var out []string + for _, v := range versions { + out = append(out, v.Number) + } + return out +} + +func TestFetchVersions(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + versions, err := r.FetchVersions(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"8.15.0-10.fc43", "8.15.0-4.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (newest first, by rpm version)", numbers(versions), want) + } + older := versions[1] + if !older.PublishedAt.Equal(time.Unix(1771515699, 0)) { + t.Errorf("published = %v, want the build time", older.PublishedAt) + } + if older.Licenses != "curl" || older.Metadata["epoch"] != "0" || older.Metadata["source_rpm"] != "curl-8.15.0-4.fc43.src.rpm" { + t.Errorf("version = %+v", older) + } + want := []core.Artifact{ + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.x86_64.rpm", + Filename: "curl-8.15.0-4.fc43.x86_64.rpm", + Integrity: "sha256-235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a", + Size: 234567, + Metadata: map[string]any{"arch": "x86_64", "distro": "fedora-43", "epoch": "0"}, + }, + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.i686.rpm", + Filename: "curl-8.15.0-4.fc43.i686.rpm", + Integrity: "sha256-335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b", + Metadata: map[string]any{"arch": "i686", "distro": "fedora-43", "epoch": "0"}, + }, + } + if !reflect.DeepEqual(older.Artifacts, want) { + t.Errorf("artifacts =\n %+v\nwant\n %+v", older.Artifacts, want) + } + // The checksum's own type names the integrity, lowercased. + if got := versions[0].Artifacts[0].Integrity; got != "sha512-ab5b92fa" { + t.Errorf("sha512 integrity = %q", got) + } + }) + } +} + +// The epoch orders versions but is not part of the version number, as in +// the rpm PURL type, where it is a qualifier. +func TestEpoch(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "bind") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"9.18.41-1.fc43", "9.20.1-1.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (epoch 32 before epoch 31)", numbers(versions), want) + } + if e := versions[0].Metadata["epoch"]; e != "32" { + t.Errorf("epoch = %v", e) + } + if e := versions[0].Artifacts[0].Metadata["epoch"]; e != "32" { + t.Errorf("artifact epoch = %v", e) + } +} + +// Without a distro in the URL none is recorded, and noarch packages record +// their own architecture. +func TestNoarchAndNoDistro(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base+"/", core.DefaultClient()).FetchVersions(context.Background(), "tzdata") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 1 { + t.Fatalf("FetchVersions = %+v, %v", versions, err) + } + if got, want := versions[0].Artifacts[0].Metadata, (map[string]any{"arch": "noarch", "epoch": "0"}); !reflect.DeepEqual(got, want) { + t.Errorf("metadata = %v, want %v", got, want) + } + if got, want := versions[0].Artifacts[0].URL, base+"/Packages/t/tzdata-2026a-1.fc43.noarch.rpm"; got != want { + t.Errorf("URL = %q, want %q", got, want) + } +} + +// Primary metadata is streamed, so the client's limit on buffered +// responses doesn't apply to it. +func TestPrimaryLargerThanResponseCap(t *testing.T) { + _, base := newRepo(t, "", primaryXML) + client := core.NewClient(core.WithMaxRetries(0)) + client.MaxResponseSize = 1024 // repomd.xml fits, primary.xml doesn't + if _, err := New(base, client).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestChecksumMismatch(t *testing.T) { + other := strings.Replace(primaryXML, "8.15.0", "8.15.1", 1) + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + rp, base := newRepo(t, ext, primaryXML) + href := rp.primaryHref() + good := rp.files[href] + rp.files[href] = compress(t, ext, other) + + r := New(base, core.DefaultClient()) + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("err = %v, want ErrChecksumMismatch", err) + } + // Nothing was kept: the next lookup reads the repository again. + rp.files[href] = good + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("after the repository was fixed: %v", err) + } + }) + } +} + +// The right hash with another size is rejected too. +func TestChecksumSizeMismatch(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + sum := sha256.Sum256([]byte(primaryXML)) + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), rp.primaryHref(), len(primaryXML)+1) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +func TestChecksumTypes(t *testing.T) { + sum512 := sha512.Sum512([]byte(primaryXML)) + for name, tt := range map[string]struct { + sumType, sum string + wantErr error + }{ + "sha512": {"sha512", hex.EncodeToString(sum512[:]), nil}, + "sha512 upper": {"SHA512", strings.ToUpper(hex.EncodeToString(sum512[:])), nil}, + "sha512 wrong": {"sha512", "00", ErrChecksumMismatch}, + "unknown type": {"md5", "00", ErrMalformedMetadata}, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd(tt.sumType, tt.sum, rp.primaryHref(), len(primaryXML)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, tt.wantErr) { + t.Errorf("err = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestInvalidConfiguration(t *testing.T) { + for name, url := range map[string]string{"no url": "", "no host": "/pub/fedora"} { + t.Run(name, func(t *testing.T) { + _, err := New(url, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("err = %v, want ErrInvalidConfig", err) + } + }) + } +} + +func TestMalformedRepomd(t *testing.T) { + for name, repomd := range map[string]string{ + "not xml": "", + "no primary": ``, + "href leaves the repository": ``, + "absolute href": ``, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(repomd) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } +} + +func TestUnsupportedCompression(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd("sha256", "00", "repodata/primary.xml.zck", 1) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrUnsupportedCompression) { + t.Errorf("err = %v, want ErrUnsupportedCompression", err) + } +} + +// A repomd.xml that lists a checksum but no size checks the checksum alone. +func TestChecksumWithoutSize(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + href := rp.primaryHref() + sum := sha256.Sum256(rp.files[href]) + repomd := `%s` + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, hex.EncodeToString(sum[:]))) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, "00")) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("wrong checksum: err = %v, want ErrChecksumMismatch", err) + } +} + +// A repomd.xml without a checksum for primary leaves it unchecked. +func TestNoChecksum(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(``) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestMalformedPrimary(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz"} { + t.Run("not"+ext, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + garbage := []byte(strings.Repeat("not compressed data ", 10)) + sum := sha256.Sum256(garbage) + rp.files["repodata/primary.xml"+ext] = garbage + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml"+ext, len(garbage)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } + t.Run("truncated xml", func(t *testing.T) { + _, base := newRepo(t, "", primaryXML[:len(primaryXML)/2]) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + t.Run("package without a name", func(t *testing.T) { + _, base := newRepo(t, "", `x86_64`) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) +} + +func TestDecompressedLimit(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + // Cut mid-document, the limit is still what the error names, + // not the XML it broke. + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) / 2 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("far over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } + }) + } +} + +// A zstd frame asking for a 256 MiB window, which the decoder would allow +// by default, is refused rather than allocated. +func TestZstdWindowRefused(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + // Magic, a frame header with no content size, a window descriptor of + // 2^28 bytes, and one last raw block holding a document. + frame := append([]byte{0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x90, 0x61, 0x00, 0x00}, "\n"...) + sum := sha256.Sum256(frame) + rp.files["repodata/primary.xml.zst"] = frame + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml.zst", len(frame)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } +} + +func TestNotFound(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "missing"); !errors.As(err, ¬Found) { + t.Errorf("missing package: err = %v, want NotFoundError", err) + } + if _, err := r.FetchDependencies(context.Background(), "curl", "1.0-1"); !errors.As(err, ¬Found) || notFound.Version != "1.0-1" { + t.Errorf("missing version: err = %v, want NotFoundError for 1.0-1", err) + } +} + +// A package whose file is outside the repository, by its href or by an +// xml:base pointing elsewhere, gets no artifact. Other package types are +// not read. +func TestUnsafeLocations(t *testing.T) { + primary := ` +upx86_64 +basedx86_64 +nonex86_64 +barex86_64 +barex86_64 +otherx86_64 +` + _, base := newRepo(t, "", primary) + r := New(base, core.DefaultClient()) + for _, name := range []string{"up", "based", "none"} { + versions, err := r.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 0 { + t.Errorf("%s: versions = %+v, %v; want one without artifacts", name, versions, err) + } + } + // No epoch means 0, no checksum means no integrity, and a package + // listed twice is recorded once. + bare, err := r.FetchVersions(context.Background(), "bare") + if err != nil || len(bare) != 1 || len(bare[0].Artifacts) != 1 { + t.Fatalf("bare: versions = %+v, %v; want one with one artifact", bare, err) + } + if a := bare[0].Artifacts[0]; a.Integrity != "" || a.Metadata["epoch"] != "0" || a.Filename != "bare-1-1.x86_64.rpm" || !strings.HasSuffix(a.URL, "/Packages/bare-1-1.x86_64.rpm") { + t.Errorf("bare artifact = %+v", a) + } + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "other"); !errors.As(err, ¬Found) { + t.Errorf("non-rpm package: err = %v, want NotFoundError", err) + } + if got, err := r.FetchMaintainers(context.Background(), "up"); err != nil || got != nil { + t.Errorf("maintainers = %+v, %v; want none without a packager", got, err) + } +} + +func TestFetchPackage(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + pkg, err := r.FetchPackage(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + // The newest build has no summary of its own in this fixture. + if pkg.Name != "curl" || pkg.Namespace != "fedora" || pkg.LatestVersion != "8.15.0-10.fc43" { + t.Errorf("package = %+v", pkg) + } + pkg, err = New(base, core.DefaultClient()).FetchPackage(context.Background(), "bind") + if err != nil || pkg.LatestVersion != "9.18.41-1.fc43" || pkg.Metadata["epoch"] != "32" { + t.Errorf("bind = %+v, %v", pkg, err) + } +} + +func TestFetchPackageDetails(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + pkg, err := New(base, core.DefaultClient()).FetchPackage(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Description != "A utility for getting files from remote servers" || pkg.Homepage != "https://curl.se/" || pkg.Licenses != "curl" { + t.Errorf("package = %+v", pkg) + } +} + +// Dependencies are the requirements that name packages: file paths, +// sonames and other capabilities are left out. +func TestFetchDependencies(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + deps, err := New(base, core.DefaultClient()).FetchDependencies(context.Background(), "curl", "8.15.0-4.fc43") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "libcurl", Requirements: ">= 8.15.0-4.fc43", Scope: core.Runtime}, + {Name: "openssl-libs", Requirements: "= 1:3.5.4", Scope: core.Runtime}, + {Name: "zlib", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +func TestRequirement(t *testing.T) { + for _, tt := range []struct { + flags, epoch, ver, rel, want string + }{ + {"EQ", "0", "1.0", "1", "= 1.0-1"}, + {"GE", "", "1.0", "", ">= 1.0"}, + {"LE", "2", "1.0", "1", "<= 2:1.0-1"}, + {"GT", "0", "1.0", "", "> 1.0"}, + {"LT", "0", "1.0", "", "< 1.0"}, + {"", "0", "", "", ""}, + {"EQ", "0", "", "", ""}, + {"XX", "0", "1.0", "", ""}, + } { + if got := requirement(tt.flags, tt.epoch, tt.ver, tt.rel); got != tt.want { + t.Errorf("requirement(%q, %q, %q, %q) = %q, want %q", tt.flags, tt.epoch, tt.ver, tt.rel, got, tt.want) + } + } +} + +func TestFetchMaintainers(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + got, err := New(base, core.DefaultClient()).FetchMaintainers(context.Background(), "curl") + if err != nil || !reflect.DeepEqual(got, []core.Maintainer{{Name: "Fedora Project"}}) { + t.Errorf("maintainers = %+v, %v", got, err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os/?distro=fedora-43", nil).URLs() + if got := u.Download("curl", "8.15.0-4.fc43"); got != "" { + t.Errorf("Download = %q, want none without the metadata", got) + } + // repository_url is the whole configured URL, distro included. + const repo = "?repository_url=https:%2F%2Fdl.fedoraproject.org%2Fpub%2Ffedora%2Flinux%2Freleases%2F43%2FEverything%2Fx86_64%2Fos" + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo+"%3Fdistro%3Dfedora-43"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } + u = New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", nil).URLs() + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo; got != want { + t.Errorf("PURL without a distro = %q, want %q", got, want) + } + if got, want := New("", nil).URLs().PURL("fedora/curl", "1-1"), "pkg:rpm/fedora/curl@1-1"; got != want { + t.Errorf("PURL without a repository = %q, want %q", got, want) + } +} + +// A PURL the registry builds leads back to the same repository and distro. +func TestPURLRoundTrip(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + reg, name, version, err := core.NewFromPURL(r.URLs().PURL("fedora/curl", "8.15.0-4.fc43"), core.DefaultClient()) + if err != nil || name != "fedora/curl" || version != "8.15.0-4.fc43" { + t.Fatalf("NewFromPURL = %q, %q, %v", name, version, err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 2 || versions[1].Artifacts[0].Metadata["distro"] != "fedora-43" { + t.Errorf("FetchVersions = %+v, %v", versions, err) + } +} + +// A distro with characters a query escapes survives the round trip. +func TestPURLRoundTripEscapedDistro(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base+"?distro=custom%2Bstable", core.DefaultClient()) + reg, name, _, err := core.NewFromPURL(r.URLs().PURL("fedora/curl", "8.15.0-4.fc43"), core.DefaultClient()) + if err != nil { + t.Fatalf("NewFromPURL: %v", err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) == 0 || versions[0].Artifacts[0].Metadata["distro"] != "custom+stable" { + t.Errorf("FetchVersions = %+v, %v; want distro custom+stable", versions, err) + } +} + +// Requirements on one package with different constraints are all kept, as +// RPM requires all of them; an identical one is listed once. A bare soname +// names no package. +func TestFetchDependenciesKeepsEveryConstraint(t *testing.T) { + _, base := newRepo(t, ".gz", ` + toolx86_64 + + + + + + + + +`) + deps, err := New(base, core.DefaultClient()).FetchDependencies(context.Background(), "tool", "1.0-1") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "zlib", Requirements: ">= 1.2.13", Scope: core.Runtime}, + {Name: "zlib", Requirements: "< 2", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +// An empty response, or a document that isn't repository metadata, is not +// an empty repository: caching it would hide every package until restart. +func TestPrimaryNeedsAMetadataRoot(t *testing.T) { + for name, body := range map[string]string{"empty": "", "whitespace": " \n", "other root": ``} { + t.Run(name, func(t *testing.T) { + _, base := newRepo(t, "", body) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } + _, base := newRepo(t, "", ``) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Errorf("empty metadata: err = %v, want NotFoundError", err) + } +} + +// The metadata is read once per Registry. +func TestMetadataReadOnce(t *testing.T) { + rp, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + for range 3 { + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + } + if len(rp.requests) != 2 { + t.Errorf("requests = %v, want repomd.xml and primary once each", rp.requests) + } +} + +func TestSafeHref(t *testing.T) { + for in, ok := range map[string]bool{ + "Packages/c/curl-8.15.0-4.fc43.x86_64.rpm": true, + "": false, + "/etc/passwd": false, + "Packages/../../etc/passwd": false, + "Packages/x.rpm?query": false, + "Packages/x.rpm#frag": false, + `Packages\x.rpm`: false, + "Packages/%2e%2e/x.rpm": false, + "http://evil.example/x.rpm": false, + } { + if got := safeHref(in); got != ok { + t.Errorf("safeHref(%q) = %v, want %v", in, got, ok) + } + } +} diff --git a/registries_test.go b/registries_test.go index 6d3a5eb..1fdc450 100644 --- a/registries_test.go +++ b/registries_test.go @@ -20,7 +20,7 @@ import ( func TestSupportedEcosystems(t *testing.T) { ecosystems := registries.SupportedEcosystems() - expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform", "vscode-extension"} + expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform", "vscode-extension"} sort.Strings(ecosystems) if len(ecosystems) != len(expected) { @@ -71,6 +71,7 @@ func TestNew(t *testing.T) { {"apm", true}, {"apk", true}, {"deb", true}, + {"rpm", true}, {"unknown", true}, } @@ -121,6 +122,7 @@ func TestDefaultURL(t *testing.T) { {"apm", ""}, {"apk", ""}, {"deb", ""}, + {"rpm", ""}, } for _, tt := range tests {