diff --git a/README.md b/README.md index 8c65094..ee0263b 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Go | `pkg:golang/github.com/gorilla/mux@v1.8.0` | | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` | | RubyGems | `pkg:gem/rails@7.1.0` | -| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | +| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` (module), `pkg:terraform/hashicorp/random@3.6.0` (provider) | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | VS Code extension | `pkg:vscode-extension/redhat/vscode-yaml@1.15.0` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | @@ -213,6 +213,8 @@ The RPM URL is one repository, the directory that holds `repodata/`, such as `ht GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. +The Terraform URL is a module and provider registry, such as `https://registry.opentofu.org`. The client reads the registry's service discovery document, `/.well-known/terraform.json` on its host, once, and sends module and provider requests to the `modules.v1` and `providers.v1` URLs it names. A registry that has no document (a 4xx other than 429, or a body that isn't JSON), or no entry for a service, is read at `{URL}/v1/modules/` and `{URL}/v1/providers/`, and registry.terraform.io is always read there without asking. `URLs().Download` builds a module's download URL from the base URL without reading the document. + VS Code extensions are read from [Open VSX](https://open-vsx.org), or from a self-hosted Open VSX server given as the base URL. The purl spec makes the Visual Studio Marketplace the default for `vscode-extension`, but the client has no Marketplace support, so a PURL without `repository_url` also resolves to Open VSX. `URLs().PURL` always adds the registry as `repository_url`. Names are `publisher/name` or VS Code's `publisher.name`. An extension can publish a separate build for each target platform, such as `linux-x64` or `darwin-arm64`. Each version from `FetchVersions` and `FetchVersion` carries one artifact per build, with `platform`, `signature` (the `.sigzip`) and `sha256` (the URL of the checksum file) in its metadata. `URLs().Download` returns the universal build, which a version built only for specific platforms doesn't have; the `platform` qualifier, as in `pkg:vscode-extension/muhammad-sammy/csharp@2.160.9?platform=linux-x64`, makes `fetch.Resolver` pick the build for that platform, or the universal one when there is none. `FetchDependencies` lists both the extensions a version depends on and the ones it bundles as an extension pack. `FetchMaintainers` returns the user who published the latest version. ### Swift registries diff --git a/client/client.go b/client/client.go index 6caae2a..3dce032 100644 --- a/client/client.go +++ b/client/client.go @@ -10,6 +10,7 @@ import ( "math" "math/rand" "net/http" + "net/url" "time" "github.com/git-pkgs/registries/safehttp" @@ -76,6 +77,32 @@ func (c *Client) GetJSONWithHeaders(ctx context.Context, url string, headers htt return responseHeaders, json.Unmarshal(body, v) } +// GetJSONWithFinalURL is GetJSON that also returns the URL the response came +// from, after redirects, for documents whose relative URLs resolve against +// it. +func (c *Client) GetJSONWithFinalURL(ctx context.Context, rawURL string, v any) (*url.URL, error) { + var body []byte + var final *url.URL + _, _, err := c.open(ctx, http.MethodGet, rawURL, nil, func(resp *http.Response) error { + defer func() { _ = resp.Body.Close() }() + if resp.Request != nil { + final = resp.Request.URL + } + var err error + body, err = readLimited(resp.Body, rawURL, c.maxResponseSize()) + return err + }) + if err != nil { + return nil, err + } + if final == nil { + if final, err = url.Parse(rawURL); err != nil { + return nil, err + } + } + return final, json.Unmarshal(body, v) +} + // GetBody fetches a URL and returns the response body. A body larger than // the client's MaxResponseSize fails with ErrResponseTooLarge. func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) { diff --git a/client/client_test.go b/client/client_test.go index cec5761..625be06 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -337,3 +337,24 @@ func TestStream(t *testing.T) { t.Errorf("fn error: err = %v, want it returned as is", err) } } + +// GetJSONWithFinalURL reports the URL the response came from, after redirects. +func TestGetJSONWithFinalURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + http.Redirect(w, r, "/moved/doc.json", http.StatusFound) + return + } + _, _ = w.Write([]byte(`{"ok": true}`)) + })) + defer srv.Close() + + var v struct{ OK bool } + final, err := DefaultClient().GetJSONWithFinalURL(context.Background(), srv.URL+"/start", &v) + if err != nil || !v.OK { + t.Fatalf("GetJSONWithFinalURL = %+v, %v", v, err) + } + if final == nil || final.Path != "/moved/doc.json" { + t.Errorf("final URL = %v, want /moved/doc.json", final) + } +} diff --git a/internal/terraform/discovery.go b/internal/terraform/discovery.go new file mode 100644 index 0000000..6b075f7 --- /dev/null +++ b/internal/terraform/discovery.go @@ -0,0 +1,93 @@ +package terraform + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/git-pkgs/registries/internal/core" +) + +const ( + discoveryPath = "/.well-known/terraform.json" + modulesService = "modules.v1" + providersService = "providers.v1" +) + +// services are the base URLs of a registry's module and provider APIs, each +// ending in a slash. +type services struct { + modules string + providers string +} + +// serviceURL reads the discovery document once; a failure to read it is +// returned and retried on the next call. +func (r *Registry) serviceURL(ctx context.Context, service string) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.services == nil { + found, err := r.discover(ctx) + if err != nil { + return "", err + } + r.services = found + } + if service == modulesService { + return r.services.modules, nil + } + return r.services.providers, nil +} + +// refused reports whether a registry answered with a 4xx other than 429: it +// doesn't serve what was asked for, as opposed to failing for now. +func refused(err error) bool { + var httpErr *core.HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode >= 400 && httpErr.StatusCode < 500 && + httpErr.StatusCode != http.StatusTooManyRequests +} + +func (r *Registry) discover(ctx context.Context) (*services, error) { + found := &services{modules: r.baseURL + "/v1/modules/", providers: r.baseURL + "/v1/providers/"} + // registry.terraform.io serves these paths, and the PURL helpers build a + // Registry per call, so asking it would double every lookup. + if r.baseURL == DefaultURL { + return found, nil + } + base, err := url.Parse(r.baseURL) + if err != nil || base.Host == "" { + return found, nil //nolint:nilerr // a base URL without a host has no discovery document + } + document := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: discoveryPath} + var entries map[string]json.RawMessage + // Service URLs are relative to where the document was found, after + // redirects. + final, err := r.client.GetJSONWithFinalURL(ctx, document.String(), &entries) + if err != nil { + // A refusal, or a body that isn't a JSON object, such as an HTML + // page for any unknown path, means the host has no document. + var syntaxErr *json.SyntaxError + var typeErr *json.UnmarshalTypeError + if refused(err) || errors.As(err, &syntaxErr) || errors.As(err, &typeErr) { + return found, nil + } + return nil, fmt.Errorf("terraform service discovery: %w", err) + } + for service, target := range map[string]*string{modulesService: &found.modules, providersService: &found.providers} { + var location string + // Other services, and entries that aren't a URL, are left alone. + if raw, ok := entries[service]; !ok || json.Unmarshal(raw, &location) != nil || location == "" { + continue + } + resolved, err := final.Parse(location) + if err != nil || (resolved.Scheme != "https" && resolved.Scheme != "http") || resolved.Host == "" { + return nil, fmt.Errorf("terraform service discovery: %s is %q, not an http or https URL", service, location) + } + *target = strings.TrimSuffix(resolved.String(), "/") + "/" + } + return found, nil +} diff --git a/internal/terraform/discovery_test.go b/internal/terraform/discovery_test.go new file mode 100644 index 0000000..9d15821 --- /dev/null +++ b/internal/terraform/discovery_test.go @@ -0,0 +1,187 @@ +package terraform + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/internal/core" +) + +// A registry names where its module and provider APIs live in its service +// discovery document. Requests go there, and the document is read once. +func TestServiceDiscovery(t *testing.T) { + var discoveries atomic.Int32 + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case discoveryPath: + discoveries.Add(1) + // A relative and an absolute URL, as the protocol allows both. + _, _ = w.Write([]byte(`{"modules.v1": "/api/terraform/modules/", "providers.v1": "` + server.URL + `/api/terraform/providers/"}`)) + case "/api/terraform/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + case "/api/terraform/modules/hashicorp/consul/aws/versions": + _, _ = w.Write([]byte(`{"modules": [{"versions": [{"version": "0.11.0"}]}]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + + r := New(server.URL, core.DefaultClient()) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("provider versions = %v, %v; want 3", versions, err) + } + if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 { + t.Errorf("module versions = %v, %v; want 1", versions, err) + } + if got := discoveries.Load(); got != 1 { + t.Errorf("discovery read %d times, want once", got) + } +} + +// Without a discovery document, or without an entry for a service, the +// registry's base URL serves the protocol at /v1/modules and /v1/providers. +func TestServiceDiscoveryFallsBackToTheBaseURL(t *testing.T) { + for name, document := range map[string]string{ + "no document": "", + "no entry": `{"login.v1": {"client": "terraform-cli"}}`, + "entry string": `{"providers.v1": 42, "modules.v1": ["x"]}`, + // A host that answers unknown paths with an HTML page. + "not json": `app`, + // Refused, as by a host that serves no discovery. + "forbidden": "403", + } { + t.Run(name, func(t *testing.T) { + routes := map[string]string{ + "/registry/v1/providers/hashicorp/random/versions": providerVersionsJSON, + "/registry/v1/modules/hashicorp/consul/aws/versions": `{"modules": [{"versions": [{"version": "0.11.0"}]}]}`, + } + if document != "" && document != "403" { + routes[discoveryPath] = document + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == discoveryPath && document == "403" { + w.WriteHeader(http.StatusForbidden) + return + } + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + r := New(server.URL+"/registry", core.DefaultClient()) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("provider versions = %v, %v; want 3", versions, err) + } + if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 { + t.Errorf("module versions = %v, %v; want 1", versions, err) + } + }) + } +} + +// A discovery document that can't be read is an error, and is read again +// next time rather than replaced by the fallback for good. +func TestServiceDiscoveryFailureIsRetried(t *testing.T) { + var failing atomic.Bool + failing.Store(true) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == discoveryPath && failing.Load(): + w.WriteHeader(http.StatusInternalServerError) + case r.URL.Path == discoveryPath: + _, _ = w.Write([]byte(`{"providers.v1": "/p/"}`)) + case r.URL.Path == "/p/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + + client := core.DefaultClient() + client.MaxRetries = 0 + r := New(server.URL, client) + if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err == nil { + t.Fatal("FetchVersions succeeded although discovery failed") + } + failing.Store(false) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("after recovery: %v, %v; want 3 versions", versions, err) + } +} + +// Only http and https service URLs are followed. +func TestServiceDiscoveryRefusesOtherSchemes(t *testing.T) { + for _, location := range []string{"file:///etc/", "ftp://registry.example.com/providers/"} { + server := providerServer(t, map[string]string{discoveryPath: `{"providers.v1": "` + location + `"}`}) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err == nil || !strings.Contains(err.Error(), "providers.v1") { + t.Errorf("%s: err = %v, want a refused providers.v1", location, err) + } + } +} + +// Service URLs are relative to the document's final URL, after redirects. +func TestServiceDiscoveryResolvesAgainstTheRedirectedDocument(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case discoveryPath: + http.Redirect(w, r, "/discovery/terraform.json", http.StatusFound) + case "/discovery/terraform.json": + _, _ = w.Write([]byte(`{"providers.v1": "./providers/"}`)) + case "/discovery/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + if versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("versions = %v, %v; want 3", versions, err) + } +} + +type recordingTransport struct { + paths []string +} + +func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) { + rt.paths = append(rt.paths, r.URL.Host+r.URL.Path) + body := `{"modules": [{"versions": [{"version": "0.11.0"}]}]}` + if strings.HasPrefix(r.URL.Path, "/v1/providers/") { + body = providerVersionsJSON + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil +} + +// registry.terraform.io serves the protocol at /v1/modules/ and +// /v1/providers/, so the default registry is asked without discovery: the +// PURL helpers build a new Registry per call. +func TestDefaultRegistrySkipsDiscovery(t *testing.T) { + rt := &recordingTransport{} + r := New(DefaultURL, core.NewClient(client.WithTransport(rt))) + if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil { + t.Fatalf("provider versions: %v", err) + } + if _, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil { + t.Fatalf("module versions: %v", err) + } + want := []string{ + "registry.terraform.io/v1/providers/hashicorp/random/versions", + "registry.terraform.io/v1/modules/hashicorp/consul/aws/versions", + } + if strings.Join(rt.paths, " ") != strings.Join(want, " ") { + t.Errorf("requests = %v, want %v", rt.paths, want) + } +} diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go new file mode 100644 index 0000000..08be186 --- /dev/null +++ b/internal/terraform/provider.go @@ -0,0 +1,157 @@ +package terraform + +import ( + "context" + "fmt" + "net/url" + "sort" + "strings" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/urlparser" + "github.com/git-pkgs/vers" +) + +// parseProviderName parses "namespace/type". +func parseProviderName(name string) (namespace, typ string, ok bool) { + namespace, typ, ok = strings.Cut(name, "/") + if !ok || namespace == "" || typ == "" || strings.Contains(typ, "/") { + return "", "", false + } + return namespace, typ, true +} + +type providerResponse struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Source string `json:"source"` + Downloads int64 `json:"downloads"` + Tier string `json:"tier"` +} + +type providerVersionsResponse struct { + Versions []providerVersion `json:"versions"` +} + +type providerVersion struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []providerPlatform `json:"platforms"` +} + +type providerPlatform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +func (r *Registry) providerURL(ctx context.Context, namespace, typ string) (string, error) { + providers, err := r.serviceURL(ctx, providersService) + if err != nil { + return "", err + } + return providers + url.PathEscape(namespace) + "/" + url.PathEscape(typ), nil +} + +// fetchProviderPackage reads registry.terraform.io's provider details; other +// registries serve none, and the package comes from the version list. +func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + providerURL, err := r.providerURL(ctx, namespace, typ) + if err != nil { + return nil, err + } + var resp providerResponse + err = r.client.GetJSON(ctx, providerURL, &resp) + if refused(err) { + return r.fetchProtocolProviderPackage(ctx, name, namespace, typ) + } + if err != nil { + return nil, err + } + return &core.Package{ + Name: name, + Description: resp.Description, + Homepage: r.urls.Registry(name, ""), + Repository: urlparser.Parse(resp.Source), + Namespace: namespace, + LatestVersion: resp.Version, + Metadata: map[string]any{ + "tier": resp.Tier, + "downloads": resp.Downloads, + }, + }, nil +} + +// fetchProtocolProviderPackage describes a provider from its version list +// alone, for registries that serve no details. +func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + versions, err := r.fetchProviderVersions(ctx, name, namespace, typ) + if err != nil { + return nil, err + } + pkg := &core.Package{Name: name, Homepage: r.urls.Registry(name, ""), Namespace: namespace} + if latest := core.SelectLatestVersion(versions, ecosystem, ""); latest != nil { + pkg.LatestVersion = latest.Number + } + return pkg, nil +} + +func (r *Registry) fetchProviderVersions(ctx context.Context, name, namespace, typ string) ([]core.Version, error) { + providerURL, err := r.providerURL(ctx, namespace, typ) + if err != nil { + return nil, err + } + var resp providerVersionsResponse + if err := r.client.GetJSON(ctx, providerURL+"/versions", &resp); err != nil { + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return nil, err + } + + versions := make([]core.Version, 0, len(resp.Versions)) + for _, v := range resp.Versions { + if v.Version == "" { + continue + } + platforms := make([]string, 0, len(v.Platforms)) + for _, p := range v.Platforms { + platforms = append(platforms, p.OS+"_"+p.Arch) + } + versions = append(versions, core.Version{ + Number: v.Version, + Metadata: map[string]any{ + "protocols": v.Protocols, + "platforms": platforms, + }, + }) + } + // The protocol lists versions in no particular order. + sort.SliceStable(versions, func(i, j int) bool { + return vers.CompareWithScheme(versions[i].Number, versions[j].Number, ecosystem) > 0 + }) + return versions, nil +} + +// providerPage is the provider's page on registry.terraform.io, or "" for +// another registry, which has no pages at these paths. +func (u *URLs) providerPage(namespace, typ, version, suffix string) string { + if u.baseURL != DefaultURL { + return "" + } + if version == "" { + version = "latest" + } + return fmt.Sprintf("%s/providers/%s/%s/%s%s", DefaultURL, namespace, typ, version, suffix) +} + +// providerPURL carries repository_url for any registry but the default one. +func (u *URLs) providerPURL(namespace, typ, version string) string { + var qualifiers map[string]string + if u.baseURL != DefaultURL { + qualifiers = map[string]string{"repository_url": u.baseURL} + } + return purl.New(ecosystem, namespace, typ, version, qualifiers).String() +} diff --git a/internal/terraform/provider_test.go b/internal/terraform/provider_test.go new file mode 100644 index 0000000..a842827 --- /dev/null +++ b/internal/terraform/provider_test.go @@ -0,0 +1,226 @@ +package terraform + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/git-pkgs/registries/internal/core" +) + +// Trimmed from registry.terraform.io's /v1/providers/hashicorp/random. +const providerDetailJSON = `{ + "id": "hashicorp/random/3.9.1", + "owner": "hashicorp", + "namespace": "hashicorp", + "name": "random", + "version": "3.9.1", + "description": "terraform-provider-random", + "source": "https://github.com/hashicorp/terraform-provider-random", + "published_at": "2026-09-11T15:17:02Z", + "downloads": 3603195929, + "tier": "official", + "versions": ["3.6.0", "3.9.1"] +}` + +// The Provider Registry Protocol's version list, which every registry +// serves, trimmed from registry.opentofu.org. Versions are unordered. +const providerVersionsJSON = `{ + "versions": [ + {"version": "3.6.0", "protocols": ["5.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}, {"os": "darwin", "arch": "arm64"}]}, + {"version": "3.10.0-beta1", "protocols": ["5.0"], "platforms": [{"os": "linux", "arch": "amd64"}]}, + {"version": "3.9.1", "protocols": ["5.0", "6.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}]} + ] +}` + +// providerServer serves path → body, and 404 for anything else. +func providerServer(t *testing.T, routes map[string]string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFetchProviderPackage(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random": providerDetailJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.Description != "terraform-provider-random" { + t.Errorf("description = %q", pkg.Description) + } + if pkg.Repository != "https://github.com/hashicorp/terraform-provider-random" { + t.Errorf("repository = %q", pkg.Repository) + } + if pkg.LatestVersion != "3.9.1" { + t.Errorf("latest = %q, want 3.9.1", pkg.LatestVersion) + } + if pkg.Metadata["tier"] != "official" || pkg.Metadata["downloads"] != int64(3603195929) { + t.Errorf("metadata = %v", pkg.Metadata) + } +} + +// Registries other than registry.terraform.io, such as registry.opentofu.org, +// only serve the Provider Registry Protocol, so the package falls back to +// its version list. +func TestFetchProviderPackageFromProtocolOnlyRegistry(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("latest = %q, want the highest version, 3.10.0-beta1", pkg.LatestVersion) + } +} + +// The details endpoint is an extension of registry.terraform.io, not part +// of the protocol, so any refusal of it, not only a 404, falls back to the +// version list. +func TestFetchProviderPackageFallsBackOnAnyRefusal(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/providers/hashicorp/random": + w.WriteHeader(http.StatusMethodNotAllowed) + case "/v1/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil || pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("FetchPackage = %+v, %v; want the version list's latest", pkg, err) + } +} + +func TestFetchProviderPackageNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +func TestFetchProviderVersions(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + var got []string + for _, v := range versions { + got = append(got, v.Number) + } + if want := []string{"3.10.0-beta1", "3.9.1", "3.6.0"}; !reflect.DeepEqual(got, want) { + t.Fatalf("versions = %v, want %v (newest first, by version not by string)", got, want) + } + v := versions[1] + if !reflect.DeepEqual(v.Metadata["protocols"], []string{"5.0", "6.0"}) { + t.Errorf("protocols = %v", v.Metadata["protocols"]) + } + if !reflect.DeepEqual(versions[2].Metadata["platforms"], []string{"linux_amd64", "darwin_arm64"}) { + t.Errorf("platforms = %v", versions[2].Metadata["platforms"]) + } +} + +func TestFetchProviderVersionsNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +// Providers declare no dependencies, so no request is made. +func TestFetchProviderDependencies(t *testing.T) { + server := providerServer(t, nil) + deps, err := New(server.URL, core.DefaultClient()).FetchDependencies(context.Background(), "hashicorp/random", "3.6.0") + if err != nil || len(deps) != 0 { + t.Errorf("deps = %v, %v; want none", deps, err) + } +} + +func TestFetchProviderMaintainers(t *testing.T) { + maintainers, err := New("", nil).FetchMaintainers(context.Background(), "hashicorp/random") + if err != nil || len(maintainers) != 1 || maintainers[0].Login != "hashicorp" { + t.Errorf("maintainers = %v, %v; want the namespace", maintainers, err) + } +} + +func TestParseProviderName(t *testing.T) { + tests := []struct { + input, namespace, typ string + ok bool + }{ + {"hashicorp/random", "hashicorp", "random", true}, + {"integrations/github", "integrations", "github", true}, + {"random", "", "", false}, + {"hashicorp/consul/aws", "", "", false}, + {"/random", "", "", false}, + {"hashicorp/", "", "", false}, + } + for _, tt := range tests { + namespace, typ, ok := parseProviderName(tt.input) + if namespace != tt.namespace || typ != tt.typ || ok != tt.ok { + t.Errorf("parseProviderName(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.input, namespace, typ, ok, tt.namespace, tt.typ, tt.ok) + } + } +} + +func TestProviderURLs(t *testing.T) { + urls := New("https://registry.terraform.io", nil).URLs() + other := New("https://registry.opentofu.org", nil).URLs() + tests := []struct { + name, got, want string + }{ + {"registry", urls.Registry("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0"}, + {"registry latest", urls.Registry("hashicorp/random", ""), "https://registry.terraform.io/providers/hashicorp/random/latest"}, + {"documentation", urls.Documentation("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs"}, + // Archives are per platform; the download endpoint names them. + {"download", urls.Download("hashicorp/random", "3.6.0"), ""}, + {"purl", urls.PURL("hashicorp/random", "3.6.0"), "pkg:terraform/hashicorp/random@3.6.0"}, + {"purl without version", urls.PURL("hashicorp/random", ""), "pkg:terraform/hashicorp/random"}, + // Other registries have no pages at these paths. + {"other registry page", other.Registry("hashicorp/random", "3.6.0"), ""}, + {"other registry purl", other.PURL("hashicorp/random", "3.6.0"), + "pkg:terraform/hashicorp/random@3.6.0?repository_url=https:%2F%2Fregistry.opentofu.org"}, + } + for _, tt := range tests { + if tt.got != tt.want { + t.Errorf("%s = %q, want %q", tt.name, tt.got, tt.want) + } + } +} + +func TestFetchProviderVersionsSkipsEntriesWithoutAVersion(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": `{"versions": [{"version": ""}, {"version": "3.6.0"}]}`}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil || len(versions) != 1 || versions[0].Number != "3.6.0" { + t.Errorf("versions = %v, %v; want only 3.6.0", versions, err) + } +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index a2e74b1..248dae1 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -3,9 +3,11 @@ package terraform import ( "context" + "errors" "fmt" "sort" "strings" + "sync" "github.com/git-pkgs/registries/internal/core" "github.com/git-pkgs/registries/internal/urlparser" @@ -22,10 +24,15 @@ func init() { }) } +var errInvalidName = errors.New("terraform name must be 'namespace/type' for a provider or 'namespace/name/provider' for a module") + type Registry struct { baseURL string client *core.Client urls *URLs + + mu sync.Mutex + services *services } func New(baseURL string, client *core.Client) *Registry { @@ -108,12 +115,19 @@ type providerEntry struct { } func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderPackage(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s", r.baseURL, namespace, moduleName, provider) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s", modules, namespace, moduleName, provider) var resp moduleResponse if err := r.client.GetJSON(ctx, url, &resp); err != nil { @@ -141,12 +155,19 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package } func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderVersions(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", r.baseURL, namespace, moduleName, provider) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s/versions", modules, namespace, moduleName, provider) var resp moduleVersionsResponse if err := r.client.GetJSON(ctx, url, &resp); err != nil { @@ -176,12 +197,20 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi } func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + // Providers declare no dependencies. + if _, _, ok := parseProviderName(name); ok { + return nil, nil + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", r.baseURL, namespace, moduleName, provider, version) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s/%s", modules, namespace, moduleName, provider, version) var resp versionEntry if err := r.client.GetJSON(ctx, url, &resp); err != nil { @@ -224,6 +253,9 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string) func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { namespace, _, _, ok := parseModuleName(name) + if !ok { + namespace, _, ok = parseProviderName(name) + } if !ok { return nil, nil } @@ -240,6 +272,9 @@ type URLs struct { } func (u *URLs) Registry(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "") + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" @@ -250,7 +285,12 @@ func (u *URLs) Registry(name, version string) string { return fmt.Sprintf("https://registry.terraform.io/modules/%s/%s/%s", namespace, moduleName, provider) } +// Download is the module's download endpoint. A provider has an archive per +// platform, named by the download endpoint for that platform, so it has none. func (u *URLs) Download(name, version string) string { + if _, _, ok := parseProviderName(name); ok { + return "" + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok || version == "" { return "" @@ -259,10 +299,16 @@ func (u *URLs) Download(name, version string) string { } func (u *URLs) Documentation(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "/docs") + } return u.Registry(name, version) } func (u *URLs) PURL(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPURL(namespace, typ, version) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" diff --git a/internal/terraform/terraform_test.go b/internal/terraform/terraform_test.go index 9025e3c..d4a75da 100644 --- a/internal/terraform/terraform_test.go +++ b/internal/terraform/terraform_test.go @@ -12,6 +12,10 @@ import ( func TestFetchPackage(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == discoveryPath { + w.WriteHeader(404) + return + } if r.URL.Path != "/v1/modules/hashicorp/consul/aws" { t.Errorf("unexpected path: %s", r.URL.Path) w.WriteHeader(404)