From dc0dcc228b5503780ec9ff3e2a1353ac4e1761c1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 19:28:01 +0100 Subject: [PATCH 1/4] Support Terraform providers next to modules A two-segment name, namespace/type, is a provider. Versions come from the Provider Registry Protocol's version list, which every registry serves, with each version's protocols and platforms in its metadata. The package reads registry.terraform.io's provider details and falls back to the version list on registries that have none, such as registry.opentofu.org. Providers declare no dependencies, and have no single download URL, since each platform has its own archive. --- README.md | 2 +- internal/terraform/provider.go | 154 +++++++++++++++++++++ internal/terraform/provider_test.go | 205 ++++++++++++++++++++++++++++ internal/terraform/terraform.go | 36 ++++- 4 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 internal/terraform/provider.go create mode 100644 internal/terraform/provider_test.go diff --git a/README.md b/README.md index d953bfe..96be189 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Go | `pkg:golang/github.com/gorilla/mux@v1.8.0` | | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` | | RubyGems | `pkg:gem/rails@7.1.0` | -| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | +| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` (module), `pkg:terraform/hashicorp/random@3.6.0` (provider) | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go new file mode 100644 index 0000000..2932439 --- /dev/null +++ b/internal/terraform/provider.go @@ -0,0 +1,154 @@ +package terraform + +import ( + "context" + "fmt" + "net/url" + "sort" + "strings" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/urlparser" + "github.com/git-pkgs/vers" +) + +// Providers are named namespace/type, as in a provider source address +// without its hostname (hashicorp/random); modules have three segments. +// +// Only the Provider Registry Protocol's version list +// (/v1/providers/{namespace}/{type}/versions) is served by every registry. +// registry.terraform.io also serves the provider's details at +// /v1/providers/{namespace}/{type}; others, such as registry.opentofu.org, +// answer 404 there. + +// parseProviderName parses "namespace/type". +func parseProviderName(name string) (namespace, typ string, ok bool) { + namespace, typ, ok = strings.Cut(name, "/") + if !ok || namespace == "" || typ == "" || strings.Contains(typ, "/") { + return "", "", false + } + return namespace, typ, true +} + +type providerResponse struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt string `json:"published_at"` + Downloads int64 `json:"downloads"` + Tier string `json:"tier"` +} + +type providerVersionsResponse struct { + Versions []providerVersion `json:"versions"` +} + +type providerVersion struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []providerPlatform `json:"platforms"` +} + +type providerPlatform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +func (r *Registry) providerURL(namespace, typ string) string { + return fmt.Sprintf("%s/v1/providers/%s/%s", r.baseURL, url.PathEscape(namespace), url.PathEscape(typ)) +} + +func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + var resp providerResponse + err := r.client.GetJSON(ctx, r.providerURL(namespace, typ), &resp) + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return r.fetchProtocolProviderPackage(ctx, name, namespace) + } + if err != nil { + return nil, err + } + return &core.Package{ + Name: name, + Description: resp.Description, + Homepage: r.urls.Registry(name, ""), + Repository: urlparser.Parse(resp.Source), + Namespace: namespace, + LatestVersion: resp.Version, + Metadata: map[string]any{ + "tier": resp.Tier, + "downloads": resp.Downloads, + }, + }, nil +} + +// fetchProtocolProviderPackage describes a provider from its version list +// alone, for registries that serve no details. +func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace string) (*core.Package, error) { + typ := strings.TrimPrefix(name, namespace+"/") + versions, err := r.fetchProviderVersions(ctx, name, namespace, typ) + if err != nil { + return nil, err + } + pkg := &core.Package{Name: name, Homepage: r.urls.Registry(name, ""), Namespace: namespace} + if latest := core.SelectLatestVersion(versions, ecosystem, ""); latest != nil { + pkg.LatestVersion = latest.Number + } + return pkg, nil +} + +func (r *Registry) fetchProviderVersions(ctx context.Context, name, namespace, typ string) ([]core.Version, error) { + var resp providerVersionsResponse + if err := r.client.GetJSON(ctx, r.providerURL(namespace, typ)+"/versions", &resp); err != nil { + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return nil, err + } + + versions := make([]core.Version, 0, len(resp.Versions)) + for _, v := range resp.Versions { + if v.Version == "" { + continue + } + platforms := make([]string, 0, len(v.Platforms)) + for _, p := range v.Platforms { + platforms = append(platforms, p.OS+"_"+p.Arch) + } + versions = append(versions, core.Version{ + Number: v.Version, + Metadata: map[string]any{ + "protocols": v.Protocols, + "platforms": platforms, + }, + }) + } + // The protocol lists versions in no particular order. + sort.SliceStable(versions, func(i, j int) bool { + return vers.CompareWithScheme(versions[i].Number, versions[j].Number, ecosystem) > 0 + }) + return versions, nil +} + +// providerPage is the provider's page on registry.terraform.io, or "" for +// another registry, which has no pages at these paths. +func (u *URLs) providerPage(namespace, typ, version, suffix string) string { + if u.baseURL != DefaultURL { + return "" + } + if version == "" { + version = "latest" + } + return fmt.Sprintf("%s/providers/%s/%s/%s%s", DefaultURL, namespace, typ, version, suffix) +} + +// providerPURL carries repository_url for any registry but the default one. +func (u *URLs) providerPURL(namespace, typ, version string) string { + var qualifiers map[string]string + if u.baseURL != DefaultURL { + qualifiers = map[string]string{"repository_url": u.baseURL} + } + return purl.New(ecosystem, namespace, typ, version, qualifiers).String() +} diff --git a/internal/terraform/provider_test.go b/internal/terraform/provider_test.go new file mode 100644 index 0000000..85ee3cd --- /dev/null +++ b/internal/terraform/provider_test.go @@ -0,0 +1,205 @@ +package terraform + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/git-pkgs/registries/internal/core" +) + +// Trimmed from registry.terraform.io's /v1/providers/hashicorp/random. +const providerDetailJSON = `{ + "id": "hashicorp/random/3.9.1", + "owner": "hashicorp", + "namespace": "hashicorp", + "name": "random", + "version": "3.9.1", + "description": "terraform-provider-random", + "source": "https://github.com/hashicorp/terraform-provider-random", + "published_at": "2026-09-11T15:17:02Z", + "downloads": 3603195929, + "tier": "official", + "versions": ["3.6.0", "3.9.1"] +}` + +// The Provider Registry Protocol's version list, which every registry +// serves, trimmed from registry.opentofu.org. Versions are unordered. +const providerVersionsJSON = `{ + "versions": [ + {"version": "3.6.0", "protocols": ["5.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}, {"os": "darwin", "arch": "arm64"}]}, + {"version": "3.10.0-beta1", "protocols": ["5.0"], "platforms": [{"os": "linux", "arch": "amd64"}]}, + {"version": "3.9.1", "protocols": ["5.0", "6.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}]} + ] +}` + +// providerServer serves path → body, and 404 for anything else. +func providerServer(t *testing.T, routes map[string]string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFetchProviderPackage(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random": providerDetailJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.Description != "terraform-provider-random" { + t.Errorf("description = %q", pkg.Description) + } + if pkg.Repository != "https://github.com/hashicorp/terraform-provider-random" { + t.Errorf("repository = %q", pkg.Repository) + } + if pkg.LatestVersion != "3.9.1" { + t.Errorf("latest = %q, want 3.9.1", pkg.LatestVersion) + } + if pkg.Metadata["tier"] != "official" || pkg.Metadata["downloads"] != int64(3603195929) { + t.Errorf("metadata = %v", pkg.Metadata) + } +} + +// Registries other than registry.terraform.io, such as registry.opentofu.org, +// only serve the Provider Registry Protocol, so the package falls back to +// its version list. +func TestFetchProviderPackageFromProtocolOnlyRegistry(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("latest = %q, want the highest version, 3.10.0-beta1", pkg.LatestVersion) + } +} + +func TestFetchProviderPackageNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +func TestFetchProviderVersions(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + var got []string + for _, v := range versions { + got = append(got, v.Number) + } + if want := []string{"3.10.0-beta1", "3.9.1", "3.6.0"}; !reflect.DeepEqual(got, want) { + t.Fatalf("versions = %v, want %v (newest first, by version not by string)", got, want) + } + v := versions[1] + if !reflect.DeepEqual(v.Metadata["protocols"], []string{"5.0", "6.0"}) { + t.Errorf("protocols = %v", v.Metadata["protocols"]) + } + if !reflect.DeepEqual(versions[2].Metadata["platforms"], []string{"linux_amd64", "darwin_arm64"}) { + t.Errorf("platforms = %v", versions[2].Metadata["platforms"]) + } +} + +func TestFetchProviderVersionsNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +// Providers declare no dependencies, so no request is made. +func TestFetchProviderDependencies(t *testing.T) { + server := providerServer(t, nil) + deps, err := New(server.URL, core.DefaultClient()).FetchDependencies(context.Background(), "hashicorp/random", "3.6.0") + if err != nil || len(deps) != 0 { + t.Errorf("deps = %v, %v; want none", deps, err) + } +} + +func TestFetchProviderMaintainers(t *testing.T) { + maintainers, err := New("", nil).FetchMaintainers(context.Background(), "hashicorp/random") + if err != nil || len(maintainers) != 1 || maintainers[0].Login != "hashicorp" { + t.Errorf("maintainers = %v, %v; want the namespace", maintainers, err) + } +} + +func TestParseProviderName(t *testing.T) { + tests := []struct { + input, namespace, typ string + ok bool + }{ + {"hashicorp/random", "hashicorp", "random", true}, + {"integrations/github", "integrations", "github", true}, + {"random", "", "", false}, + {"hashicorp/consul/aws", "", "", false}, + {"/random", "", "", false}, + {"hashicorp/", "", "", false}, + } + for _, tt := range tests { + namespace, typ, ok := parseProviderName(tt.input) + if namespace != tt.namespace || typ != tt.typ || ok != tt.ok { + t.Errorf("parseProviderName(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.input, namespace, typ, ok, tt.namespace, tt.typ, tt.ok) + } + } +} + +func TestProviderURLs(t *testing.T) { + urls := New("https://registry.terraform.io", nil).URLs() + other := New("https://registry.opentofu.org", nil).URLs() + tests := []struct { + name, got, want string + }{ + {"registry", urls.Registry("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0"}, + {"registry latest", urls.Registry("hashicorp/random", ""), "https://registry.terraform.io/providers/hashicorp/random/latest"}, + {"documentation", urls.Documentation("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs"}, + // Archives are per platform; the download endpoint names them. + {"download", urls.Download("hashicorp/random", "3.6.0"), ""}, + {"purl", urls.PURL("hashicorp/random", "3.6.0"), "pkg:terraform/hashicorp/random@3.6.0"}, + {"purl without version", urls.PURL("hashicorp/random", ""), "pkg:terraform/hashicorp/random"}, + // Other registries have no pages at these paths. + {"other registry page", other.Registry("hashicorp/random", "3.6.0"), ""}, + {"other registry purl", other.PURL("hashicorp/random", "3.6.0"), + "pkg:terraform/hashicorp/random@3.6.0?repository_url=https:%2F%2Fregistry.opentofu.org"}, + } + for _, tt := range tests { + if tt.got != tt.want { + t.Errorf("%s = %q, want %q", tt.name, tt.got, tt.want) + } + } +} + +func TestFetchProviderVersionsSkipsEntriesWithoutAVersion(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": `{"versions": [{"version": ""}, {"version": "3.6.0"}]}`}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil || len(versions) != 1 || versions[0].Number != "3.6.0" { + t.Errorf("versions = %v, %v; want only 3.6.0", versions, err) + } +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index a2e74b1..e317528 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -3,6 +3,7 @@ package terraform import ( "context" + "errors" "fmt" "sort" "strings" @@ -22,6 +23,8 @@ func init() { }) } +var errInvalidName = errors.New("terraform name must be 'namespace/type' for a provider or 'namespace/name/provider' for a module") + type Registry struct { baseURL string client *core.Client @@ -108,9 +111,12 @@ type providerEntry struct { } func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderPackage(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s", r.baseURL, namespace, moduleName, provider) @@ -141,9 +147,12 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package } func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderVersions(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", r.baseURL, namespace, moduleName, provider) @@ -176,9 +185,13 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi } func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + // Providers declare no dependencies. + if _, _, ok := parseProviderName(name); ok { + return nil, nil + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", r.baseURL, namespace, moduleName, provider, version) @@ -224,6 +237,9 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string) func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { namespace, _, _, ok := parseModuleName(name) + if !ok { + namespace, _, ok = parseProviderName(name) + } if !ok { return nil, nil } @@ -240,6 +256,9 @@ type URLs struct { } func (u *URLs) Registry(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "") + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" @@ -250,7 +269,12 @@ func (u *URLs) Registry(name, version string) string { return fmt.Sprintf("https://registry.terraform.io/modules/%s/%s/%s", namespace, moduleName, provider) } +// Download is the module's download endpoint. A provider has an archive per +// platform, named by the download endpoint for that platform, so it has none. func (u *URLs) Download(name, version string) string { + if _, _, ok := parseProviderName(name); ok { + return "" + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok || version == "" { return "" @@ -259,10 +283,16 @@ func (u *URLs) Download(name, version string) string { } func (u *URLs) Documentation(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "/docs") + } return u.Registry(name, version) } func (u *URLs) PURL(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPURL(namespace, typ, version) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" From 542631a174be752608721b9687ee164ac30df567 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 18:44:28 +0100 Subject: [PATCH 2/4] Find the Terraform module and provider APIs through service discovery A registry names where its module and provider APIs live in /.well-known/terraform.json. The client now reads that document once and sends requests to the modules.v1 and providers.v1 URLs it names, relative or absolute, http or https. Without the document, or without an entry, it keeps the {base}/v1/modules/ and {base}/v1/providers/ paths. A failure to read the document is returned and retried on the next call. --- README.md | 2 + internal/terraform/discovery.go | 75 +++++++++++++++++ internal/terraform/discovery_test.go | 115 +++++++++++++++++++++++++++ internal/terraform/provider.go | 20 ++++- internal/terraform/terraform.go | 22 ++++- internal/terraform/terraform_test.go | 4 + 6 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 internal/terraform/discovery.go create mode 100644 internal/terraform/discovery_test.go diff --git a/README.md b/README.md index c7a269a..27cca18 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,8 @@ The RPM URL is one repository, the directory that holds `repodata/`, such as `ht GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. +The Terraform URL is a module and provider registry, such as `https://registry.opentofu.org`. The client reads the registry's service discovery document, `/.well-known/terraform.json` on its host, once, and sends module and provider requests to the `modules.v1` and `providers.v1` URLs it names. A registry without the document, or without one of the entries, is read at `{URL}/v1/modules/` and `{URL}/v1/providers/`. `URLs().Download` builds a module's download URL from the base URL without reading the document. + VS Code extensions are read from [Open VSX](https://open-vsx.org), or from a self-hosted Open VSX server given as the base URL. The purl spec makes the Visual Studio Marketplace the default for `vscode-extension`, but the client has no Marketplace support, so a PURL without `repository_url` also resolves to Open VSX. `URLs().PURL` always adds the registry as `repository_url`. Names are `publisher/name` or VS Code's `publisher.name`. An extension can publish a separate build for each target platform, such as `linux-x64` or `darwin-arm64`. Each version from `FetchVersions` and `FetchVersion` carries one artifact per build, with `platform`, `signature` (the `.sigzip`) and `sha256` (the URL of the checksum file) in its metadata. `URLs().Download` returns the universal build, which a version built only for specific platforms doesn't have; the `platform` qualifier, as in `pkg:vscode-extension/muhammad-sammy/csharp@2.160.9?platform=linux-x64`, makes `fetch.Resolver` pick the build for that platform, or the universal one when there is none. `FetchDependencies` lists both the extensions a version depends on and the ones it bundles as an extension pack. `FetchMaintainers` returns the user who published the latest version. ### Swift registries diff --git a/internal/terraform/discovery.go b/internal/terraform/discovery.go new file mode 100644 index 0000000..0514921 --- /dev/null +++ b/internal/terraform/discovery.go @@ -0,0 +1,75 @@ +package terraform + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/git-pkgs/registries/internal/core" +) + +const ( + discoveryPath = "/.well-known/terraform.json" + modulesService = "modules.v1" + providersService = "providers.v1" +) + +// services are the base URLs of a registry's module and provider APIs, each +// ending in a slash. +type services struct { + modules string + providers string +} + +// serviceURL returns the base URL of a service, read from the registry's +// service discovery document at its host's /.well-known/terraform.json. A +// registry without the document, or without an entry for the service, +// serves the protocol at {base}/v1/modules/ and {base}/v1/providers/. The +// document is read once; a failure to read it is returned and retried on +// the next call. +func (r *Registry) serviceURL(ctx context.Context, service string) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.services == nil { + found, err := r.discover(ctx) + if err != nil { + return "", err + } + r.services = found + } + if service == modulesService { + return r.services.modules, nil + } + return r.services.providers, nil +} + +func (r *Registry) discover(ctx context.Context) (*services, error) { + found := &services{modules: r.baseURL + "/v1/modules/", providers: r.baseURL + "/v1/providers/"} + base, err := url.Parse(r.baseURL) + if err != nil || base.Host == "" { + return found, nil //nolint:nilerr // a base URL without a host has no discovery document + } + document := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: discoveryPath} + var entries map[string]json.RawMessage + if err := r.client.GetJSON(ctx, document.String(), &entries); err != nil { + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return found, nil + } + return nil, fmt.Errorf("terraform service discovery: %w", err) + } + for service, target := range map[string]*string{modulesService: &found.modules, providersService: &found.providers} { + var location string + // Other services, and entries that aren't a URL, are left alone. + if raw, ok := entries[service]; !ok || json.Unmarshal(raw, &location) != nil || location == "" { + continue + } + resolved, err := document.Parse(location) + if err != nil || (resolved.Scheme != "https" && resolved.Scheme != "http") || resolved.Host == "" { + return nil, fmt.Errorf("terraform service discovery: %s is %q, not an http or https URL", service, location) + } + *target = strings.TrimSuffix(resolved.String(), "/") + "/" + } + return found, nil +} diff --git a/internal/terraform/discovery_test.go b/internal/terraform/discovery_test.go new file mode 100644 index 0000000..00f9bcc --- /dev/null +++ b/internal/terraform/discovery_test.go @@ -0,0 +1,115 @@ +package terraform + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries/internal/core" +) + +// A registry names where its module and provider APIs live in its service +// discovery document. Requests go there, and the document is read once. +func TestServiceDiscovery(t *testing.T) { + var discoveries atomic.Int32 + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case discoveryPath: + discoveries.Add(1) + // A relative and an absolute URL, as the protocol allows both. + _, _ = w.Write([]byte(`{"modules.v1": "/api/terraform/modules/", "providers.v1": "` + server.URL + `/api/terraform/providers/"}`)) + case "/api/terraform/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + case "/api/terraform/modules/hashicorp/consul/aws/versions": + _, _ = w.Write([]byte(`{"modules": [{"versions": [{"version": "0.11.0"}]}]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + + r := New(server.URL, core.DefaultClient()) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("provider versions = %v, %v; want 3", versions, err) + } + if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 { + t.Errorf("module versions = %v, %v; want 1", versions, err) + } + if got := discoveries.Load(); got != 1 { + t.Errorf("discovery read %d times, want once", got) + } +} + +// Without a discovery document, or without an entry for a service, the +// registry's base URL serves the protocol at /v1/modules and /v1/providers. +func TestServiceDiscoveryFallsBackToTheBaseURL(t *testing.T) { + for name, document := range map[string]string{ + "no document": "", + "no entry": `{"login.v1": {"client": "terraform-cli"}}`, + "entry string": `{"providers.v1": 42, "modules.v1": ["x"]}`, + } { + t.Run(name, func(t *testing.T) { + routes := map[string]string{ + "/registry/v1/providers/hashicorp/random/versions": providerVersionsJSON, + "/registry/v1/modules/hashicorp/consul/aws/versions": `{"modules": [{"versions": [{"version": "0.11.0"}]}]}`, + } + if document != "" { + routes[discoveryPath] = document + } + server := providerServer(t, routes) + r := New(server.URL+"/registry", core.DefaultClient()) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("provider versions = %v, %v; want 3", versions, err) + } + if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 { + t.Errorf("module versions = %v, %v; want 1", versions, err) + } + }) + } +} + +// A discovery document that can't be read is an error, and is read again +// next time rather than replaced by the fallback for good. +func TestServiceDiscoveryFailureIsRetried(t *testing.T) { + var failing atomic.Bool + failing.Store(true) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == discoveryPath && failing.Load(): + w.WriteHeader(http.StatusInternalServerError) + case r.URL.Path == discoveryPath: + _, _ = w.Write([]byte(`{"providers.v1": "/p/"}`)) + case r.URL.Path == "/p/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + + client := core.DefaultClient() + client.MaxRetries = 0 + r := New(server.URL, client) + if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err == nil { + t.Fatal("FetchVersions succeeded although discovery failed") + } + failing.Store(false) + if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("after recovery: %v, %v; want 3 versions", versions, err) + } +} + +// Only http and https service URLs are followed. +func TestServiceDiscoveryRefusesOtherSchemes(t *testing.T) { + for _, location := range []string{"file:///etc/", "ftp://registry.example.com/providers/"} { + server := providerServer(t, map[string]string{discoveryPath: `{"providers.v1": "` + location + `"}`}) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err == nil || !strings.Contains(err.Error(), "providers.v1") { + t.Errorf("%s: err = %v, want a refused providers.v1", location, err) + } + } +} diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go index 2932439..8d3aede 100644 --- a/internal/terraform/provider.go +++ b/internal/terraform/provider.go @@ -57,13 +57,21 @@ type providerPlatform struct { Arch string `json:"arch"` } -func (r *Registry) providerURL(namespace, typ string) string { - return fmt.Sprintf("%s/v1/providers/%s/%s", r.baseURL, url.PathEscape(namespace), url.PathEscape(typ)) +func (r *Registry) providerURL(ctx context.Context, namespace, typ string) (string, error) { + providers, err := r.serviceURL(ctx, providersService) + if err != nil { + return "", err + } + return providers + url.PathEscape(namespace) + "/" + url.PathEscape(typ), nil } func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + providerURL, err := r.providerURL(ctx, namespace, typ) + if err != nil { + return nil, err + } var resp providerResponse - err := r.client.GetJSON(ctx, r.providerURL(namespace, typ), &resp) + err = r.client.GetJSON(ctx, providerURL, &resp) if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { return r.fetchProtocolProviderPackage(ctx, name, namespace) } @@ -100,8 +108,12 @@ func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, names } func (r *Registry) fetchProviderVersions(ctx context.Context, name, namespace, typ string) ([]core.Version, error) { + providerURL, err := r.providerURL(ctx, namespace, typ) + if err != nil { + return nil, err + } var resp providerVersionsResponse - if err := r.client.GetJSON(ctx, r.providerURL(namespace, typ)+"/versions", &resp); err != nil { + if err := r.client.GetJSON(ctx, providerURL+"/versions", &resp); err != nil { if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} } diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index e317528..248dae1 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/git-pkgs/registries/internal/core" "github.com/git-pkgs/registries/internal/urlparser" @@ -29,6 +30,9 @@ type Registry struct { baseURL string client *core.Client urls *URLs + + mu sync.Mutex + services *services } func New(baseURL string, client *core.Client) *Registry { @@ -119,7 +123,11 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s", r.baseURL, namespace, moduleName, provider) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s", modules, namespace, moduleName, provider) var resp moduleResponse if err := r.client.GetJSON(ctx, url, &resp); err != nil { @@ -155,7 +163,11 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", r.baseURL, namespace, moduleName, provider) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s/versions", modules, namespace, moduleName, provider) var resp moduleVersionsResponse if err := r.client.GetJSON(ctx, url, &resp); err != nil { @@ -194,7 +206,11 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string) return nil, errInvalidName } - url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", r.baseURL, namespace, moduleName, provider, version) + modules, err := r.serviceURL(ctx, modulesService) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s/%s/%s/%s", modules, namespace, moduleName, provider, version) var resp versionEntry if err := r.client.GetJSON(ctx, url, &resp); err != nil { diff --git a/internal/terraform/terraform_test.go b/internal/terraform/terraform_test.go index 9025e3c..d4a75da 100644 --- a/internal/terraform/terraform_test.go +++ b/internal/terraform/terraform_test.go @@ -12,6 +12,10 @@ import ( func TestFetchPackage(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == discoveryPath { + w.WriteHeader(404) + return + } if r.URL.Path != "/v1/modules/hashicorp/consul/aws" { t.Errorf("unexpected path: %s", r.URL.Path) w.WriteHeader(404) From 76fa2b6edbbcb05430784ef5f1673773928e664f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 18:53:15 +0100 Subject: [PATCH 3/4] Resolve discovered service URLs after redirects and widen the details fallback Relative service URLs in the discovery document now resolve against the URL the document came from after redirects, as the protocol says. The new Client.GetJSONWithFinalURL reports it. The provider details endpoint is registry.terraform.io's extension, so any 4xx answer to it but 429, not only 404, falls back to the version list. --- client/client.go | 27 +++++++++++++++++++++++++++ client/client_test.go | 21 +++++++++++++++++++++ internal/terraform/discovery.go | 7 +++++-- internal/terraform/discovery_test.go | 20 ++++++++++++++++++++ internal/terraform/provider.go | 7 ++++++- internal/terraform/provider_test.go | 21 +++++++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index 6caae2a..3dce032 100644 --- a/client/client.go +++ b/client/client.go @@ -10,6 +10,7 @@ import ( "math" "math/rand" "net/http" + "net/url" "time" "github.com/git-pkgs/registries/safehttp" @@ -76,6 +77,32 @@ func (c *Client) GetJSONWithHeaders(ctx context.Context, url string, headers htt return responseHeaders, json.Unmarshal(body, v) } +// GetJSONWithFinalURL is GetJSON that also returns the URL the response came +// from, after redirects, for documents whose relative URLs resolve against +// it. +func (c *Client) GetJSONWithFinalURL(ctx context.Context, rawURL string, v any) (*url.URL, error) { + var body []byte + var final *url.URL + _, _, err := c.open(ctx, http.MethodGet, rawURL, nil, func(resp *http.Response) error { + defer func() { _ = resp.Body.Close() }() + if resp.Request != nil { + final = resp.Request.URL + } + var err error + body, err = readLimited(resp.Body, rawURL, c.maxResponseSize()) + return err + }) + if err != nil { + return nil, err + } + if final == nil { + if final, err = url.Parse(rawURL); err != nil { + return nil, err + } + } + return final, json.Unmarshal(body, v) +} + // GetBody fetches a URL and returns the response body. A body larger than // the client's MaxResponseSize fails with ErrResponseTooLarge. func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) { diff --git a/client/client_test.go b/client/client_test.go index cec5761..625be06 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -337,3 +337,24 @@ func TestStream(t *testing.T) { t.Errorf("fn error: err = %v, want it returned as is", err) } } + +// GetJSONWithFinalURL reports the URL the response came from, after redirects. +func TestGetJSONWithFinalURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + http.Redirect(w, r, "/moved/doc.json", http.StatusFound) + return + } + _, _ = w.Write([]byte(`{"ok": true}`)) + })) + defer srv.Close() + + var v struct{ OK bool } + final, err := DefaultClient().GetJSONWithFinalURL(context.Background(), srv.URL+"/start", &v) + if err != nil || !v.OK { + t.Fatalf("GetJSONWithFinalURL = %+v, %v", v, err) + } + if final == nil || final.Path != "/moved/doc.json" { + t.Errorf("final URL = %v, want /moved/doc.json", final) + } +} diff --git a/internal/terraform/discovery.go b/internal/terraform/discovery.go index 0514921..0203b53 100644 --- a/internal/terraform/discovery.go +++ b/internal/terraform/discovery.go @@ -53,7 +53,10 @@ func (r *Registry) discover(ctx context.Context) (*services, error) { } document := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: discoveryPath} var entries map[string]json.RawMessage - if err := r.client.GetJSON(ctx, document.String(), &entries); err != nil { + // Service URLs are relative to where the document was found, after + // redirects. + final, err := r.client.GetJSONWithFinalURL(ctx, document.String(), &entries) + if err != nil { if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { return found, nil } @@ -65,7 +68,7 @@ func (r *Registry) discover(ctx context.Context) (*services, error) { if raw, ok := entries[service]; !ok || json.Unmarshal(raw, &location) != nil || location == "" { continue } - resolved, err := document.Parse(location) + resolved, err := final.Parse(location) if err != nil || (resolved.Scheme != "https" && resolved.Scheme != "http") || resolved.Host == "" { return nil, fmt.Errorf("terraform service discovery: %s is %q, not an http or https URL", service, location) } diff --git a/internal/terraform/discovery_test.go b/internal/terraform/discovery_test.go index 00f9bcc..b5fc8be 100644 --- a/internal/terraform/discovery_test.go +++ b/internal/terraform/discovery_test.go @@ -113,3 +113,23 @@ func TestServiceDiscoveryRefusesOtherSchemes(t *testing.T) { } } } + +// Service URLs are relative to the document's final URL, after redirects. +func TestServiceDiscoveryResolvesAgainstTheRedirectedDocument(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case discoveryPath: + http.Redirect(w, r, "/discovery/terraform.json", http.StatusFound) + case "/discovery/terraform.json": + _, _ = w.Write([]byte(`{"providers.v1": "./providers/"}`)) + case "/discovery/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + if versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { + t.Errorf("versions = %v, %v; want 3", versions, err) + } +} diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go index 8d3aede..cc51632 100644 --- a/internal/terraform/provider.go +++ b/internal/terraform/provider.go @@ -3,6 +3,7 @@ package terraform import ( "context" "fmt" + "net/http" "net/url" "sort" "strings" @@ -72,7 +73,11 @@ func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, ty } var resp providerResponse err = r.client.GetJSON(ctx, providerURL, &resp) - if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + // The details endpoint is registry.terraform.io's extension of the + // protocol; a registry that refuses it in any way serves the version + // list. + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.StatusCode >= 400 && httpErr.StatusCode < 500 && + httpErr.StatusCode != http.StatusTooManyRequests { return r.fetchProtocolProviderPackage(ctx, name, namespace) } if err != nil { diff --git a/internal/terraform/provider_test.go b/internal/terraform/provider_test.go index 85ee3cd..a842827 100644 --- a/internal/terraform/provider_test.go +++ b/internal/terraform/provider_test.go @@ -94,6 +94,27 @@ func TestFetchProviderPackageFromProtocolOnlyRegistry(t *testing.T) { } } +// The details endpoint is an extension of registry.terraform.io, not part +// of the protocol, so any refusal of it, not only a 404, falls back to the +// version list. +func TestFetchProviderPackageFallsBackOnAnyRefusal(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/providers/hashicorp/random": + w.WriteHeader(http.StatusMethodNotAllowed) + case "/v1/providers/hashicorp/random/versions": + _, _ = w.Write([]byte(providerVersionsJSON)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil || pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("FetchPackage = %+v, %v; want the version list's latest", pkg, err) + } +} + func TestFetchProviderPackageNotFound(t *testing.T) { server := providerServer(t, nil) _, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/missing") From bc3801ff78918ca30e5ba8b87680eada60ceb9c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 19:25:51 +0100 Subject: [PATCH 4/4] Skip discovery for registry.terraform.io and fall back on any refusal The PURL helpers build a Registry per call, so discovery doubled every lookup against registry.terraform.io; the default registry now uses /v1/modules/ and /v1/providers/ without asking. Any 4xx but 429, or a discovery body that isn't JSON, now means no document, as for the details endpoint, so hosts that refused or served HTML keep working. Also drop the unused PublishedAt, pass typ to fetchProtocolProviderPackage and trim the comments that repeated the README. --- README.md | 2 +- internal/terraform/discovery.go | 29 ++++++++++---- internal/terraform/discovery_test.go | 56 +++++++++++++++++++++++++++- internal/terraform/provider.go | 24 +++--------- 4 files changed, 82 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 27cca18..ee0263b 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ The RPM URL is one repository, the directory that holds `repodata/`, such as `ht GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. -The Terraform URL is a module and provider registry, such as `https://registry.opentofu.org`. The client reads the registry's service discovery document, `/.well-known/terraform.json` on its host, once, and sends module and provider requests to the `modules.v1` and `providers.v1` URLs it names. A registry without the document, or without one of the entries, is read at `{URL}/v1/modules/` and `{URL}/v1/providers/`. `URLs().Download` builds a module's download URL from the base URL without reading the document. +The Terraform URL is a module and provider registry, such as `https://registry.opentofu.org`. The client reads the registry's service discovery document, `/.well-known/terraform.json` on its host, once, and sends module and provider requests to the `modules.v1` and `providers.v1` URLs it names. A registry that has no document (a 4xx other than 429, or a body that isn't JSON), or no entry for a service, is read at `{URL}/v1/modules/` and `{URL}/v1/providers/`, and registry.terraform.io is always read there without asking. `URLs().Download` builds a module's download URL from the base URL without reading the document. VS Code extensions are read from [Open VSX](https://open-vsx.org), or from a self-hosted Open VSX server given as the base URL. The purl spec makes the Visual Studio Marketplace the default for `vscode-extension`, but the client has no Marketplace support, so a PURL without `repository_url` also resolves to Open VSX. `URLs().PURL` always adds the registry as `repository_url`. Names are `publisher/name` or VS Code's `publisher.name`. An extension can publish a separate build for each target platform, such as `linux-x64` or `darwin-arm64`. Each version from `FetchVersions` and `FetchVersion` carries one artifact per build, with `platform`, `signature` (the `.sigzip`) and `sha256` (the URL of the checksum file) in its metadata. `URLs().Download` returns the universal build, which a version built only for specific platforms doesn't have; the `platform` qualifier, as in `pkg:vscode-extension/muhammad-sammy/csharp@2.160.9?platform=linux-x64`, makes `fetch.Resolver` pick the build for that platform, or the universal one when there is none. `FetchDependencies` lists both the extensions a version depends on and the ones it bundles as an extension pack. `FetchMaintainers` returns the user who published the latest version. diff --git a/internal/terraform/discovery.go b/internal/terraform/discovery.go index 0203b53..6b075f7 100644 --- a/internal/terraform/discovery.go +++ b/internal/terraform/discovery.go @@ -3,7 +3,9 @@ package terraform import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "net/url" "strings" @@ -23,12 +25,8 @@ type services struct { providers string } -// serviceURL returns the base URL of a service, read from the registry's -// service discovery document at its host's /.well-known/terraform.json. A -// registry without the document, or without an entry for the service, -// serves the protocol at {base}/v1/modules/ and {base}/v1/providers/. The -// document is read once; a failure to read it is returned and retried on -// the next call. +// serviceURL reads the discovery document once; a failure to read it is +// returned and retried on the next call. func (r *Registry) serviceURL(ctx context.Context, service string) (string, error) { r.mu.Lock() defer r.mu.Unlock() @@ -45,8 +43,21 @@ func (r *Registry) serviceURL(ctx context.Context, service string) (string, erro return r.services.providers, nil } +// refused reports whether a registry answered with a 4xx other than 429: it +// doesn't serve what was asked for, as opposed to failing for now. +func refused(err error) bool { + var httpErr *core.HTTPError + return errors.As(err, &httpErr) && httpErr.StatusCode >= 400 && httpErr.StatusCode < 500 && + httpErr.StatusCode != http.StatusTooManyRequests +} + func (r *Registry) discover(ctx context.Context) (*services, error) { found := &services{modules: r.baseURL + "/v1/modules/", providers: r.baseURL + "/v1/providers/"} + // registry.terraform.io serves these paths, and the PURL helpers build a + // Registry per call, so asking it would double every lookup. + if r.baseURL == DefaultURL { + return found, nil + } base, err := url.Parse(r.baseURL) if err != nil || base.Host == "" { return found, nil //nolint:nilerr // a base URL without a host has no discovery document @@ -57,7 +68,11 @@ func (r *Registry) discover(ctx context.Context) (*services, error) { // redirects. final, err := r.client.GetJSONWithFinalURL(ctx, document.String(), &entries) if err != nil { - if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + // A refusal, or a body that isn't a JSON object, such as an HTML + // page for any unknown path, means the host has no document. + var syntaxErr *json.SyntaxError + var typeErr *json.UnmarshalTypeError + if refused(err) || errors.As(err, &syntaxErr) || errors.As(err, &typeErr) { return found, nil } return nil, fmt.Errorf("terraform service discovery: %w", err) diff --git a/internal/terraform/discovery_test.go b/internal/terraform/discovery_test.go index b5fc8be..9d15821 100644 --- a/internal/terraform/discovery_test.go +++ b/internal/terraform/discovery_test.go @@ -2,12 +2,14 @@ package terraform import ( "context" + "io" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" + "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/internal/core" ) @@ -51,16 +53,32 @@ func TestServiceDiscoveryFallsBackToTheBaseURL(t *testing.T) { "no document": "", "no entry": `{"login.v1": {"client": "terraform-cli"}}`, "entry string": `{"providers.v1": 42, "modules.v1": ["x"]}`, + // A host that answers unknown paths with an HTML page. + "not json": `app`, + // Refused, as by a host that serves no discovery. + "forbidden": "403", } { t.Run(name, func(t *testing.T) { routes := map[string]string{ "/registry/v1/providers/hashicorp/random/versions": providerVersionsJSON, "/registry/v1/modules/hashicorp/consul/aws/versions": `{"modules": [{"versions": [{"version": "0.11.0"}]}]}`, } - if document != "" { + if document != "" && document != "403" { routes[discoveryPath] = document } - server := providerServer(t, routes) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == discoveryPath && document == "403" { + w.WriteHeader(http.StatusForbidden) + return + } + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) r := New(server.URL+"/registry", core.DefaultClient()) if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 { t.Errorf("provider versions = %v, %v; want 3", versions, err) @@ -133,3 +151,37 @@ func TestServiceDiscoveryResolvesAgainstTheRedirectedDocument(t *testing.T) { t.Errorf("versions = %v, %v; want 3", versions, err) } } + +type recordingTransport struct { + paths []string +} + +func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) { + rt.paths = append(rt.paths, r.URL.Host+r.URL.Path) + body := `{"modules": [{"versions": [{"version": "0.11.0"}]}]}` + if strings.HasPrefix(r.URL.Path, "/v1/providers/") { + body = providerVersionsJSON + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil +} + +// registry.terraform.io serves the protocol at /v1/modules/ and +// /v1/providers/, so the default registry is asked without discovery: the +// PURL helpers build a new Registry per call. +func TestDefaultRegistrySkipsDiscovery(t *testing.T) { + rt := &recordingTransport{} + r := New(DefaultURL, core.NewClient(client.WithTransport(rt))) + if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil { + t.Fatalf("provider versions: %v", err) + } + if _, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil { + t.Fatalf("module versions: %v", err) + } + want := []string{ + "registry.terraform.io/v1/providers/hashicorp/random/versions", + "registry.terraform.io/v1/modules/hashicorp/consul/aws/versions", + } + if strings.Join(rt.paths, " ") != strings.Join(want, " ") { + t.Errorf("requests = %v, want %v", rt.paths, want) + } +} diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go index cc51632..08be186 100644 --- a/internal/terraform/provider.go +++ b/internal/terraform/provider.go @@ -3,7 +3,6 @@ package terraform import ( "context" "fmt" - "net/http" "net/url" "sort" "strings" @@ -14,15 +13,6 @@ import ( "github.com/git-pkgs/vers" ) -// Providers are named namespace/type, as in a provider source address -// without its hostname (hashicorp/random); modules have three segments. -// -// Only the Provider Registry Protocol's version list -// (/v1/providers/{namespace}/{type}/versions) is served by every registry. -// registry.terraform.io also serves the provider's details at -// /v1/providers/{namespace}/{type}; others, such as registry.opentofu.org, -// answer 404 there. - // parseProviderName parses "namespace/type". func parseProviderName(name string) (namespace, typ string, ok bool) { namespace, typ, ok = strings.Cut(name, "/") @@ -38,7 +28,6 @@ type providerResponse struct { Version string `json:"version"` Description string `json:"description"` Source string `json:"source"` - PublishedAt string `json:"published_at"` Downloads int64 `json:"downloads"` Tier string `json:"tier"` } @@ -66,6 +55,8 @@ func (r *Registry) providerURL(ctx context.Context, namespace, typ string) (stri return providers + url.PathEscape(namespace) + "/" + url.PathEscape(typ), nil } +// fetchProviderPackage reads registry.terraform.io's provider details; other +// registries serve none, and the package comes from the version list. func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { providerURL, err := r.providerURL(ctx, namespace, typ) if err != nil { @@ -73,12 +64,8 @@ func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, ty } var resp providerResponse err = r.client.GetJSON(ctx, providerURL, &resp) - // The details endpoint is registry.terraform.io's extension of the - // protocol; a registry that refuses it in any way serves the version - // list. - if httpErr, ok := err.(*core.HTTPError); ok && httpErr.StatusCode >= 400 && httpErr.StatusCode < 500 && - httpErr.StatusCode != http.StatusTooManyRequests { - return r.fetchProtocolProviderPackage(ctx, name, namespace) + if refused(err) { + return r.fetchProtocolProviderPackage(ctx, name, namespace, typ) } if err != nil { return nil, err @@ -99,8 +86,7 @@ func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, ty // fetchProtocolProviderPackage describes a provider from its version list // alone, for registries that serve no details. -func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace string) (*core.Package, error) { - typ := strings.TrimPrefix(name, namespace+"/") +func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { versions, err := r.fetchProviderVersions(ctx, name, namespace, typ) if err != nil { return nil, err