Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5)
| Go | `pkg:golang/github.com/gorilla/mux@v1.8.0` |
| Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` |
| RubyGems | `pkg:gem/rails@7.1.0` |
| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` |
| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` (module), `pkg:terraform/hashicorp/random@3.6.0` (provider) |
| GitHub Actions | `pkg:githubactions/actions/checkout@v4` |
| VS Code extension | `pkg:vscode-extension/redhat/vscode-yaml@1.15.0` |
| Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` |
Expand Down Expand Up @@ -213,6 +213,8 @@ The RPM URL is one repository, the directory that holds `repodata/`, such as `ht

GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL.

The Terraform URL is a module and provider registry, such as `https://registry.opentofu.org`. The client reads the registry's service discovery document, `/.well-known/terraform.json` on its host, once, and sends module and provider requests to the `modules.v1` and `providers.v1` URLs it names. A registry that has no document (a 4xx other than 429, or a body that isn't JSON), or no entry for a service, is read at `{URL}/v1/modules/` and `{URL}/v1/providers/`, and registry.terraform.io is always read there without asking. `URLs().Download` builds a module's download URL from the base URL without reading the document.

VS Code extensions are read from [Open VSX](https://open-vsx.org), or from a self-hosted Open VSX server given as the base URL. The purl spec makes the Visual Studio Marketplace the default for `vscode-extension`, but the client has no Marketplace support, so a PURL without `repository_url` also resolves to Open VSX. `URLs().PURL` always adds the registry as `repository_url`. Names are `publisher/name` or VS Code's `publisher.name`. An extension can publish a separate build for each target platform, such as `linux-x64` or `darwin-arm64`. Each version from `FetchVersions` and `FetchVersion` carries one artifact per build, with `platform`, `signature` (the `.sigzip`) and `sha256` (the URL of the checksum file) in its metadata. `URLs().Download` returns the universal build, which a version built only for specific platforms doesn't have; the `platform` qualifier, as in `pkg:vscode-extension/muhammad-sammy/csharp@2.160.9?platform=linux-x64`, makes `fetch.Resolver` pick the build for that platform, or the universal one when there is none. `FetchDependencies` lists both the extensions a version depends on and the ones it bundles as an extension pack. `FetchMaintainers` returns the user who published the latest version.

### Swift registries
Expand Down
27 changes: 27 additions & 0 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"math"
"math/rand"
"net/http"
"net/url"
"time"

"github.com/git-pkgs/registries/safehttp"
Expand Down Expand Up @@ -76,6 +77,32 @@ func (c *Client) GetJSONWithHeaders(ctx context.Context, url string, headers htt
return responseHeaders, json.Unmarshal(body, v)
}

// GetJSONWithFinalURL is GetJSON that also returns the URL the response came
// from, after redirects, for documents whose relative URLs resolve against
// it.
func (c *Client) GetJSONWithFinalURL(ctx context.Context, rawURL string, v any) (*url.URL, error) {
var body []byte
var final *url.URL
_, _, err := c.open(ctx, http.MethodGet, rawURL, nil, func(resp *http.Response) error {
defer func() { _ = resp.Body.Close() }()
if resp.Request != nil {
final = resp.Request.URL
}
var err error
body, err = readLimited(resp.Body, rawURL, c.maxResponseSize())
return err
})
if err != nil {
return nil, err
}
if final == nil {
if final, err = url.Parse(rawURL); err != nil {
return nil, err
}
}
return final, json.Unmarshal(body, v)
}

// GetBody fetches a URL and returns the response body. A body larger than
// the client's MaxResponseSize fails with ErrResponseTooLarge.
func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {
Expand Down
21 changes: 21 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -337,3 +337,24 @@ func TestStream(t *testing.T) {
t.Errorf("fn error: err = %v, want it returned as is", err)
}
}

// GetJSONWithFinalURL reports the URL the response came from, after redirects.
func TestGetJSONWithFinalURL(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/start" {
http.Redirect(w, r, "/moved/doc.json", http.StatusFound)
return
}
_, _ = w.Write([]byte(`{"ok": true}`))
}))
defer srv.Close()

var v struct{ OK bool }
final, err := DefaultClient().GetJSONWithFinalURL(context.Background(), srv.URL+"/start", &v)
if err != nil || !v.OK {
t.Fatalf("GetJSONWithFinalURL = %+v, %v", v, err)
}
if final == nil || final.Path != "/moved/doc.json" {
t.Errorf("final URL = %v, want /moved/doc.json", final)
}
}
93 changes: 93 additions & 0 deletions internal/terraform/discovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package terraform

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"

"github.com/git-pkgs/registries/internal/core"
)

const (
discoveryPath = "/.well-known/terraform.json"
modulesService = "modules.v1"
providersService = "providers.v1"
)

// services are the base URLs of a registry's module and provider APIs, each
// ending in a slash.
type services struct {
modules string
providers string
}

// serviceURL reads the discovery document once; a failure to read it is
// returned and retried on the next call.
func (r *Registry) serviceURL(ctx context.Context, service string) (string, error) {
r.mu.Lock()
defer r.mu.Unlock()
if r.services == nil {
found, err := r.discover(ctx)
if err != nil {
return "", err
}
r.services = found
}
if service == modulesService {
return r.services.modules, nil
}
return r.services.providers, nil
}

// refused reports whether a registry answered with a 4xx other than 429: it
// doesn't serve what was asked for, as opposed to failing for now.
func refused(err error) bool {
var httpErr *core.HTTPError
return errors.As(err, &httpErr) && httpErr.StatusCode >= 400 && httpErr.StatusCode < 500 &&
httpErr.StatusCode != http.StatusTooManyRequests
}

func (r *Registry) discover(ctx context.Context) (*services, error) {
found := &services{modules: r.baseURL + "/v1/modules/", providers: r.baseURL + "/v1/providers/"}
// registry.terraform.io serves these paths, and the PURL helpers build a
// Registry per call, so asking it would double every lookup.
if r.baseURL == DefaultURL {
return found, nil
}
base, err := url.Parse(r.baseURL)
if err != nil || base.Host == "" {
return found, nil //nolint:nilerr // a base URL without a host has no discovery document
}
document := &url.URL{Scheme: base.Scheme, Host: base.Host, Path: discoveryPath}
var entries map[string]json.RawMessage
// Service URLs are relative to where the document was found, after
// redirects.
final, err := r.client.GetJSONWithFinalURL(ctx, document.String(), &entries)
if err != nil {
// A refusal, or a body that isn't a JSON object, such as an HTML
// page for any unknown path, means the host has no document.
var syntaxErr *json.SyntaxError
var typeErr *json.UnmarshalTypeError
if refused(err) || errors.As(err, &syntaxErr) || errors.As(err, &typeErr) {
return found, nil
}
return nil, fmt.Errorf("terraform service discovery: %w", err)
}
for service, target := range map[string]*string{modulesService: &found.modules, providersService: &found.providers} {
var location string
// Other services, and entries that aren't a URL, are left alone.
if raw, ok := entries[service]; !ok || json.Unmarshal(raw, &location) != nil || location == "" {
continue
}
resolved, err := final.Parse(location)
if err != nil || (resolved.Scheme != "https" && resolved.Scheme != "http") || resolved.Host == "" {
return nil, fmt.Errorf("terraform service discovery: %s is %q, not an http or https URL", service, location)
}
*target = strings.TrimSuffix(resolved.String(), "/") + "/"
}
return found, nil
}
187 changes: 187 additions & 0 deletions internal/terraform/discovery_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
package terraform

import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"

"github.com/git-pkgs/registries/client"
"github.com/git-pkgs/registries/internal/core"
)

// A registry names where its module and provider APIs live in its service
// discovery document. Requests go there, and the document is read once.
func TestServiceDiscovery(t *testing.T) {
var discoveries atomic.Int32
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case discoveryPath:
discoveries.Add(1)
// A relative and an absolute URL, as the protocol allows both.
_, _ = w.Write([]byte(`{"modules.v1": "/api/terraform/modules/", "providers.v1": "` + server.URL + `/api/terraform/providers/"}`))
case "/api/terraform/providers/hashicorp/random/versions":
_, _ = w.Write([]byte(providerVersionsJSON))
case "/api/terraform/modules/hashicorp/consul/aws/versions":
_, _ = w.Write([]byte(`{"modules": [{"versions": [{"version": "0.11.0"}]}]}`))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)

r := New(server.URL, core.DefaultClient())
if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 {
t.Errorf("provider versions = %v, %v; want 3", versions, err)
}
if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 {
t.Errorf("module versions = %v, %v; want 1", versions, err)
}
if got := discoveries.Load(); got != 1 {
t.Errorf("discovery read %d times, want once", got)
}
}

// Without a discovery document, or without an entry for a service, the
// registry's base URL serves the protocol at /v1/modules and /v1/providers.
func TestServiceDiscoveryFallsBackToTheBaseURL(t *testing.T) {
for name, document := range map[string]string{
"no document": "",
"no entry": `{"login.v1": {"client": "terraform-cli"}}`,
"entry string": `{"providers.v1": 42, "modules.v1": ["x"]}`,
// A host that answers unknown paths with an HTML page.
"not json": `<!doctype html><html><body>app</body></html>`,
// Refused, as by a host that serves no discovery.
"forbidden": "403",
} {
t.Run(name, func(t *testing.T) {
routes := map[string]string{
"/registry/v1/providers/hashicorp/random/versions": providerVersionsJSON,
"/registry/v1/modules/hashicorp/consul/aws/versions": `{"modules": [{"versions": [{"version": "0.11.0"}]}]}`,
}
if document != "" && document != "403" {
routes[discoveryPath] = document
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == discoveryPath && document == "403" {
w.WriteHeader(http.StatusForbidden)
return
}
body, ok := routes[r.URL.Path]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
_, _ = w.Write([]byte(body))
}))
t.Cleanup(server.Close)
r := New(server.URL+"/registry", core.DefaultClient())
if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 {
t.Errorf("provider versions = %v, %v; want 3", versions, err)
}
if versions, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil || len(versions) != 1 {
t.Errorf("module versions = %v, %v; want 1", versions, err)
}
})
}
}

// A discovery document that can't be read is an error, and is read again
// next time rather than replaced by the fallback for good.
func TestServiceDiscoveryFailureIsRetried(t *testing.T) {
var failing atomic.Bool
failing.Store(true)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == discoveryPath && failing.Load():
w.WriteHeader(http.StatusInternalServerError)
case r.URL.Path == discoveryPath:
_, _ = w.Write([]byte(`{"providers.v1": "/p/"}`))
case r.URL.Path == "/p/hashicorp/random/versions":
_, _ = w.Write([]byte(providerVersionsJSON))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)

client := core.DefaultClient()
client.MaxRetries = 0
r := New(server.URL, client)
if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err == nil {
t.Fatal("FetchVersions succeeded although discovery failed")
}
failing.Store(false)
if versions, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 {
t.Errorf("after recovery: %v, %v; want 3 versions", versions, err)
}
}

// Only http and https service URLs are followed.
func TestServiceDiscoveryRefusesOtherSchemes(t *testing.T) {
for _, location := range []string{"file:///etc/", "ftp://registry.example.com/providers/"} {
server := providerServer(t, map[string]string{discoveryPath: `{"providers.v1": "` + location + `"}`})
_, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random")
if err == nil || !strings.Contains(err.Error(), "providers.v1") {
t.Errorf("%s: err = %v, want a refused providers.v1", location, err)
}
}
}

// Service URLs are relative to the document's final URL, after redirects.
func TestServiceDiscoveryResolvesAgainstTheRedirectedDocument(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case discoveryPath:
http.Redirect(w, r, "/discovery/terraform.json", http.StatusFound)
case "/discovery/terraform.json":
_, _ = w.Write([]byte(`{"providers.v1": "./providers/"}`))
case "/discovery/providers/hashicorp/random/versions":
_, _ = w.Write([]byte(providerVersionsJSON))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
if versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random"); err != nil || len(versions) != 3 {
t.Errorf("versions = %v, %v; want 3", versions, err)
}
}

type recordingTransport struct {
paths []string
}

func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
rt.paths = append(rt.paths, r.URL.Host+r.URL.Path)
body := `{"modules": [{"versions": [{"version": "0.11.0"}]}]}`
if strings.HasPrefix(r.URL.Path, "/v1/providers/") {
body = providerVersionsJSON
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
}

// registry.terraform.io serves the protocol at /v1/modules/ and
// /v1/providers/, so the default registry is asked without discovery: the
// PURL helpers build a new Registry per call.
func TestDefaultRegistrySkipsDiscovery(t *testing.T) {
rt := &recordingTransport{}
r := New(DefaultURL, core.NewClient(client.WithTransport(rt)))
if _, err := r.FetchVersions(context.Background(), "hashicorp/random"); err != nil {
t.Fatalf("provider versions: %v", err)
}
if _, err := r.FetchVersions(context.Background(), "hashicorp/consul/aws"); err != nil {
t.Fatalf("module versions: %v", err)
}
want := []string{
"registry.terraform.io/v1/providers/hashicorp/random/versions",
"registry.terraform.io/v1/modules/hashicorp/consul/aws/versions",
}
if strings.Join(rt.paths, " ") != strings.Join(want, " ") {
t.Errorf("requests = %v, want %v", rt.paths, want)
}
}
Loading
Loading