From dc0dcc228b5503780ec9ff3e2a1353ac4e1761c1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 19:28:01 +0100 Subject: [PATCH] Support Terraform providers next to modules A two-segment name, namespace/type, is a provider. Versions come from the Provider Registry Protocol's version list, which every registry serves, with each version's protocols and platforms in its metadata. The package reads registry.terraform.io's provider details and falls back to the version list on registries that have none, such as registry.opentofu.org. Providers declare no dependencies, and have no single download URL, since each platform has its own archive. --- README.md | 2 +- internal/terraform/provider.go | 154 +++++++++++++++++++++ internal/terraform/provider_test.go | 205 ++++++++++++++++++++++++++++ internal/terraform/terraform.go | 36 ++++- 4 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 internal/terraform/provider.go create mode 100644 internal/terraform/provider_test.go diff --git a/README.md b/README.md index d953bfe..96be189 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Go | `pkg:golang/github.com/gorilla/mux@v1.8.0` | | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` | | RubyGems | `pkg:gem/rails@7.1.0` | -| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | +| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` (module), `pkg:terraform/hashicorp/random@3.6.0` (provider) | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go new file mode 100644 index 0000000..2932439 --- /dev/null +++ b/internal/terraform/provider.go @@ -0,0 +1,154 @@ +package terraform + +import ( + "context" + "fmt" + "net/url" + "sort" + "strings" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/urlparser" + "github.com/git-pkgs/vers" +) + +// Providers are named namespace/type, as in a provider source address +// without its hostname (hashicorp/random); modules have three segments. +// +// Only the Provider Registry Protocol's version list +// (/v1/providers/{namespace}/{type}/versions) is served by every registry. +// registry.terraform.io also serves the provider's details at +// /v1/providers/{namespace}/{type}; others, such as registry.opentofu.org, +// answer 404 there. + +// parseProviderName parses "namespace/type". +func parseProviderName(name string) (namespace, typ string, ok bool) { + namespace, typ, ok = strings.Cut(name, "/") + if !ok || namespace == "" || typ == "" || strings.Contains(typ, "/") { + return "", "", false + } + return namespace, typ, true +} + +type providerResponse struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt string `json:"published_at"` + Downloads int64 `json:"downloads"` + Tier string `json:"tier"` +} + +type providerVersionsResponse struct { + Versions []providerVersion `json:"versions"` +} + +type providerVersion struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []providerPlatform `json:"platforms"` +} + +type providerPlatform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +func (r *Registry) providerURL(namespace, typ string) string { + return fmt.Sprintf("%s/v1/providers/%s/%s", r.baseURL, url.PathEscape(namespace), url.PathEscape(typ)) +} + +func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + var resp providerResponse + err := r.client.GetJSON(ctx, r.providerURL(namespace, typ), &resp) + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return r.fetchProtocolProviderPackage(ctx, name, namespace) + } + if err != nil { + return nil, err + } + return &core.Package{ + Name: name, + Description: resp.Description, + Homepage: r.urls.Registry(name, ""), + Repository: urlparser.Parse(resp.Source), + Namespace: namespace, + LatestVersion: resp.Version, + Metadata: map[string]any{ + "tier": resp.Tier, + "downloads": resp.Downloads, + }, + }, nil +} + +// fetchProtocolProviderPackage describes a provider from its version list +// alone, for registries that serve no details. +func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace string) (*core.Package, error) { + typ := strings.TrimPrefix(name, namespace+"/") + versions, err := r.fetchProviderVersions(ctx, name, namespace, typ) + if err != nil { + return nil, err + } + pkg := &core.Package{Name: name, Homepage: r.urls.Registry(name, ""), Namespace: namespace} + if latest := core.SelectLatestVersion(versions, ecosystem, ""); latest != nil { + pkg.LatestVersion = latest.Number + } + return pkg, nil +} + +func (r *Registry) fetchProviderVersions(ctx context.Context, name, namespace, typ string) ([]core.Version, error) { + var resp providerVersionsResponse + if err := r.client.GetJSON(ctx, r.providerURL(namespace, typ)+"/versions", &resp); err != nil { + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return nil, err + } + + versions := make([]core.Version, 0, len(resp.Versions)) + for _, v := range resp.Versions { + if v.Version == "" { + continue + } + platforms := make([]string, 0, len(v.Platforms)) + for _, p := range v.Platforms { + platforms = append(platforms, p.OS+"_"+p.Arch) + } + versions = append(versions, core.Version{ + Number: v.Version, + Metadata: map[string]any{ + "protocols": v.Protocols, + "platforms": platforms, + }, + }) + } + // The protocol lists versions in no particular order. + sort.SliceStable(versions, func(i, j int) bool { + return vers.CompareWithScheme(versions[i].Number, versions[j].Number, ecosystem) > 0 + }) + return versions, nil +} + +// providerPage is the provider's page on registry.terraform.io, or "" for +// another registry, which has no pages at these paths. +func (u *URLs) providerPage(namespace, typ, version, suffix string) string { + if u.baseURL != DefaultURL { + return "" + } + if version == "" { + version = "latest" + } + return fmt.Sprintf("%s/providers/%s/%s/%s%s", DefaultURL, namespace, typ, version, suffix) +} + +// providerPURL carries repository_url for any registry but the default one. +func (u *URLs) providerPURL(namespace, typ, version string) string { + var qualifiers map[string]string + if u.baseURL != DefaultURL { + qualifiers = map[string]string{"repository_url": u.baseURL} + } + return purl.New(ecosystem, namespace, typ, version, qualifiers).String() +} diff --git a/internal/terraform/provider_test.go b/internal/terraform/provider_test.go new file mode 100644 index 0000000..85ee3cd --- /dev/null +++ b/internal/terraform/provider_test.go @@ -0,0 +1,205 @@ +package terraform + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/git-pkgs/registries/internal/core" +) + +// Trimmed from registry.terraform.io's /v1/providers/hashicorp/random. +const providerDetailJSON = `{ + "id": "hashicorp/random/3.9.1", + "owner": "hashicorp", + "namespace": "hashicorp", + "name": "random", + "version": "3.9.1", + "description": "terraform-provider-random", + "source": "https://github.com/hashicorp/terraform-provider-random", + "published_at": "2026-09-11T15:17:02Z", + "downloads": 3603195929, + "tier": "official", + "versions": ["3.6.0", "3.9.1"] +}` + +// The Provider Registry Protocol's version list, which every registry +// serves, trimmed from registry.opentofu.org. Versions are unordered. +const providerVersionsJSON = `{ + "versions": [ + {"version": "3.6.0", "protocols": ["5.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}, {"os": "darwin", "arch": "arm64"}]}, + {"version": "3.10.0-beta1", "protocols": ["5.0"], "platforms": [{"os": "linux", "arch": "amd64"}]}, + {"version": "3.9.1", "protocols": ["5.0", "6.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}]} + ] +}` + +// providerServer serves path → body, and 404 for anything else. +func providerServer(t *testing.T, routes map[string]string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFetchProviderPackage(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random": providerDetailJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.Description != "terraform-provider-random" { + t.Errorf("description = %q", pkg.Description) + } + if pkg.Repository != "https://github.com/hashicorp/terraform-provider-random" { + t.Errorf("repository = %q", pkg.Repository) + } + if pkg.LatestVersion != "3.9.1" { + t.Errorf("latest = %q, want 3.9.1", pkg.LatestVersion) + } + if pkg.Metadata["tier"] != "official" || pkg.Metadata["downloads"] != int64(3603195929) { + t.Errorf("metadata = %v", pkg.Metadata) + } +} + +// Registries other than registry.terraform.io, such as registry.opentofu.org, +// only serve the Provider Registry Protocol, so the package falls back to +// its version list. +func TestFetchProviderPackageFromProtocolOnlyRegistry(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("latest = %q, want the highest version, 3.10.0-beta1", pkg.LatestVersion) + } +} + +func TestFetchProviderPackageNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +func TestFetchProviderVersions(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + var got []string + for _, v := range versions { + got = append(got, v.Number) + } + if want := []string{"3.10.0-beta1", "3.9.1", "3.6.0"}; !reflect.DeepEqual(got, want) { + t.Fatalf("versions = %v, want %v (newest first, by version not by string)", got, want) + } + v := versions[1] + if !reflect.DeepEqual(v.Metadata["protocols"], []string{"5.0", "6.0"}) { + t.Errorf("protocols = %v", v.Metadata["protocols"]) + } + if !reflect.DeepEqual(versions[2].Metadata["platforms"], []string{"linux_amd64", "darwin_arm64"}) { + t.Errorf("platforms = %v", versions[2].Metadata["platforms"]) + } +} + +func TestFetchProviderVersionsNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +// Providers declare no dependencies, so no request is made. +func TestFetchProviderDependencies(t *testing.T) { + server := providerServer(t, nil) + deps, err := New(server.URL, core.DefaultClient()).FetchDependencies(context.Background(), "hashicorp/random", "3.6.0") + if err != nil || len(deps) != 0 { + t.Errorf("deps = %v, %v; want none", deps, err) + } +} + +func TestFetchProviderMaintainers(t *testing.T) { + maintainers, err := New("", nil).FetchMaintainers(context.Background(), "hashicorp/random") + if err != nil || len(maintainers) != 1 || maintainers[0].Login != "hashicorp" { + t.Errorf("maintainers = %v, %v; want the namespace", maintainers, err) + } +} + +func TestParseProviderName(t *testing.T) { + tests := []struct { + input, namespace, typ string + ok bool + }{ + {"hashicorp/random", "hashicorp", "random", true}, + {"integrations/github", "integrations", "github", true}, + {"random", "", "", false}, + {"hashicorp/consul/aws", "", "", false}, + {"/random", "", "", false}, + {"hashicorp/", "", "", false}, + } + for _, tt := range tests { + namespace, typ, ok := parseProviderName(tt.input) + if namespace != tt.namespace || typ != tt.typ || ok != tt.ok { + t.Errorf("parseProviderName(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.input, namespace, typ, ok, tt.namespace, tt.typ, tt.ok) + } + } +} + +func TestProviderURLs(t *testing.T) { + urls := New("https://registry.terraform.io", nil).URLs() + other := New("https://registry.opentofu.org", nil).URLs() + tests := []struct { + name, got, want string + }{ + {"registry", urls.Registry("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0"}, + {"registry latest", urls.Registry("hashicorp/random", ""), "https://registry.terraform.io/providers/hashicorp/random/latest"}, + {"documentation", urls.Documentation("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs"}, + // Archives are per platform; the download endpoint names them. + {"download", urls.Download("hashicorp/random", "3.6.0"), ""}, + {"purl", urls.PURL("hashicorp/random", "3.6.0"), "pkg:terraform/hashicorp/random@3.6.0"}, + {"purl without version", urls.PURL("hashicorp/random", ""), "pkg:terraform/hashicorp/random"}, + // Other registries have no pages at these paths. + {"other registry page", other.Registry("hashicorp/random", "3.6.0"), ""}, + {"other registry purl", other.PURL("hashicorp/random", "3.6.0"), + "pkg:terraform/hashicorp/random@3.6.0?repository_url=https:%2F%2Fregistry.opentofu.org"}, + } + for _, tt := range tests { + if tt.got != tt.want { + t.Errorf("%s = %q, want %q", tt.name, tt.got, tt.want) + } + } +} + +func TestFetchProviderVersionsSkipsEntriesWithoutAVersion(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": `{"versions": [{"version": ""}, {"version": "3.6.0"}]}`}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil || len(versions) != 1 || versions[0].Number != "3.6.0" { + t.Errorf("versions = %v, %v; want only 3.6.0", versions, err) + } +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index a2e74b1..e317528 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -3,6 +3,7 @@ package terraform import ( "context" + "errors" "fmt" "sort" "strings" @@ -22,6 +23,8 @@ func init() { }) } +var errInvalidName = errors.New("terraform name must be 'namespace/type' for a provider or 'namespace/name/provider' for a module") + type Registry struct { baseURL string client *core.Client @@ -108,9 +111,12 @@ type providerEntry struct { } func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderPackage(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s", r.baseURL, namespace, moduleName, provider) @@ -141,9 +147,12 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package } func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderVersions(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", r.baseURL, namespace, moduleName, provider) @@ -176,9 +185,13 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi } func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + // Providers declare no dependencies. + if _, _, ok := parseProviderName(name); ok { + return nil, nil + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", r.baseURL, namespace, moduleName, provider, version) @@ -224,6 +237,9 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string) func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { namespace, _, _, ok := parseModuleName(name) + if !ok { + namespace, _, ok = parseProviderName(name) + } if !ok { return nil, nil } @@ -240,6 +256,9 @@ type URLs struct { } func (u *URLs) Registry(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "") + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" @@ -250,7 +269,12 @@ func (u *URLs) Registry(name, version string) string { return fmt.Sprintf("https://registry.terraform.io/modules/%s/%s/%s", namespace, moduleName, provider) } +// Download is the module's download endpoint. A provider has an archive per +// platform, named by the download endpoint for that platform, so it has none. func (u *URLs) Download(name, version string) string { + if _, _, ok := parseProviderName(name); ok { + return "" + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok || version == "" { return "" @@ -259,10 +283,16 @@ func (u *URLs) Download(name, version string) string { } func (u *URLs) Documentation(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "/docs") + } return u.Registry(name, version) } func (u *URLs) PURL(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPURL(namespace, typ, version) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return ""