diff --git a/README.md b/README.md index d953bfe..96be189 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Go | `pkg:golang/github.com/gorilla/mux@v1.8.0` | | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` | | RubyGems | `pkg:gem/rails@7.1.0` | -| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | +| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` (module), `pkg:terraform/hashicorp/random@3.6.0` (provider) | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | diff --git a/internal/terraform/provider.go b/internal/terraform/provider.go new file mode 100644 index 0000000..2932439 --- /dev/null +++ b/internal/terraform/provider.go @@ -0,0 +1,154 @@ +package terraform + +import ( + "context" + "fmt" + "net/url" + "sort" + "strings" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/urlparser" + "github.com/git-pkgs/vers" +) + +// Providers are named namespace/type, as in a provider source address +// without its hostname (hashicorp/random); modules have three segments. +// +// Only the Provider Registry Protocol's version list +// (/v1/providers/{namespace}/{type}/versions) is served by every registry. +// registry.terraform.io also serves the provider's details at +// /v1/providers/{namespace}/{type}; others, such as registry.opentofu.org, +// answer 404 there. + +// parseProviderName parses "namespace/type". +func parseProviderName(name string) (namespace, typ string, ok bool) { + namespace, typ, ok = strings.Cut(name, "/") + if !ok || namespace == "" || typ == "" || strings.Contains(typ, "/") { + return "", "", false + } + return namespace, typ, true +} + +type providerResponse struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Source string `json:"source"` + PublishedAt string `json:"published_at"` + Downloads int64 `json:"downloads"` + Tier string `json:"tier"` +} + +type providerVersionsResponse struct { + Versions []providerVersion `json:"versions"` +} + +type providerVersion struct { + Version string `json:"version"` + Protocols []string `json:"protocols"` + Platforms []providerPlatform `json:"platforms"` +} + +type providerPlatform struct { + OS string `json:"os"` + Arch string `json:"arch"` +} + +func (r *Registry) providerURL(namespace, typ string) string { + return fmt.Sprintf("%s/v1/providers/%s/%s", r.baseURL, url.PathEscape(namespace), url.PathEscape(typ)) +} + +func (r *Registry) fetchProviderPackage(ctx context.Context, name, namespace, typ string) (*core.Package, error) { + var resp providerResponse + err := r.client.GetJSON(ctx, r.providerURL(namespace, typ), &resp) + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return r.fetchProtocolProviderPackage(ctx, name, namespace) + } + if err != nil { + return nil, err + } + return &core.Package{ + Name: name, + Description: resp.Description, + Homepage: r.urls.Registry(name, ""), + Repository: urlparser.Parse(resp.Source), + Namespace: namespace, + LatestVersion: resp.Version, + Metadata: map[string]any{ + "tier": resp.Tier, + "downloads": resp.Downloads, + }, + }, nil +} + +// fetchProtocolProviderPackage describes a provider from its version list +// alone, for registries that serve no details. +func (r *Registry) fetchProtocolProviderPackage(ctx context.Context, name, namespace string) (*core.Package, error) { + typ := strings.TrimPrefix(name, namespace+"/") + versions, err := r.fetchProviderVersions(ctx, name, namespace, typ) + if err != nil { + return nil, err + } + pkg := &core.Package{Name: name, Homepage: r.urls.Registry(name, ""), Namespace: namespace} + if latest := core.SelectLatestVersion(versions, ecosystem, ""); latest != nil { + pkg.LatestVersion = latest.Number + } + return pkg, nil +} + +func (r *Registry) fetchProviderVersions(ctx context.Context, name, namespace, typ string) ([]core.Version, error) { + var resp providerVersionsResponse + if err := r.client.GetJSON(ctx, r.providerURL(namespace, typ)+"/versions", &resp); err != nil { + if httpErr, ok := err.(*core.HTTPError); ok && httpErr.IsNotFound() { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return nil, err + } + + versions := make([]core.Version, 0, len(resp.Versions)) + for _, v := range resp.Versions { + if v.Version == "" { + continue + } + platforms := make([]string, 0, len(v.Platforms)) + for _, p := range v.Platforms { + platforms = append(platforms, p.OS+"_"+p.Arch) + } + versions = append(versions, core.Version{ + Number: v.Version, + Metadata: map[string]any{ + "protocols": v.Protocols, + "platforms": platforms, + }, + }) + } + // The protocol lists versions in no particular order. + sort.SliceStable(versions, func(i, j int) bool { + return vers.CompareWithScheme(versions[i].Number, versions[j].Number, ecosystem) > 0 + }) + return versions, nil +} + +// providerPage is the provider's page on registry.terraform.io, or "" for +// another registry, which has no pages at these paths. +func (u *URLs) providerPage(namespace, typ, version, suffix string) string { + if u.baseURL != DefaultURL { + return "" + } + if version == "" { + version = "latest" + } + return fmt.Sprintf("%s/providers/%s/%s/%s%s", DefaultURL, namespace, typ, version, suffix) +} + +// providerPURL carries repository_url for any registry but the default one. +func (u *URLs) providerPURL(namespace, typ, version string) string { + var qualifiers map[string]string + if u.baseURL != DefaultURL { + qualifiers = map[string]string{"repository_url": u.baseURL} + } + return purl.New(ecosystem, namespace, typ, version, qualifiers).String() +} diff --git a/internal/terraform/provider_test.go b/internal/terraform/provider_test.go new file mode 100644 index 0000000..85ee3cd --- /dev/null +++ b/internal/terraform/provider_test.go @@ -0,0 +1,205 @@ +package terraform + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/git-pkgs/registries/internal/core" +) + +// Trimmed from registry.terraform.io's /v1/providers/hashicorp/random. +const providerDetailJSON = `{ + "id": "hashicorp/random/3.9.1", + "owner": "hashicorp", + "namespace": "hashicorp", + "name": "random", + "version": "3.9.1", + "description": "terraform-provider-random", + "source": "https://github.com/hashicorp/terraform-provider-random", + "published_at": "2026-09-11T15:17:02Z", + "downloads": 3603195929, + "tier": "official", + "versions": ["3.6.0", "3.9.1"] +}` + +// The Provider Registry Protocol's version list, which every registry +// serves, trimmed from registry.opentofu.org. Versions are unordered. +const providerVersionsJSON = `{ + "versions": [ + {"version": "3.6.0", "protocols": ["5.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}, {"os": "darwin", "arch": "arm64"}]}, + {"version": "3.10.0-beta1", "protocols": ["5.0"], "platforms": [{"os": "linux", "arch": "amd64"}]}, + {"version": "3.9.1", "protocols": ["5.0", "6.0"], "platforms": [ + {"os": "linux", "arch": "amd64"}]} + ] +}` + +// providerServer serves path → body, and 404 for anything else. +func providerServer(t *testing.T, routes map[string]string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFetchProviderPackage(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random": providerDetailJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.Description != "terraform-provider-random" { + t.Errorf("description = %q", pkg.Description) + } + if pkg.Repository != "https://github.com/hashicorp/terraform-provider-random" { + t.Errorf("repository = %q", pkg.Repository) + } + if pkg.LatestVersion != "3.9.1" { + t.Errorf("latest = %q, want 3.9.1", pkg.LatestVersion) + } + if pkg.Metadata["tier"] != "official" || pkg.Metadata["downloads"] != int64(3603195929) { + t.Errorf("metadata = %v", pkg.Metadata) + } +} + +// Registries other than registry.terraform.io, such as registry.opentofu.org, +// only serve the Provider Registry Protocol, so the package falls back to +// its version list. +func TestFetchProviderPackageFromProtocolOnlyRegistry(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + pkg, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "hashicorp/random" || pkg.Namespace != "hashicorp" { + t.Errorf("name = %q, namespace = %q", pkg.Name, pkg.Namespace) + } + if pkg.LatestVersion != "3.10.0-beta1" { + t.Errorf("latest = %q, want the highest version, 3.10.0-beta1", pkg.LatestVersion) + } +} + +func TestFetchProviderPackageNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchPackage(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +func TestFetchProviderVersions(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": providerVersionsJSON}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + var got []string + for _, v := range versions { + got = append(got, v.Number) + } + if want := []string{"3.10.0-beta1", "3.9.1", "3.6.0"}; !reflect.DeepEqual(got, want) { + t.Fatalf("versions = %v, want %v (newest first, by version not by string)", got, want) + } + v := versions[1] + if !reflect.DeepEqual(v.Metadata["protocols"], []string{"5.0", "6.0"}) { + t.Errorf("protocols = %v", v.Metadata["protocols"]) + } + if !reflect.DeepEqual(versions[2].Metadata["platforms"], []string{"linux_amd64", "darwin_arm64"}) { + t.Errorf("platforms = %v", versions[2].Metadata["platforms"]) + } +} + +func TestFetchProviderVersionsNotFound(t *testing.T) { + server := providerServer(t, nil) + _, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/missing") + var notFound *core.NotFoundError + if !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want a NotFoundError", err) + } +} + +// Providers declare no dependencies, so no request is made. +func TestFetchProviderDependencies(t *testing.T) { + server := providerServer(t, nil) + deps, err := New(server.URL, core.DefaultClient()).FetchDependencies(context.Background(), "hashicorp/random", "3.6.0") + if err != nil || len(deps) != 0 { + t.Errorf("deps = %v, %v; want none", deps, err) + } +} + +func TestFetchProviderMaintainers(t *testing.T) { + maintainers, err := New("", nil).FetchMaintainers(context.Background(), "hashicorp/random") + if err != nil || len(maintainers) != 1 || maintainers[0].Login != "hashicorp" { + t.Errorf("maintainers = %v, %v; want the namespace", maintainers, err) + } +} + +func TestParseProviderName(t *testing.T) { + tests := []struct { + input, namespace, typ string + ok bool + }{ + {"hashicorp/random", "hashicorp", "random", true}, + {"integrations/github", "integrations", "github", true}, + {"random", "", "", false}, + {"hashicorp/consul/aws", "", "", false}, + {"/random", "", "", false}, + {"hashicorp/", "", "", false}, + } + for _, tt := range tests { + namespace, typ, ok := parseProviderName(tt.input) + if namespace != tt.namespace || typ != tt.typ || ok != tt.ok { + t.Errorf("parseProviderName(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.input, namespace, typ, ok, tt.namespace, tt.typ, tt.ok) + } + } +} + +func TestProviderURLs(t *testing.T) { + urls := New("https://registry.terraform.io", nil).URLs() + other := New("https://registry.opentofu.org", nil).URLs() + tests := []struct { + name, got, want string + }{ + {"registry", urls.Registry("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0"}, + {"registry latest", urls.Registry("hashicorp/random", ""), "https://registry.terraform.io/providers/hashicorp/random/latest"}, + {"documentation", urls.Documentation("hashicorp/random", "3.6.0"), "https://registry.terraform.io/providers/hashicorp/random/3.6.0/docs"}, + // Archives are per platform; the download endpoint names them. + {"download", urls.Download("hashicorp/random", "3.6.0"), ""}, + {"purl", urls.PURL("hashicorp/random", "3.6.0"), "pkg:terraform/hashicorp/random@3.6.0"}, + {"purl without version", urls.PURL("hashicorp/random", ""), "pkg:terraform/hashicorp/random"}, + // Other registries have no pages at these paths. + {"other registry page", other.Registry("hashicorp/random", "3.6.0"), ""}, + {"other registry purl", other.PURL("hashicorp/random", "3.6.0"), + "pkg:terraform/hashicorp/random@3.6.0?repository_url=https:%2F%2Fregistry.opentofu.org"}, + } + for _, tt := range tests { + if tt.got != tt.want { + t.Errorf("%s = %q, want %q", tt.name, tt.got, tt.want) + } + } +} + +func TestFetchProviderVersionsSkipsEntriesWithoutAVersion(t *testing.T) { + server := providerServer(t, map[string]string{"/v1/providers/hashicorp/random/versions": `{"versions": [{"version": ""}, {"version": "3.6.0"}]}`}) + versions, err := New(server.URL, core.DefaultClient()).FetchVersions(context.Background(), "hashicorp/random") + if err != nil || len(versions) != 1 || versions[0].Number != "3.6.0" { + t.Errorf("versions = %v, %v; want only 3.6.0", versions, err) + } +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index a2e74b1..e317528 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -3,6 +3,7 @@ package terraform import ( "context" + "errors" "fmt" "sort" "strings" @@ -22,6 +23,8 @@ func init() { }) } +var errInvalidName = errors.New("terraform name must be 'namespace/type' for a provider or 'namespace/name/provider' for a module") + type Registry struct { baseURL string client *core.Client @@ -108,9 +111,12 @@ type providerEntry struct { } func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderPackage(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s", r.baseURL, namespace, moduleName, provider) @@ -141,9 +147,12 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package } func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + if namespace, typ, ok := parseProviderName(name); ok { + return r.fetchProviderVersions(ctx, name, namespace, typ) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", r.baseURL, namespace, moduleName, provider) @@ -176,9 +185,13 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi } func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + // Providers declare no dependencies. + if _, _, ok := parseProviderName(name); ok { + return nil, nil + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { - return nil, fmt.Errorf("terraform module name must be in format 'namespace/name/provider'") + return nil, errInvalidName } url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", r.baseURL, namespace, moduleName, provider, version) @@ -224,6 +237,9 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string) func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { namespace, _, _, ok := parseModuleName(name) + if !ok { + namespace, _, ok = parseProviderName(name) + } if !ok { return nil, nil } @@ -240,6 +256,9 @@ type URLs struct { } func (u *URLs) Registry(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "") + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return "" @@ -250,7 +269,12 @@ func (u *URLs) Registry(name, version string) string { return fmt.Sprintf("https://registry.terraform.io/modules/%s/%s/%s", namespace, moduleName, provider) } +// Download is the module's download endpoint. A provider has an archive per +// platform, named by the download endpoint for that platform, so it has none. func (u *URLs) Download(name, version string) string { + if _, _, ok := parseProviderName(name); ok { + return "" + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok || version == "" { return "" @@ -259,10 +283,16 @@ func (u *URLs) Download(name, version string) string { } func (u *URLs) Documentation(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPage(namespace, typ, version, "/docs") + } return u.Registry(name, version) } func (u *URLs) PURL(name, version string) string { + if namespace, typ, ok := parseProviderName(name); ok { + return u.providerPURL(namespace, typ, version) + } namespace, moduleName, provider, ok := parseModuleName(name) if !ok { return ""