Add an RPM repository client

README.md

@@ -136,6 +136,8 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5)
 | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` |
 | GitHub Actions | `pkg:githubactions/actions/checkout@v4` |
 | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` |
+| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` |
+| RPM | `pkg:rpm/fedora/curl@8.15.0-4.fc43?arch=x86_64&distro=fedora-43&repository_url=https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os%3Fdistro%3Dfedora-43` |
 
 ## Direct Registry Usage
 
@@ -196,11 +198,17 @@ import _ "github.com/git-pkgs/registries/all"
 | Terraform | `terraform` | https://registry.terraform.io |
 | Swift | `swift` | URL required |
 | Alpine | `apk` | URL required |
+| Debian | `deb` | URL required |
+| RPM | `rpm` | URL required |
 | GitHub Actions | `githubactions` | https://api.github.com |
 | APM | None | URL required |
 
 The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index.
 
+The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists/<suite>/InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded.
+
+The RPM URL is one repository, the directory that holds `repodata/`, such as `https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os`. The client doesn't pick a release, variant or architecture from a mirror root, and doesn't follow mirror lists or metalinks. It reads `repodata/repomd.xml`, follows its `primary` entry, checks that file against the checksum and size listed there, and decodes `.zst`, `.xz`, `.gz` or plain XML as a stream. Repository metadata doesn't name its distribution, so the `distro` to record on artifacts is given as a query parameter: `.../x86_64/os?distro=fedora-43`. Each artifact records `arch`, `epoch` and that `distro`, with the package's own checksum as integrity (`sha256-...`). The version number is `version-release`, as in the `rpm` PURL type, where the epoch is a qualifier; versions are ordered by epoch, version and release. Dependencies are the requirements that name packages, leaving out file paths, sonames and other capabilities. In a PURL, `repository_url` is the whole URL, `distro` parameter included, so its `?` and `=` are percent-encoded.
+
 GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL.
 
 ### Swift registries

all/all.go

@@ -9,7 +9,7 @@
 //
 //	// Now all ecosystems are available
 //	ecosystems := registries.SupportedEcosystems()
-//	// ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"]
+//	// ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform"]
 package all
 
 import (
@@ -21,6 +21,7 @@ import (
 	_ "github.com/git-pkgs/registries/internal/conda"
 	_ "github.com/git-pkgs/registries/internal/cpan"
 	_ "github.com/git-pkgs/registries/internal/cran"
+	_ "github.com/git-pkgs/registries/internal/deb"
 	_ "github.com/git-pkgs/registries/internal/deno"
 	_ "github.com/git-pkgs/registries/internal/dub"
 	_ "github.com/git-pkgs/registries/internal/elm"
@@ -40,6 +41,7 @@ import (
 	_ "github.com/git-pkgs/registries/internal/packagist"
 	_ "github.com/git-pkgs/registries/internal/pub"
 	_ "github.com/git-pkgs/registries/internal/pypi"
+	_ "github.com/git-pkgs/registries/internal/rpm"
 	_ "github.com/git-pkgs/registries/internal/rubygems"
 	_ "github.com/git-pkgs/registries/internal/swift"
 	_ "github.com/git-pkgs/registries/internal/terraform"

client/client.go

@@ -121,6 +121,20 @@ func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decod
 	return nil
 }
 
+// Stream fetches a URL and passes the response body to fn, for large bodies
+// that are not JSON, such as compressed indexes. Bodies are limited to
+// MaxStreamSize: reading past it fails with ErrResponseTooLarge. Retries
+// cover getting a response; an error from fn is returned as is.
+func (c *Client) Stream(ctx context.Context, url string, fn func(io.Reader) error) error {
+	resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	limit := c.maxStreamSize()
+	return fn(&limitedReader{r: resp.Body, remaining: limit, limit: limit, url: url})
+}
+
 func (c *Client) maxResponseSize() int64 {
 	if c.MaxResponseSize > 0 {
 		return c.MaxResponseSize

client/client_test.go

@@ -293,3 +293,47 @@ func TestHeadRetriesAndRateLimits(t *testing.T) {
 		t.Errorf("rate limiter waits = %d, want 3", limiter.waits)
 	}
 }
+
+func TestStream(t *testing.T) {
+	const doc = "Package: curl\nVersion: 7.88.1-10\n"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/missing" {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(doc))
+	}))
+	defer srv.Close()
+
+	readAll := func(c *Client, url string) (string, error) {
+		var got []byte
+		err := c.Stream(context.Background(), url, func(r io.Reader) error {
+			var err error
+			got, err = io.ReadAll(r)
+			return err
+		})
+		return string(got), err
+	}
+
+	got, err := readAll(NewClient(WithMaxResponseSize(10), WithMaxRetries(0)), srv.URL)
+	if err != nil || got != doc {
+		t.Errorf("Stream = %q, %v; the buffered limit should not apply", got, err)
+	}
+	if _, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc)-1)), WithMaxRetries(0)), srv.URL); !errors.Is(err, ErrResponseTooLarge) {
+		t.Errorf("over the stream limit: err = %v, want ErrResponseTooLarge", err)
+	}
+	if got, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc))), WithMaxRetries(0)), srv.URL); err != nil || got != doc {
+		t.Errorf("at the stream limit: %q, %v", got, err)
+	}
+
+	var httpErr *HTTPError
+	_, err = readAll(NewClient(WithMaxRetries(0)), srv.URL+"/missing")
+	if !errors.As(err, &httpErr) || httpErr.StatusCode != http.StatusNotFound {
+		t.Errorf("missing: err = %v, want a 404 HTTPError", err)
+	}
+
+	sentinel := errors.New("stop")
+	if err := NewClient().Stream(context.Background(), srv.URL, func(io.Reader) error { return sentinel }); !errors.Is(err, sentinel) {
+		t.Errorf("fn error: err = %v, want it returned as is", err)
+	}
+}

fetch/deb_resolver_test.go

@@ -0,0 +1,64 @@
+//go:build !tinygo
+
+package fetch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/git-pkgs/registries"
+	_ "github.com/git-pkgs/registries/internal/deb"
+	"github.com/git-pkgs/registries/safehttp"
+)
+
+// debArchive serves bookworm amd64 and arm64 plain Packages indexes with no
+// Release, over TLS: the resolver only hands out https download URLs.
+func debArchive(t *testing.T) (string, *registries.Client) {
+	t.Helper()
+	indexes := map[string]string{
+		"/debian/dists/bookworm/main/binary-amd64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n" +
+			"Filename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\nSize: 3\nSHA256: aa\n",
+		"/debian/dists/bookworm/main/binary-arm64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: arm64\n" +
+			"Filename: pool/main/c/curl/curl_7.88.1-10_arm64.deb\nSize: 4\nSHA256: bb\n",
+	}
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, ok := indexes[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(server.Close)
+	client := registries.DefaultClient()
+	client.HTTPClient = server.Client()
+	return server.URL + "/debian", client
+}
+
+func TestResolveDebByQualifiers(t *testing.T) {
+	base, client := debArchive(t)
+	reg, err := registries.New("deb", base+"?suites=bookworm&architectures=amd64,arm64", client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true}))
+	r.RegisterRegistry(reg)
+
+	info, err := r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "arm64", "distro": "bookworm"}})
+	if err != nil {
+		t.Fatalf("Resolve: %v", err)
+	}
+	if want := base + "/pool/main/c/curl/curl_7.88.1-10_arm64.deb"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 {
+		t.Errorf("info = %+v, want %s", info, want)
+	}
+
+	_, err = r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "amd64", "distro": "trixie"}})
+	if !errors.Is(err, ErrNoMatchingArtifact) {
+		t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err)
+	}
+}

fetch/rpm_resolver_test.go

@@ -0,0 +1,67 @@
+//go:build !tinygo
+
+package fetch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/git-pkgs/registries"
+	_ "github.com/git-pkgs/registries/internal/rpm"
+	"github.com/git-pkgs/registries/safehttp"
+)
+
+// rpmRepository serves a repository with one version of curl for two
+// architectures, over TLS: the resolver only hands out https download URLs.
+func rpmRepository(t *testing.T) (string, *registries.Client) {
+	t.Helper()
+	files := map[string]string{
+		"/os/repodata/repomd.xml": `<repomd><data type="primary"><location href="repodata/primary.xml"/></data></repomd>`,
+		"/os/repodata/primary.xml": `<metadata>
+<package type="rpm"><name>curl</name><arch>x86_64</arch><version epoch="0" ver="8.15.0" rel="4.fc43"/>
+  <checksum type="sha256">aa</checksum><size package="3"/><location href="Packages/c/curl-8.15.0-4.fc43.x86_64.rpm"/></package>
+<package type="rpm"><name>curl</name><arch>aarch64</arch><version epoch="0" ver="8.15.0" rel="4.fc43"/>
+  <checksum type="sha256">bb</checksum><size package="4"/><location href="Packages/c/curl-8.15.0-4.fc43.aarch64.rpm"/></package>
+</metadata>`,
+	}
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, ok := files[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(server.Close)
+	client := registries.DefaultClient()
+	client.HTTPClient = server.Client()
+	return server.URL + "/os", client
+}
+
+func TestResolveRPMByQualifiers(t *testing.T) {
+	base, client := rpmRepository(t)
+	reg, err := registries.New("rpm", base+"?distro=fedora-43", client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true}))
+	r.RegisterRegistry(reg)
+
+	info, err := r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64", "distro": "fedora-43"}})
+	if err != nil {
+		t.Fatalf("Resolve: %v", err)
+	}
+	if want := base + "/Packages/c/curl-8.15.0-4.fc43.aarch64.rpm"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 {
+		t.Errorf("info = %+v, want %s", info, want)
+	}
+
+	_, err = r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "x86_64", "distro": "fedora-42"}})
+	if !errors.Is(err, ErrNoMatchingArtifact) {
+		t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err)
+	}
+}

go.mod

@@ -12,9 +12,11 @@ require (
 	github.com/git-pkgs/purl v0.2.1
 	github.com/git-pkgs/spdx v0.3.6
 	github.com/git-pkgs/vers v0.7.2
+	github.com/klauspost/compress v1.20.3
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
 	github.com/rubyist/circuitbreaker v2.2.1+incompatible
+	github.com/ulikunitz/xz v0.5.17
 	go.yaml.in/yaml/v3 v3.0.6
 )
 

go.sum

@@ -16,6 +16,8 @@ github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs=
 github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A=
 github.com/github/go-spdx/v2 v2.8.0 h1:WKCnF2Ee/x+zwSIEeQdm6+be0H6PiCNLQmSZ5CFmxMI=
 github.com/github/go-spdx/v2 v2.8.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
+github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
+github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY=
@@ -30,6 +32,8 @@ github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6
 github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4=
+github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
 go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
 go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=