Add a Debian repository client

README.md

@@ -136,6 +136,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5)
 | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` |
 | GitHub Actions | `pkg:githubactions/actions/checkout@v4` |
 | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` |
+| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` |
 
 ## Direct Registry Usage
 
@@ -196,11 +197,14 @@ import _ "github.com/git-pkgs/registries/all"
 | Terraform | `terraform` | https://registry.terraform.io |
 | Swift | `swift` | URL required |
 | Alpine | `apk` | URL required |
+| Debian | `deb` | URL required |
 | GitHub Actions | `githubactions` | https://api.github.com |
 | APM | None | URL required |
 
 The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index.
 
+The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists/<suite>/InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded.
+
 GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL.
 
 ### Swift registries

all/all.go

@@ -9,7 +9,7 @@
 //
 //	// Now all ecosystems are available
 //	ecosystems := registries.SupportedEcosystems()
-//	// ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"]
+//	// ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"]
 package all
 
 import (
@@ -21,6 +21,7 @@ import (
 	_ "github.com/git-pkgs/registries/internal/conda"
 	_ "github.com/git-pkgs/registries/internal/cpan"
 	_ "github.com/git-pkgs/registries/internal/cran"
+	_ "github.com/git-pkgs/registries/internal/deb"
 	_ "github.com/git-pkgs/registries/internal/deno"
 	_ "github.com/git-pkgs/registries/internal/dub"
 	_ "github.com/git-pkgs/registries/internal/elm"

client/client.go

@@ -121,6 +121,20 @@ func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decod
 	return nil
 }
 
+// Stream fetches a URL and passes the response body to fn, for large bodies
+// that are not JSON, such as compressed indexes. Bodies are limited to
+// MaxStreamSize: reading past it fails with ErrResponseTooLarge. Retries
+// cover getting a response; an error from fn is returned as is.
+func (c *Client) Stream(ctx context.Context, url string, fn func(io.Reader) error) error {
+	resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	limit := c.maxStreamSize()
+	return fn(&limitedReader{r: resp.Body, remaining: limit, limit: limit, url: url})
+}
+
 func (c *Client) maxResponseSize() int64 {
 	if c.MaxResponseSize > 0 {
 		return c.MaxResponseSize

client/client_test.go

@@ -293,3 +293,47 @@ func TestHeadRetriesAndRateLimits(t *testing.T) {
 		t.Errorf("rate limiter waits = %d, want 3", limiter.waits)
 	}
 }
+
+func TestStream(t *testing.T) {
+	const doc = "Package: curl\nVersion: 7.88.1-10\n"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/missing" {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(doc))
+	}))
+	defer srv.Close()
+
+	readAll := func(c *Client, url string) (string, error) {
+		var got []byte
+		err := c.Stream(context.Background(), url, func(r io.Reader) error {
+			var err error
+			got, err = io.ReadAll(r)
+			return err
+		})
+		return string(got), err
+	}
+
+	got, err := readAll(NewClient(WithMaxResponseSize(10), WithMaxRetries(0)), srv.URL)
+	if err != nil || got != doc {
+		t.Errorf("Stream = %q, %v; the buffered limit should not apply", got, err)
+	}
+	if _, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc)-1)), WithMaxRetries(0)), srv.URL); !errors.Is(err, ErrResponseTooLarge) {
+		t.Errorf("over the stream limit: err = %v, want ErrResponseTooLarge", err)
+	}
+	if got, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc))), WithMaxRetries(0)), srv.URL); err != nil || got != doc {
+		t.Errorf("at the stream limit: %q, %v", got, err)
+	}
+
+	var httpErr *HTTPError
+	_, err = readAll(NewClient(WithMaxRetries(0)), srv.URL+"/missing")
+	if !errors.As(err, &httpErr) || httpErr.StatusCode != http.StatusNotFound {
+		t.Errorf("missing: err = %v, want a 404 HTTPError", err)
+	}
+
+	sentinel := errors.New("stop")
+	if err := NewClient().Stream(context.Background(), srv.URL, func(io.Reader) error { return sentinel }); !errors.Is(err, sentinel) {
+		t.Errorf("fn error: err = %v, want it returned as is", err)
+	}
+}

fetch/deb_resolver_test.go

@@ -0,0 +1,64 @@
+//go:build !tinygo
+
+package fetch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/git-pkgs/registries"
+	_ "github.com/git-pkgs/registries/internal/deb"
+	"github.com/git-pkgs/registries/safehttp"
+)
+
+// debArchive serves bookworm amd64 and arm64 plain Packages indexes with no
+// Release, over TLS: the resolver only hands out https download URLs.
+func debArchive(t *testing.T) (string, *registries.Client) {
+	t.Helper()
+	indexes := map[string]string{
+		"/debian/dists/bookworm/main/binary-amd64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n" +
+			"Filename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\nSize: 3\nSHA256: aa\n",
+		"/debian/dists/bookworm/main/binary-arm64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: arm64\n" +
+			"Filename: pool/main/c/curl/curl_7.88.1-10_arm64.deb\nSize: 4\nSHA256: bb\n",
+	}
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, ok := indexes[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(server.Close)
+	client := registries.DefaultClient()
+	client.HTTPClient = server.Client()
+	return server.URL + "/debian", client
+}
+
+func TestResolveDebByQualifiers(t *testing.T) {
+	base, client := debArchive(t)
+	reg, err := registries.New("deb", base+"?suites=bookworm&architectures=amd64,arm64", client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true}))
+	r.RegisterRegistry(reg)
+
+	info, err := r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "arm64", "distro": "bookworm"}})
+	if err != nil {
+		t.Fatalf("Resolve: %v", err)
+	}
+	if want := base + "/pool/main/c/curl/curl_7.88.1-10_arm64.deb"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 {
+		t.Errorf("info = %+v, want %s", info, want)
+	}
+
+	_, err = r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "amd64", "distro": "trixie"}})
+	if !errors.Is(err, ErrNoMatchingArtifact) {
+		t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err)
+	}
+}

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
 	github.com/rubyist/circuitbreaker v2.2.1+incompatible
+	github.com/ulikunitz/xz v0.5.17
 	go.yaml.in/yaml/v3 v3.0.6
 )
 

go.sum

@@ -30,6 +30,8 @@ github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6
 github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4=
+github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
 go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
 go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=