diff --git a/README.md b/README.md index d953bfe..d19d9e5 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,8 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | +| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` | +| RPM | `pkg:rpm/fedora/curl@8.15.0-4.fc43?arch=x86_64&distro=fedora-43&repository_url=https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os%3Fdistro%3Dfedora-43` | ## Direct Registry Usage @@ -196,11 +198,17 @@ import _ "github.com/git-pkgs/registries/all" | Terraform | `terraform` | https://registry.terraform.io | | Swift | `swift` | URL required | | Alpine | `apk` | URL required | +| Debian | `deb` | URL required | +| RPM | `rpm` | URL required | | GitHub Actions | `githubactions` | https://api.github.com | | APM | None | URL required | The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index. +The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded. + +The RPM URL is one repository, the directory that holds `repodata/`, such as `https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os`. The client doesn't pick a release, variant or architecture from a mirror root, and doesn't follow mirror lists or metalinks. It reads `repodata/repomd.xml`, follows its `primary` entry, checks that file against the checksum and size listed there, and decodes `.zst`, `.xz`, `.gz` or plain XML as a stream. Repository metadata doesn't name its distribution, so the `distro` to record on artifacts is given as a query parameter: `.../x86_64/os?distro=fedora-43`. Each artifact records `arch`, `epoch` and that `distro`, with the package's own checksum as integrity (`sha256-...`). The version number is `version-release`, as in the `rpm` PURL type, where the epoch is a qualifier; versions are ordered by epoch, version and release. Dependencies are the requirements that name packages, leaving out file paths, sonames and other capabilities. In a PURL, `repository_url` is the whole URL, `distro` parameter included, so its `?` and `=` are percent-encoded. + GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. ### Swift registries diff --git a/all/all.go b/all/all.go index 897535a..770a157 100644 --- a/all/all.go +++ b/all/all.go @@ -9,7 +9,7 @@ // // // Now all ecosystems are available // ecosystems := registries.SupportedEcosystems() -// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] +// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform"] package all import ( @@ -21,6 +21,7 @@ import ( _ "github.com/git-pkgs/registries/internal/conda" _ "github.com/git-pkgs/registries/internal/cpan" _ "github.com/git-pkgs/registries/internal/cran" + _ "github.com/git-pkgs/registries/internal/deb" _ "github.com/git-pkgs/registries/internal/deno" _ "github.com/git-pkgs/registries/internal/dub" _ "github.com/git-pkgs/registries/internal/elm" @@ -40,6 +41,7 @@ import ( _ "github.com/git-pkgs/registries/internal/packagist" _ "github.com/git-pkgs/registries/internal/pub" _ "github.com/git-pkgs/registries/internal/pypi" + _ "github.com/git-pkgs/registries/internal/rpm" _ "github.com/git-pkgs/registries/internal/rubygems" _ "github.com/git-pkgs/registries/internal/swift" _ "github.com/git-pkgs/registries/internal/terraform" diff --git a/client/client.go b/client/client.go index aedf379..88df2d1 100644 --- a/client/client.go +++ b/client/client.go @@ -121,6 +121,20 @@ func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decod return nil } +// Stream fetches a URL and passes the response body to fn, for large bodies +// that are not JSON, such as compressed indexes. Bodies are limited to +// MaxStreamSize: reading past it fails with ErrResponseTooLarge. Retries +// cover getting a response; an error from fn is returned as is. +func (c *Client) Stream(ctx context.Context, url string, fn func(io.Reader) error) error { + resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + limit := c.maxStreamSize() + return fn(&limitedReader{r: resp.Body, remaining: limit, limit: limit, url: url}) +} + func (c *Client) maxResponseSize() int64 { if c.MaxResponseSize > 0 { return c.MaxResponseSize diff --git a/client/client_test.go b/client/client_test.go index 0c377ea..cec5761 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -293,3 +293,47 @@ func TestHeadRetriesAndRateLimits(t *testing.T) { t.Errorf("rate limiter waits = %d, want 3", limiter.waits) } } + +func TestStream(t *testing.T) { + const doc = "Package: curl\nVersion: 7.88.1-10\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/missing" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(doc)) + })) + defer srv.Close() + + readAll := func(c *Client, url string) (string, error) { + var got []byte + err := c.Stream(context.Background(), url, func(r io.Reader) error { + var err error + got, err = io.ReadAll(r) + return err + }) + return string(got), err + } + + got, err := readAll(NewClient(WithMaxResponseSize(10), WithMaxRetries(0)), srv.URL) + if err != nil || got != doc { + t.Errorf("Stream = %q, %v; the buffered limit should not apply", got, err) + } + if _, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc)-1)), WithMaxRetries(0)), srv.URL); !errors.Is(err, ErrResponseTooLarge) { + t.Errorf("over the stream limit: err = %v, want ErrResponseTooLarge", err) + } + if got, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc))), WithMaxRetries(0)), srv.URL); err != nil || got != doc { + t.Errorf("at the stream limit: %q, %v", got, err) + } + + var httpErr *HTTPError + _, err = readAll(NewClient(WithMaxRetries(0)), srv.URL+"/missing") + if !errors.As(err, &httpErr) || httpErr.StatusCode != http.StatusNotFound { + t.Errorf("missing: err = %v, want a 404 HTTPError", err) + } + + sentinel := errors.New("stop") + if err := NewClient().Stream(context.Background(), srv.URL, func(io.Reader) error { return sentinel }); !errors.Is(err, sentinel) { + t.Errorf("fn error: err = %v, want it returned as is", err) + } +} diff --git a/fetch/deb_resolver_test.go b/fetch/deb_resolver_test.go new file mode 100644 index 0000000..0d9a678 --- /dev/null +++ b/fetch/deb_resolver_test.go @@ -0,0 +1,64 @@ +//go:build !tinygo + +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/deb" + "github.com/git-pkgs/registries/safehttp" +) + +// debArchive serves bookworm amd64 and arm64 plain Packages indexes with no +// Release, over TLS: the resolver only hands out https download URLs. +func debArchive(t *testing.T) (string, *registries.Client) { + t.Helper() + indexes := map[string]string{ + "/debian/dists/bookworm/main/binary-amd64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n" + + "Filename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\nSize: 3\nSHA256: aa\n", + "/debian/dists/bookworm/main/binary-arm64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: arm64\n" + + "Filename: pool/main/c/curl/curl_7.88.1-10_arm64.deb\nSize: 4\nSHA256: bb\n", + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := indexes[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + client := registries.DefaultClient() + client.HTTPClient = server.Client() + return server.URL + "/debian", client +} + +func TestResolveDebByQualifiers(t *testing.T) { + base, client := debArchive(t) + reg, err := registries.New("deb", base+"?suites=bookworm&architectures=amd64,arm64", client) + if err != nil { + t.Fatal(err) + } + r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true})) + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10", + ResolveOptions{Qualifiers: map[string]string{"arch": "arm64", "distro": "bookworm"}}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if want := base + "/pool/main/c/curl/curl_7.88.1-10_arm64.deb"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 { + t.Errorf("info = %+v, want %s", info, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10", + ResolveOptions{Qualifiers: map[string]string{"arch": "amd64", "distro": "trixie"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/fetch/rpm_resolver_test.go b/fetch/rpm_resolver_test.go new file mode 100644 index 0000000..6911bdc --- /dev/null +++ b/fetch/rpm_resolver_test.go @@ -0,0 +1,67 @@ +//go:build !tinygo + +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/rpm" + "github.com/git-pkgs/registries/safehttp" +) + +// rpmRepository serves a repository with one version of curl for two +// architectures, over TLS: the resolver only hands out https download URLs. +func rpmRepository(t *testing.T) (string, *registries.Client) { + t.Helper() + files := map[string]string{ + "/os/repodata/repomd.xml": ``, + "/os/repodata/primary.xml": ` +curlx86_64 + aa +curlaarch64 + bb +`, + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + client := registries.DefaultClient() + client.HTTPClient = server.Client() + return server.URL + "/os", client +} + +func TestResolveRPMByQualifiers(t *testing.T) { + base, client := rpmRepository(t) + reg, err := registries.New("rpm", base+"?distro=fedora-43", client) + if err != nil { + t.Fatal(err) + } + r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true})) + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64", "distro": "fedora-43"}}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if want := base + "/Packages/c/curl-8.15.0-4.fc43.aarch64.rpm"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 { + t.Errorf("info = %+v, want %s", info, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "x86_64", "distro": "fedora-42"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/go.mod b/go.mod index 4e0d676..9a14bd8 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,11 @@ require ( github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 + github.com/klauspost/compress v1.20.3 github.com/opencontainers/go-digest v1.0.0 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rubyist/circuitbreaker v2.2.1+incompatible + github.com/ulikunitz/xz v0.5.17 go.yaml.in/yaml/v3 v3.0.6 ) diff --git a/go.sum b/go.sum index 756e1e8..f6e12a2 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= github.com/github/go-spdx/v2 v2.8.0 h1:WKCnF2Ee/x+zwSIEeQdm6+be0H6PiCNLQmSZ5CFmxMI= github.com/github/go-spdx/v2 v2.8.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= @@ -30,6 +32,8 @@ github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6 github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/deb/deb.go b/internal/deb/deb.go new file mode 100644 index 0000000..7740942 --- /dev/null +++ b/internal/deb/deb.go @@ -0,0 +1,536 @@ +// Package deb provides a registry client for Debian repositories, read from +// their Packages indexes. +package deb + +import ( + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/url" + "path" + "sort" + "strings" + "sync" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzlimit" + "github.com/git-pkgs/vers" +) + +const ( + DefaultURL = "" + ecosystem = "deb" + + // maxIndexSize bounds one decompressed Packages index. noble universe + // amd64 is about 73 MB. + maxIndexSize = 512 << 20 + archAll = "all" +) + +var ( + // ErrInvalidConfig is returned when the repository URL names no + // archive, suites or architectures, or asks for source packages. + ErrInvalidConfig = errors.New("invalid Debian repository configuration") + // ErrNoIndex is returned when no configured suite, component and + // architecture has a Packages index. + ErrNoIndex = errors.New("no Debian Packages index found") + // ErrChecksumMismatch is returned when an index doesn't match the + // SHA256 its suite's Release lists. + ErrChecksumMismatch = errors.New("index does not match its Debian Release checksum") + // ErrMalformedIndex is returned for Release files and indexes that + // cannot be parsed. + ErrMalformedIndex = errors.New("malformed Debian index") +) + +// indexVariants are tried in this order, moving on after a 404. +var indexVariants = []string{"Packages.xz", "Packages.gz", "Packages"} + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads the Packages indexes of one archive. The URL is the +// archive root, with the suites, components and architectures to read as +// query parameters, which archives don't use: +// +// https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64 +// +// components defaults to main. An archive under another root, such as +// deb.debian.org/debian-security, is a Registry of its own. +// +// The indexes are downloaded once per Registry and kept for its lifetime, +// so callers looking up many packages should reuse one Registry. +type Registry struct { + archiveURL string + suites []string + components []string + architectures []string + configErr error + client *core.Client + urls *URLs + + maxIndexSize int64 + + mu sync.Mutex + packages map[string][]record +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + r := &Registry{client: client, maxIndexSize: maxIndexSize} + r.configErr = r.configure(baseURL) + r.urls = &URLs{repositoryURL: r.repositoryURL()} + return r +} + +// repositoryURL is the URL that configures this Registry, as a PURL's +// repository_url carries it: the archive root with what to read from it. +func (r *Registry) repositoryURL() string { + if r.archiveURL == "" { + return "" + } + return r.archiveURL + "?suites=" + strings.Join(r.suites, ",") + + "&components=" + strings.Join(r.components, ",") + + "&architectures=" + strings.Join(r.architectures, ",") +} + +func (r *Registry) configure(baseURL string) error { + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("%w: %q is not an archive URL", ErrInvalidConfig, baseURL) + } + q := u.Query() + u.RawQuery, u.Fragment = "", "" + r.archiveURL = strings.TrimSuffix(u.String(), "/") + r.suites = splitList(q.Get("suites")) + r.components = splitList(q.Get("components")) + r.architectures = splitList(q.Get("architectures")) + if len(r.components) == 0 { + r.components = []string{"main"} + } + switch { + case len(r.suites) == 0: + return fmt.Errorf("%w: no suites in %q", ErrInvalidConfig, baseURL) + case len(r.architectures) == 0: + return fmt.Errorf("%w: no architectures in %q", ErrInvalidConfig, baseURL) + } + for _, list := range [][]string{r.suites, r.components, r.architectures} { + for _, item := range list { + if !safeSegment(item) { + return fmt.Errorf("%w: %q", ErrInvalidConfig, item) + } + } + } + for _, arch := range r.architectures { + if arch == "source" { + return fmt.Errorf("%w: source packages are not supported", ErrInvalidConfig) + } + } + return nil +} + +func splitList(s string) []string { + var out []string + for _, item := range strings.Split(s, ",") { + if item = strings.TrimSpace(item); item != "" { + out = append(out, item) + } + } + return out +} + +// safeSegment refuses suite, component and architecture names that would +// leave the dists/ path they are placed in. +func safeSegment(s string) bool { + return s != "." && s != ".." && !strings.ContainsAny(s, `/\?#%`) +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// index reads every configured index, once. Nothing is kept unless all of +// them were read and checked, so a failed read is tried again next time. +func (r *Registry) index(ctx context.Context) (map[string][]record, error) { + if r.configErr != nil { + return nil, r.configErr + } + r.mu.Lock() + defer r.mu.Unlock() + if r.packages != nil { + return r.packages, nil + } + + packages := map[string][]record{} + seen := map[string]bool{} + // A version is recorded once per release: bookworm-updates repeating + // one, or binary-all repeating an Architecture: all package, adds + // nothing, while the same version in trixie is another artifact. + add := func(rec record) { + key := rec.name + "\x00" + rec.version + "\x00" + rec.arch + "\x00" + rec.distro + if seen[key] { + return + } + seen[key] = true + packages[rec.name] = append(packages[rec.name], rec) + } + read := 0 + for _, suite := range r.suites { + n, err := r.readSuite(ctx, suite, add) + if err != nil { + return nil, err + } + read += n + } + if read == 0 { + return nil, fmt.Errorf("%w: under %s for suites %v", ErrNoIndex, r.archiveURL, r.suites) + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].version, recs[j].version, ecosystem) > 0 + }) + } + r.packages = packages + return packages, nil +} + +// readSuite reads a suite's indexes and returns how many it found. +func (r *Registry) readSuite(ctx context.Context, suite string, add func(record)) (int, error) { + rel, err := r.fetchRelease(ctx, suite) + if err != nil { + return 0, err + } + codename := suite + if rel != nil && rel.codename != "" { + codename = rel.codename + } + + read := 0 + for _, component := range r.components { + dirs := make([]string, 0, len(r.architectures)+1) + for _, arch := range r.architectures { + dirs = append(dirs, component+"/binary-"+arch) + } + // Architecture: all packages are in every binary- index, + // and in binary-all where the Release lists one. + if rel != nil && rel.lists(component+"/binary-"+archAll) { + dirs = append(dirs, component+"/binary-"+archAll) + } + for _, dir := range dirs { + found, err := r.readIndex(ctx, suite, dir, rel, func(rec record) { + rec.suite, rec.component, rec.distro = suite, component, distro(codename) + add(rec) + }) + if err != nil { + return 0, err + } + if found { + read++ + } + } + } + return read, nil +} + +// fetchRelease reads InRelease, falling back to Release. It returns nil +// when the suite has neither, and its indexes are read unchecked. +func (r *Registry) fetchRelease(ctx context.Context, suite string) (*release, error) { + for _, name := range []string{"InRelease", "Release"} { + body, err := r.client.GetBody(ctx, r.archiveURL+"/dists/"+suite+"/"+name) + if isNotFound(err) { + continue + } + if err != nil { + return nil, err + } + return parseRelease(body) + } + return nil, nil +} + +// lists reports whether the Release has an index variant for dir. +func (rel *release) lists(dir string) bool { + for _, variant := range indexVariants { + if _, ok := rel.files[dir+"/"+variant]; ok { + return true + } + } + return false +} + +// readIndex reads the first variant of dir's index that is served. With a +// Release, only variants it lists are tried, and each is checked against +// its entry. It reports whether an index was read: a directory the Release +// doesn't list, or without a Release one the archive doesn't serve, has +// none. A directory the Release lists must be served, or a mirror caught +// mid-sync would leave its packages out for the Registry's lifetime. +func (r *Registry) readIndex(ctx context.Context, suite, dir string, rel *release, fn func(record)) (bool, error) { + listed := false + for _, variant := range indexVariants { + var sum *fileSum + if rel != nil && len(rel.files) > 0 { + entry, ok := rel.files[dir+"/"+variant] + if !ok { + continue + } + sum = &entry + listed = true + } + indexURL := r.archiveURL + "/dists/" + suite + "/" + dir + "/" + variant + var recs []record + err := r.client.Stream(ctx, indexURL, func(body io.Reader) error { + var err error + recs, err = r.parseIndex(body, variant, indexURL, sum) + return err + }) + if isNotFound(err) { + continue + } + if err != nil { + return false, err + } + for _, rec := range recs { + fn(rec) + } + return true, nil + } + if listed { + return false, fmt.Errorf("%w: %s/dists/%s/%s is listed in the Release but not served", ErrNoIndex, r.archiveURL, suite, dir) + } + return false, nil +} + +// parseIndex decompresses and parses one index, checking the compressed +// bytes against sum when there is one. +func (r *Registry) parseIndex(body io.Reader, variant, indexURL string, sum *fileSum) ([]record, error) { + hasher := sha256.New() + counted := &countingReader{r: io.TeeReader(body, hasher)} + + var decoded io.Reader + switch variant { + case "Packages.xz": + // Bounded as it decodes: a block asking for more is an error. + decoded = xzlimit.NewReader(counted, xzlimit.MaxDict) + case "Packages.gz": + gz, err := gzip.NewReader(counted) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + defer func() { _ = gz.Close() }() + decoded = gz + default: + decoded = counted + } + + limited := &io.LimitedReader{R: decoded, N: r.maxIndexSize + 1} + var recs []record + if err := parseStanzas(limited, func(rec record) { recs = append(recs, rec) }); err != nil { + return nil, err + } + if limited.N == 0 { + return nil, fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedIndex, indexURL, r.maxIndexSize) + } + // The decoders and the parser read to the end of the body. + if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { + return nil, err + } + return recs, nil +} + +func checkSum(h hash.Hash, size int64, sum *fileSum, indexURL string) error { + if sum == nil { + return nil + } + if got := hex.EncodeToString(h.Sum(nil)); got != sum.sha256 || size != sum.size { + return fmt.Errorf("%w: %s is %d bytes with SHA256 %s, Release lists %d bytes with %s", + ErrChecksumMismatch, indexURL, size, got, sum.size, sum.sha256) + } + return nil +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +func isNotFound(err error) bool { + var httpErr *core.HTTPError + return errors.As(err, &httpErr) && httpErr.IsNotFound() +} + +// splitName strips a PURL vendor namespace: "debian/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + packages, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.description, + Homepage: latest.homepage, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "source": latest.source, + "section": latest.section, + }, + }, nil +} + +// FetchVersions returns one Version per version number, newest first, with +// an artifact for each architecture and suite it was found in. +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + var versions []core.Version + for _, rec := range recs { + if len(versions) == 0 || versions[len(versions)-1].Number != rec.version { + versions = append(versions, core.Version{ + Number: rec.version, + Metadata: map[string]any{ + "source": rec.source, + "section": rec.section, + "maintainer": rec.maintainer, + "installed_size": rec.installedSize, + }, + }) + } + if artifact, ok := r.artifact(rec); ok { + v := &versions[len(versions)-1] + v.Artifacts = append(v.Artifacts, artifact) + } + } + return versions, nil +} + +func (r *Registry) artifact(rec record) (core.Artifact, bool) { + if !safePoolPath(rec.filename) { + return core.Artifact{}, false + } + artifact := core.Artifact{ + URL: r.archiveURL + "/" + rec.filename, + Filename: path.Base(rec.filename), + Size: rec.size, + Metadata: map[string]any{ + "arch": rec.arch, + "distro": rec.distro, + "suite": rec.suite, + "component": rec.component, + }, + } + if rec.sha256 != "" { + artifact.Integrity = "sha256-" + strings.ToLower(rec.sha256) + } + return artifact, true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version != version { + continue + } + var deps []core.Dependency + for _, d := range parseDependencies(rec.preDepends, rec.depends) { + deps = append(deps, core.Dependency{Name: d.name, Requirements: d.requirement, Scope: core.Runtime}) + } + return deps, nil + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].maintainer == "" { + return nil, nil + } + return []core.Maintainer{parseMaintainer(recs[0].maintainer)}, nil +} + +// parseMaintainer parses "Name "; any other value is kept whole as the name. +func parseMaintainer(value string) core.Maintainer { + if open := strings.LastIndex(value, "<"); open >= 0 && strings.HasSuffix(value, ">") { + name := strings.TrimSpace(value[:open]) + email := value[open+1 : len(value)-1] + if email != "" && strings.Contains(email, "@") && !strings.ContainsAny(email, "<> ") { + return core.Maintainer{Name: name, Email: email} + } + } + return core.Maintainer{Name: strings.TrimSpace(value)} +} + +type URLs struct { + repositoryURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns "": a package's file is only known from the index. +func (u *URLs) Download(name, version string) string { + return "" +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + var qualifiers map[string]string + if u.repositoryURL != "" { + qualifiers = map[string]string{"repository_url": u.repositoryURL} + } + return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() +} diff --git a/internal/deb/deb_test.go b/internal/deb/deb_test.go new file mode 100644 index 0000000..796257e --- /dev/null +++ b/internal/deb/deb_test.go @@ -0,0 +1,714 @@ +package deb + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/registries/internal/core" + "github.com/ulikunitz/xz" +) + +// Trimmed from deb.debian.org's bookworm main/binary-amd64 and +// bookworm-updates indexes. +const bookwormAMD64 = `Package: curl +Version: 7.88.1-10+deb12u5 +Installed-Size: 500 +Maintainer: Alessandro Ghedini +Architecture: amd64 +Depends: libc6 (>= 2.34), libcurl4 (= 7.88.1-10+deb12u5), zlib1g (>= 1:1.1.4) +Description: command line tool for transferring data with URL syntax + curl is a command line tool for transferring data with URL syntax. +Homepage: https://curl.se/ +Section: web +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u5_amd64.deb +Size: 315860 +SHA256: 1b2f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f80 + +Package: openssh-client +Source: openssh +Version: 1:9.2p1-2+deb12u3 +Maintainer: Debian OpenSSH Maintainers +Architecture: amd64 +Pre-Depends: libc6 (>= 2.34) +Depends: adduser, passwd, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>= 1:1.1.4), libselinux1 [linux-any] +Tag: implemented-in::c, interface::commandline, interface::shell, + network::client, protocol::sftp +Description: secure shell (SSH) client, for secure access to remote machines +Section: net +Filename: pool/main/o/openssh/openssh-client_9.2p1-2+deb12u3_amd64.deb +Size: 991056 +SHA256: 2c3f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f81 + +Package: ca-certificates +Version: 20230311 +Maintainer: Julien Cristau +Architecture: all +Depends: openssl (>= 1.1.1), debconf (>= 0.5) | debconf-2.0 +Description: Common CA certificates +Filename: pool/main/c/ca-certificates/ca-certificates_20230311_all.deb +Size: 153656 +SHA256: 3d4f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f82 +` + +const bookwormARM64 = `Package: curl +Version: 7.88.1-10+deb12u5 +Maintainer: Alessandro Ghedini +Architecture: arm64 +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u5_arm64.deb +Size: 301024 +SHA256: 4e5f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f83 +` + +const updatesAMD64 = `Package: curl +Version: 7.88.1-10+deb12u14 +Maintainer: Alessandro Ghedini +Architecture: amd64 +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u14_amd64.deb +Size: 316112 +SHA256: 5f6f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f84 +` + +func xzData(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w, err := xz.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func gzData(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// archive is a fake Debian archive. served are the files it answers with; +// listed are entries its Release files name, served or not. +type archive struct { + t *testing.T + mu sync.Mutex + served map[string][]byte + listed map[string][]byte + requests []string +} + +func newArchive(t *testing.T) *archive { + return &archive{t: t, served: map[string][]byte{}, listed: map[string][]byte{}} +} + +// index adds one Packages index of suite, as variant ("Packages.xz", +// "Packages.gz" or "Packages"), served and listed. +func (a *archive) index(suite, dir, variant, stanzas string) { + data := []byte(stanzas) + switch variant { + case "Packages.xz": + data = xzData(a.t, stanzas) + case "Packages.gz": + data = gzData(a.t, stanzas) + } + path := "dists/" + suite + "/" + dir + "/" + variant + a.served[path] = data + a.listed[path] = data +} + +// release writes the suite's Release (or InRelease, clearsigned) over the +// files listed under it. +func (a *archive) release(suite, codename string, signed bool) { + prefix := "dists/" + suite + "/" + var paths []string + for p := range a.listed { + if strings.HasPrefix(p, prefix) { + paths = append(paths, p) + } + } + sort.Strings(paths) + var b strings.Builder + fmt.Fprintf(&b, "Origin: Debian\nSuite: %s\nCodename: %s\nArchitectures: all amd64 arm64\nComponents: main\nMD5Sum:\n", suite, codename) + for _, p := range paths { + fmt.Fprintf(&b, " 00000000000000000000000000000000 %8d %s\n", len(a.listed[p]), strings.TrimPrefix(p, prefix)) + } + b.WriteString("SHA256:\n") + for _, p := range paths { + sum := sha256.Sum256(a.listed[p]) + fmt.Fprintf(&b, " %s %8d %s\n", hex.EncodeToString(sum[:]), len(a.listed[p]), strings.TrimPrefix(p, prefix)) + } + if !signed { + a.served[prefix+"Release"] = []byte(b.String()) + return + } + a.served[prefix+"InRelease"] = []byte("-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + b.String() + + "-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCAAdFiEE\n-----END PGP SIGNATURE-----\n") +} + +func (a *archive) server() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/debian/") + a.mu.Lock() + a.requests = append(a.requests, path) + data, ok := a.served[path] + a.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(data) + })) + a.t.Cleanup(srv.Close) + return srv +} + +// standard is bookworm (InRelease, xz, amd64 and arm64) plus +// bookworm-updates (plain Release, gz only). +func standard(t *testing.T) (*archive, string) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64) + a.release("bookworm", "bookworm", true) + a.index("bookworm-updates", "main/binary-amd64", "Packages.gz", updatesAMD64) + a.release("bookworm-updates", "bookworm-updates", false) + srv := a.server() + return a, srv.URL + "/debian" +} + +func newRegistry(base, query string) *Registry { + return New(base+"?"+query, core.DefaultClient()) +} + +func numbers(versions []core.Version) []string { + var out []string + for _, v := range versions { + out = append(out, v.Number) + } + return out +} + +func TestFetchVersions(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&components=main&architectures=amd64") + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"7.88.1-10+deb12u14", "7.88.1-10+deb12u5"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (newest first, by deb version)", numbers(versions), want) + } + + newest := versions[0].Artifacts + if len(newest) != 1 { + t.Fatalf("artifacts = %+v, want one", newest) + } + got := newest[0] + want := core.Artifact{ + URL: base + "/pool/main/c/curl/curl_7.88.1-10+deb12u14_amd64.deb", + Filename: "curl_7.88.1-10+deb12u14_amd64.deb", + Integrity: "sha256-5f6f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f84", + Size: 316112, + Metadata: map[string]any{"arch": "amd64", "distro": "bookworm", "suite": "bookworm-updates", "component": "main"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("artifact =\n %+v\nwant\n %+v", got, want) + } + if d := versions[1].Artifacts[0].Metadata["distro"]; d != "bookworm" { + t.Errorf("bookworm artifact distro = %v", d) + } +} + +func TestFetchVersionsAcrossArchitectures(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + var archs []any + for _, a := range versions[0].Artifacts { + archs = append(archs, a.Metadata["arch"]) + } + if want := []any{"amd64", "arm64"}; !reflect.DeepEqual(archs, want) { + t.Errorf("artifact archs = %v, want %v", archs, want) + } +} + +// Architecture: all packages appear in every binary- index and in +// binary-all where the Release lists one; they are recorded once, with +// their own architecture. +func TestArchitectureAll(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64+"\n"+caCertificatesOnly) + a.index("bookworm", "main/binary-all", "Packages.xz", caCertificatesOnly) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + versions, err := r.FetchVersions(context.Background(), "ca-certificates") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if arts := versions[0].Artifacts; len(arts) != 1 || arts[0].Metadata["arch"] != "all" { + t.Errorf("artifacts = %+v, want one, arch all", arts) + } + if !a.requested("dists/bookworm/main/binary-all/Packages.xz") { + t.Errorf("binary-all was not read: %v", a.requests) + } +} + +const caCertificatesOnly = `Package: ca-certificates +Version: 20230311 +Architecture: all +Filename: pool/main/c/ca-certificates/ca-certificates_20230311_all.deb +Size: 153656 +SHA256: 3d4f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f82 +` + +func (a *archive) requested(path string) bool { + a.mu.Lock() + defer a.mu.Unlock() + for _, p := range a.requests { + if p == path { + return true + } + } + return false +} + +// deb.debian.org lists a plain Packages it answers 404 for, and other +// archives list no .xz: the variants are tried in order past a 404. +func TestIndexVariantFallback(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.listed["dists/bookworm/main/binary-amd64/Packages"] = []byte(bookwormAMD64) + delete(a.served, "dists/bookworm/main/binary-amd64/Packages.xz") + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + delete(a.served, "dists/bookworm/main/binary-amd64/Packages.gz") + a.served["dists/bookworm/main/binary-amd64/Packages"] = []byte(bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } +} + +// Variants a Release doesn't list are not requested. +func TestIndexVariantsNotListedAreSkipped(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + a.release("bookworm", "bookworm", true) + a.served["dists/bookworm/main/binary-amd64/Packages.xz"] = []byte("not an index") + base := a.server().URL + "/debian" + + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if a.requested("dists/bookworm/main/binary-amd64/Packages.xz") { + t.Error("requested a variant the Release doesn't list") + } +} + +// Without a Release, as some Artifactory setups serve, the index is read +// without a checksum to check against. +func TestNoRelease(t *testing.T) { + a := newArchive(t) + a.index("stable", "main/binary-amd64", "Packages.gz", bookwormAMD64) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=stable&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if d := versions[0].Artifacts[0].Metadata["distro"]; d != "stable" { + t.Errorf("distro = %v, want the suite name without a Release", d) + } +} + +func TestChecksumMismatch(t *testing.T) { + for _, variant := range []string{"Packages.xz", "Packages.gz", "Packages"} { + t.Run(variant, func(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", variant, bookwormAMD64) + a.release("bookworm", "bookworm", true) + path := "dists/bookworm/main/binary-amd64/" + variant + switch variant { + case "Packages.xz": + a.served[path] = xzData(t, updatesAMD64) + case "Packages.gz": + a.served[path] = gzData(t, updatesAMD64) + default: + a.served[path] = []byte(updatesAMD64) + } + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64") + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("err = %v, want ErrChecksumMismatch", err) + } + // Nothing was kept: the next lookup reads the archive again. + a.served[path] = a.listed[path] + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("after the archive was fixed: %v", err) + } + }) + } +} + +func TestInvalidConfiguration(t *testing.T) { + _, base := standard(t) + for name, url := range map[string]string{ + "no url": "", + "no host": "/debian?suites=bookworm&architectures=amd64", + "suite path": base + "?suites=../../etc&architectures=amd64", + "component path": base + "?suites=bookworm&components=a/b&architectures=amd64", + "architecture path": base + "?suites=bookworm&architectures=..", + "no suites": base + "?architectures=amd64", + "no architectures": base + "?suites=bookworm", + "source": base + "?suites=bookworm&architectures=source", + } { + t.Run(name, func(t *testing.T) { + _, err := New(url, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("err = %v, want ErrInvalidConfig", err) + } + }) + } +} + +func TestNoIndexFound(t *testing.T) { + a := newArchive(t) + base := a.server().URL + "/debian" + _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrNoIndex) { + t.Errorf("err = %v, want ErrNoIndex", err) + } +} + +func TestNotFound(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "missing"); !errors.As(err, ¬Found) { + t.Errorf("missing package: err = %v, want NotFoundError", err) + } + if _, err := r.FetchDependencies(context.Background(), "curl", "1.0-1"); !errors.As(err, ¬Found) || notFound.Version != "1.0-1" { + t.Errorf("missing version: err = %v, want NotFoundError for 1.0-1", err) + } +} + +// The epoch is part of Version but never of Filename. +func TestEpochVersion(t *testing.T) { + _, base := standard(t) + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "debian/openssh-client") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if v := versions[0]; v.Number != "1:9.2p1-2+deb12u3" || v.Artifacts[0].Filename != "openssh-client_9.2p1-2+deb12u3_amd64.deb" { + t.Errorf("version = %q, filename = %q", v.Number, v.Artifacts[0].Filename) + } +} + +func TestFetchPackage(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&architectures=amd64") + pkg, err := r.FetchPackage(context.Background(), "debian/curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "curl" || pkg.Namespace != "debian" || pkg.LatestVersion != "7.88.1-10+deb12u14" { + t.Errorf("name = %q, namespace = %q, latest = %q", pkg.Name, pkg.Namespace, pkg.LatestVersion) + } + pkg, err = r.FetchPackage(context.Background(), "openssh-client") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Description != "secure shell (SSH) client, for secure access to remote machines" || pkg.Metadata["source"] != "openssh" || pkg.Metadata["section"] != "net" { + t.Errorf("package = %+v", pkg) + } +} + +func TestFetchDependencies(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + deps, err := r.FetchDependencies(context.Background(), "openssh-client", "1:9.2p1-2+deb12u3") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "libc6", Requirements: ">= 2.34", Scope: core.Runtime}, + {Name: "adduser", Scope: core.Runtime}, + {Name: "passwd", Scope: core.Runtime}, + {Name: "libssl3", Requirements: ">= 3.0.13", Scope: core.Runtime}, + {Name: "zlib1g", Requirements: ">= 1:1.1.4", Scope: core.Runtime}, + {Name: "libselinux1", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +func TestFetchMaintainers(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + got, err := r.FetchMaintainers(context.Background(), "curl") + if err != nil || !reflect.DeepEqual(got, []core.Maintainer{{Name: "Alessandro Ghedini", Email: "ghedo@debian.org"}}) { + t.Errorf("maintainers = %+v, %v", got, err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://deb.debian.org/debian/?architectures=amd64&suites=bookworm,bookworm-updates", nil).URLs() + if got := u.Download("curl", "7.88.1-10"); got != "" { + t.Errorf("Download = %q, want none without the index", got) + } + // repository_url is the whole configured URL, so the PURL names what + // to read. + const repo = "?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian%3Fsuites%3Dbookworm%2Cbookworm-updates%26components%3Dmain%26architectures%3Damd64" + if got, want := u.PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10"+repo; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } + if got, want := u.PURL("debian/curl", "1:9.2p1-2"), "pkg:deb/debian/curl@1:9.2p1-2"+repo; got != want { + t.Errorf("PURL with epoch = %q, want %q", got, want) + } +} + +func TestPURLWithoutArchive(t *testing.T) { + if got, want := New("", nil).URLs().PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } +} + +// A PURL the registry builds leads back to the same archive and indexes. +func TestPURLRoundTrip(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&architectures=amd64") + reg, name, version, err := core.NewFromPURL(r.URLs().PURL("debian/curl", "7.88.1-10+deb12u14"), core.DefaultClient()) + if err != nil || name != "debian/curl" || version != "7.88.1-10+deb12u14" { + t.Fatalf("NewFromPURL = %q, %q, %v", name, version, err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 2 || versions[0].Number != version { + t.Errorf("FetchVersions = %v, %v", numbers(versions), err) + } +} + +// An index is read once per Registry. +func TestIndexReadOnce(t *testing.T) { + a, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + for range 3 { + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + } + n := 0 + for _, p := range a.requests { + if strings.HasSuffix(p, "/Packages.xz") { + n++ + } + } + if n != 1 { + t.Errorf("index requested %d times, want once", n) + } +} + +func TestDecompressedIndexLimit(t *testing.T) { + for _, variant := range []string{"Packages.xz", "Packages.gz", "Packages"} { + t.Run(variant, func(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", variant, bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxIndexSize = int64(len(bookwormAMD64)) - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("over the limit: err = %v, want ErrMalformedIndex", err) + } + r = newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxIndexSize = int64(len(bookwormAMD64)) + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } + }) + } +} + +// An .xz whose blocks ask for too large a dictionary is refused, even when +// the Release lists it. +func TestXZDictionaryRefused(t *testing.T) { + a := newArchive(t) + path := "dists/bookworm/main/binary-amd64/Packages.xz" + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.served[path] = setXZDict(t, a.served[path], 30) // 128 MiB + a.listed[path] = a.served[path] + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } +} + +// A suite named by its alias, such as stable, records the Release's +// codename. +func TestDistroFromRelease(t *testing.T) { + a := newArchive(t) + a.index("stable", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.release("stable", "bookworm", true) + base := a.server().URL + "/debian" + versions, err := newRegistry(base, "suites=stable&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatal(err) + } + if d := versions[0].Artifacts[0].Metadata["distro"]; d != "bookworm" { + t.Errorf("distro = %v, want the Release's codename", d) + } +} + +// A Release with no SHA256 entries, only MD5Sum, checks nothing and lists +// nothing, so every variant is tried. +func TestReleaseWithoutSHA256(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + a.served["dists/bookworm/Release"] = []byte("Codename: bookworm\nMD5Sum:\n 0123 12 main/binary-amd64/Packages.gz\n") + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +// Same size, other content: only the hash tells them apart. +func TestChecksumHashMismatch(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages", bookwormAMD64) + a.release("bookworm", "bookworm", true) + a.served["dists/bookworm/main/binary-amd64/Packages"] = []byte(strings.Replace(bookwormAMD64, "Size: 315860", "Size: 315861", 1)) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +// A Filename leaving the archive root gets no artifact. +func TestUnsafeFilename(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", "Package: evil\nVersion: 1\nArchitecture: amd64\nFilename: ../../etc/passwd\n") + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "evil") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 0 { + t.Errorf("versions = %+v, %v; want one without artifacts", versions, err) + } + // It has no Maintainer either. + r := newRegistry(base, "suites=bookworm&architectures=amd64") + if got, err := r.FetchMaintainers(context.Background(), "evil"); err != nil || got != nil { + t.Errorf("maintainers = %+v, %v; want none", got, err) + } +} + +// A Release whose SHA256 entry has the right hash but another size is +// rejected too. +func TestChecksumSizeMismatch(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages", bookwormAMD64) + a.release("bookworm", "bookworm", false) + rel := string(a.served["dists/bookworm/Release"]) + a.served["dists/bookworm/Release"] = []byte(strings.ReplaceAll(rel, + fmt.Sprintf(" %8d main/binary-amd64/Packages\n", len(bookwormAMD64)), + fmt.Sprintf(" %8d main/binary-amd64/Packages\n", len(bookwormAMD64)+1))) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +// The same version in two releases is one version with an artifact for +// each, so either distro resolves; within a release, -updates repeating a +// version adds nothing. +func TestSameVersionInTwoReleases(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("bookworm", "bookworm", true) + a.index("bookworm-updates", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("bookworm-updates", "bookworm-updates", true) + a.index("trixie", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("trixie", "trixie", true) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=bookworm,bookworm-updates,trixie&architectures=amd64").FetchVersions(context.Background(), "ca-certificates") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + var distros []any + for _, artifact := range versions[0].Artifacts { + distros = append(distros, artifact.Metadata["distro"]) + } + if want := []any{"bookworm", "trixie"}; !reflect.DeepEqual(distros, want) { + t.Errorf("artifact distros = %v, want %v", distros, want) + } +} + +// An index the Release lists but the archive doesn't serve, as on a mirror +// caught mid-sync, fails the read rather than leaving its packages out. +func TestListedIndexMissing(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64) + a.index("bookworm", "main/binary-arm64", "Packages.gz", bookwormARM64) + a.release("bookworm", "bookworm", true) + xzPath, gzPath := "dists/bookworm/main/binary-arm64/Packages.xz", "dists/bookworm/main/binary-arm64/Packages.gz" + xzData, gzData := a.served[xzPath], a.served[gzPath] + delete(a.served, xzPath) + delete(a.served, gzPath) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrNoIndex) { + t.Fatalf("err = %v, want ErrNoIndex", err) + } + // Nothing was kept, so the index is read once the mirror has it. + a.mu.Lock() + a.served[xzPath], a.served[gzPath] = xzData, gzData + a.mu.Unlock() + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 2 { + t.Errorf("after the mirror caught up: %+v, %v", versions, err) + } +} + +// An architecture the Release doesn't list at all is skipped, as before. +func TestUnlistedArchitectureSkipped(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64,riscv64").FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} diff --git a/internal/deb/parse.go b/internal/deb/parse.go new file mode 100644 index 0000000..76d3a50 --- /dev/null +++ b/internal/deb/parse.go @@ -0,0 +1,287 @@ +package deb + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" +) + +// maxLineSize bounds one index line. bookworm main has a 75,649-byte +// Provides: line. +const maxLineSize = 1 << 20 + +// record is one stanza of a Packages index, with the fields the client uses. +type record struct { + name string + version string + arch string + source string + maintainer string + description string + homepage string + section string + filename string + sha256 string + size int64 + installedSize int64 + depends string + preDepends string + + // Where the stanza was read. + suite string + component string + distro string +} + +// parseStanzas reads a Packages index as a stream, calling fn for each +// stanza. Field names are matched case-insensitively and folded lines are +// joined, as deb822(5) asks of readers. +func parseStanzas(r io.Reader, fn func(record)) error { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxLineSize) + var rec record + var field string // the field a continuation line belongs to + inStanza := false + lineNo := 0 + + finish := func() error { + if !inStanza { + return nil + } + if rec.name == "" || rec.version == "" { + return fmt.Errorf("%w: stanza ending at line %d has no Package or Version", ErrMalformedIndex, lineNo) + } + fn(rec) + rec, field, inStanza = record{}, "", false + return nil + } + + for scanner.Scan() { + lineNo++ + line := scanner.Text() + if strings.TrimSpace(line) == "" { + if err := finish(); err != nil { + return err + } + continue + } + if line[0] == ' ' || line[0] == '\t' { + if !inStanza { + return fmt.Errorf("%w: line %d continues no field", ErrMalformedIndex, lineNo) + } + rec.continueField(field, strings.TrimSpace(line)) + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + return fmt.Errorf("%w: line %d is not a field", ErrMalformedIndex, lineNo) + } + inStanza = true + field = strings.ToLower(key) + if err := rec.set(field, strings.TrimSpace(value)); err != nil { + return fmt.Errorf("%w: line %d: %w", ErrMalformedIndex, lineNo, err) + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + return finish() +} + +func (rec *record) set(field, value string) error { + var err error + switch field { + case "package": + rec.name = value + case "version": + rec.version = value + case "architecture": + rec.arch = value + case "source": + rec.source = value + case "maintainer": + rec.maintainer = value + case "description": + rec.description = value + case "homepage": + rec.homepage = value + case "section": + rec.section = value + case "filename": + rec.filename = value + case "sha256": + rec.sha256 = value + case "size": + rec.size, err = strconv.ParseInt(value, 10, 64) + case "installed-size": + rec.installedSize, err = strconv.ParseInt(value, 10, 64) + case "depends": + rec.depends = value + case "pre-depends": + rec.preDepends = value + } + return err +} + +// continueField appends a folded line to the relationship fields. Other +// multi-line fields, such as Description, keep their first line. +func (rec *record) continueField(field, value string) { + switch field { + case "depends": + rec.depends = joinFolded(rec.depends, value) + case "pre-depends": + rec.preDepends = joinFolded(rec.preDepends, value) + } +} + +func joinFolded(s, more string) string { + if s == "" { + return more + } + return s + " " + more +} + +// fileSum is a Release file's SHA256 entry for one index file. +type fileSum struct { + sha256 string + size int64 +} + +// release is the part of a Release or InRelease file the client uses. +type release struct { + codename string + suite string + // files maps a path under dists// to its SHA256 entry. + files map[string]fileSum +} + +// parseRelease reads a Release file, or an InRelease file, whose +// clearsigned text is read without checking the signature. +func parseRelease(data []byte) (*release, error) { + text, err := clearsignedText(data) + if err != nil { + return nil, err + } + rel := &release{files: map[string]fileSum{}} + scanner := bufio.NewScanner(bytes.NewReader(text)) + scanner.Buffer(nil, maxLineSize) + field := "" + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + if line[0] == ' ' || line[0] == '\t' { + if field != "sha256" { + continue + } + parts := strings.Fields(line) + if len(parts) != 3 { + return nil, fmt.Errorf("%w: Release SHA256 entry %q", ErrMalformedIndex, line) + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: Release SHA256 entry %q", ErrMalformedIndex, line) + } + rel.files[parts[2]] = fileSum{sha256: strings.ToLower(parts[0]), size: size} + continue + } + key, value, _ := strings.Cut(line, ":") + field = strings.ToLower(key) + switch field { + case "codename": + rel.codename = strings.TrimSpace(value) + case "suite": + rel.suite = strings.TrimSpace(value) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + return rel, nil +} + +// clearsignedText returns the signed text of an OpenPGP clearsigned +// message, with dash escapes removed, or data unchanged if it is not one. +// A clearsigned message without its header separator or its signature is +// an error rather than an empty Release, which would check nothing. +func clearsignedText(data []byte) ([]byte, error) { + const begin, sig = "-----BEGIN PGP SIGNED MESSAGE-----", "-----BEGIN PGP SIGNATURE-----" + if !bytes.HasPrefix(data, []byte(begin)) { + return data, nil + } + data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) + // Armor headers such as Hash: end at the first blank line. + _, body, ok := bytes.Cut(data, []byte("\n\n")) + if !ok { + return nil, fmt.Errorf("%w: clearsigned Release has no header separator", ErrMalformedIndex) + } + end := bytes.Index(body, []byte("\n"+sig)) + if end < 0 { + return nil, fmt.Errorf("%w: clearsigned Release has no signature", ErrMalformedIndex) + } + lines := bytes.SplitAfter(body[:end+1], []byte("\n")) + for i, line := range lines { + lines[i] = bytes.TrimPrefix(line, []byte("- ")) + } + return bytes.Join(lines, nil), nil +} + +// distro is the codename a suite's packages are recorded under: +// bookworm-updates and bookworm-security are bookworm. +func distro(codename string) string { + name, _, _ := strings.Cut(codename, "-") + return name +} + +// dependency is one relationship from Depends or Pre-Depends. +type dependency struct { + name string + requirement string +} + +// parseDependencies reads relationship fields. Of alternatives ("a | b") +// the first is kept, as apt tries it first; architecture qualifiers +// (zlib1g:any), architecture restrictions ([linux-any]) and build profiles +// () are dropped. +func parseDependencies(fields ...string) []dependency { + var deps []dependency + for _, field := range fields { + for _, group := range strings.Split(field, ",") { + first, _, _ := strings.Cut(group, "|") + // The version comes first: its << and >> are not profiles. + name, rest, versioned := strings.Cut(first, "(") + requirement := "" + if versioned { + requirement, _, _ = strings.Cut(rest, ")") + } + // What is left of the name ends at a qualifier, restriction or + // profile. + if i := strings.IndexAny(name, ":[<"); i >= 0 { + name = name[:i] + } + if name = strings.TrimSpace(name); name == "" { + continue + } + deps = append(deps, dependency{name: name, requirement: strings.TrimSpace(requirement)}) + } + } + return deps +} + +// safePoolPath reports whether a Filename stays under the archive root: +// relative, without dot segments, escapes or URL syntax. +func safePoolPath(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.ContainsAny(p, `\?#%:`) { + return false + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go new file mode 100644 index 0000000..e5193d5 --- /dev/null +++ b/internal/deb/parse_test.go @@ -0,0 +1,183 @@ +package deb + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "reflect" + "strings" + "testing" +) + +func TestParseStanzas(t *testing.T) { + long := strings.Repeat("libfoo, ", 10_000) // longer than bufio's 64 KiB default + index := "package: Foo\nVERSION: 1.0-1\nArchitecture: amd64\n" + + "Depends: libc6 (>= 2.34),\n libssl3 (>= 3)\n" + + "Provides: " + long + "\n" + + "Description: first line\n second line\n" + + "Filename: pool/f/foo_1.0-1_amd64.deb\n \t\n" + + "Package: bar\nVersion: 2\nArchitecture: all\n" + var got []record + err := parseStanzas(strings.NewReader(index), func(rec record) { got = append(got, rec) }) + if err != nil { + t.Fatalf("parseStanzas: %v", err) + } + if len(got) != 2 { + t.Fatalf("records = %+v, want two", got) + } + foo := got[0] + if foo.name != "Foo" || foo.version != "1.0-1" || foo.arch != "amd64" || foo.filename != "pool/f/foo_1.0-1_amd64.deb" { + t.Errorf("foo = %+v", foo) + } + if foo.depends != "libc6 (>= 2.34), libssl3 (>= 3)" { + t.Errorf("folded Depends = %q", foo.depends) + } + if foo.description != "first line" { + t.Errorf("description = %q, want its first line", foo.description) + } + if got[1].name != "bar" || got[1].arch != "all" { + t.Errorf("bar = %+v", got[1]) + } +} + +func TestParseStanzasMalformed(t *testing.T) { + for name, index := range map[string]string{ + "no colon": "Package: foo\nVersion 1\n", + "no package": "Version: 1\nArchitecture: amd64\n", + "no version": "Package: foo\n\nPackage: bar\nVersion: 1\n", + "leading continuation": " continued\nPackage: foo\nVersion: 1\n", + } { + t.Run(name, func(t *testing.T) { + err := parseStanzas(strings.NewReader(index), func(record) {}) + if !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } + }) + } +} + +func TestParseRelease(t *testing.T) { + data := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + + "Suite: oldstable-updates\nCodename: bookworm-updates\n" + + "Architectures: all amd64\nMD5Sum:\n 0123 12 main/binary-amd64/Packages.xz\n 4567 3 main/binary-arm64/Packages\n" + + "SHA256:\n abcd 12 main/binary-amd64/Packages.xz\n ef01 0 main/binary-all/Packages\n" + + "- Dash-Escaped: kept\n" + + "-----BEGIN PGP SIGNATURE-----\n\niQIz\n-----END PGP SIGNATURE-----\n" + + // Text after the signature isn't signed, and is not read. + "Codename: unsigned\nSHA256:\n 9999 1 main/binary-amd64/Packages\n" + rel, err := parseRelease([]byte(data)) + if err != nil { + t.Fatalf("parseRelease: %v", err) + } + if rel.codename != "bookworm-updates" { + t.Errorf("codename = %q", rel.codename) + } + want := map[string]fileSum{ + "main/binary-amd64/Packages.xz": {sha256: "abcd", size: 12}, + "main/binary-all/Packages": {sha256: "ef01", size: 0}, + } + if !reflect.DeepEqual(rel.files, want) { + t.Errorf("files = %+v, want %+v", rel.files, want) + } +} + +// A clearsigned file may use CRLF line endings. +func TestParseReleaseCRLF(t *testing.T) { + data := "-----BEGIN PGP SIGNED MESSAGE-----\r\nHash: SHA256\r\n\r\n" + + "Codename: bookworm\r\nSHA256:\r\n abcd 12 main/binary-amd64/Packages.xz\r\n" + + "-----BEGIN PGP SIGNATURE-----\r\n\r\niQIz\r\n-----END PGP SIGNATURE-----\r\n" + + "Codename: unsigned\r\n" + rel, err := parseRelease([]byte(data)) + if err != nil { + t.Fatalf("parseRelease: %v", err) + } + want := map[string]fileSum{"main/binary-amd64/Packages.xz": {sha256: "abcd", size: 12}} + if rel.codename != "bookworm" || !reflect.DeepEqual(rel.files, want) { + t.Errorf("release = %+v", rel) + } +} + +// A clearsigned file without its header separator or signature is not +// read as an empty Release, which would leave the indexes unchecked. +func TestParseReleaseBrokenEnvelope(t *testing.T) { + for name, data := range map[string]string{ + "no blank line": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\nCodename: bookworm\n", + "no signature": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nCodename: bookworm\n", + } { + if _, err := parseRelease([]byte(data)); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("%s: err = %v, want ErrMalformedIndex", name, err) + } + } +} + +func TestParseReleaseMalformedEntry(t *testing.T) { + for _, entry := range []string{" abcd notasize main/binary-amd64/Packages", " abcd 12 main/binary-amd64/Packages extra", " abcd 12"} { + if _, err := parseRelease([]byte("SHA256:\n" + entry + "\n")); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("%q: err = %v, want ErrMalformedIndex", entry, err) + } + } +} + +func TestDistroFromCodename(t *testing.T) { + for in, want := range map[string]string{ + "bookworm": "bookworm", + "bookworm-updates": "bookworm", + "bookworm-security": "bookworm", + "jammy-updates": "jammy", + } { + if got := distro(in); got != want { + t.Errorf("distro(%q) = %q, want %q", in, got, want) + } + } +} + +func TestParseDependencies(t *testing.T) { + got := parseDependencies("libc6 (>= 2.34)", "adduser, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>=1:1.1.4), libselinux1 [linux-any], libfoo , "+ + "libold (<< 2.0), libnew (>> 1.0) [amd64] , libexact (= 1:2-3), libbare[i386], libfirst | libsecond (>= 1)") + want := []dependency{ + {"libc6", ">= 2.34"}, {"adduser", ""}, {"libssl3", ">= 3.0.13"}, + {"zlib1g", ">=1:1.1.4"}, {"libselinux1", ""}, {"libfoo", ""}, + {"libold", "<< 2.0"}, {"libnew", ">> 1.0"}, {"libexact", "= 1:2-3"}, {"libbare", ""}, {"libfirst", ""}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseDependencies =\n %+v\nwant\n %+v", got, want) + } +} + +func TestPoolPath(t *testing.T) { + for in, ok := range map[string]bool{ + "pool/main/c/curl/curl_7.88.1-10_amd64.deb": true, + "": false, + "/etc/passwd": false, + "pool/../../etc/passwd": false, + "pool/x.deb?query": false, + "pool/x.deb#frag": false, + `pool\x.deb`: false, + "pool/%2e%2e/x.deb": false, + "http://evil.example/x.deb": false, + } { + if got := safePoolPath(in); got != ok { + t.Errorf("safePoolPath(%q) = %v, want %v", in, got, ok) + } + } +} + +// setXZDict rewrites the dictionary size of every LZMA2 block header in +// data, fixing the header CRC, so a small file claims a large dictionary. +func setXZDict(t *testing.T, data []byte, bits byte) []byte { + t.Helper() + out := bytes.Clone(data) + // A file written by xzData holds one stream with one block, whose + // header starts after the 12-byte stream header. + const start = 12 + size := (int(out[start]) + 1) * 4 + header := out[start : start+size] + i := bytes.Index(header, []byte{0x21, 0x01}) + if i < 0 { + t.Fatal("no LZMA2 filter in the block header") + } + header[i+2] = bits + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} diff --git a/internal/rpm/rpm.go b/internal/rpm/rpm.go new file mode 100644 index 0000000..6e5b0f9 --- /dev/null +++ b/internal/rpm/rpm.go @@ -0,0 +1,630 @@ +// Package rpm provides a registry client for RPM repositories, read from +// their repodata. +package rpm + +import ( + "compress/gzip" + "context" + "crypto/sha1" //nolint:gosec // repomd.xml checksums of old repositories + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/url" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzlimit" + "github.com/git-pkgs/vers" + "github.com/klauspost/compress/zstd" +) + +const ( + DefaultURL = "" + ecosystem = "rpm" + + repomdPath = "repodata/repomd.xml" + + // maxPrimarySize bounds the decompressed primary metadata. Fedora 44 + // Everything x86_64 is about 185 MB. + maxPrimarySize = 1 << 30 + // maxZstdWindow bounds the window a zstd frame may ask the decoder to + // allocate. zstd --ultra -22 uses 128 MiB. + maxZstdWindow = 128 << 20 +) + +var ( + // ErrInvalidConfig is returned when the repository URL is missing or + // is not a URL. + ErrInvalidConfig = errors.New("invalid RPM repository configuration") + // ErrChecksumMismatch is returned when the primary metadata doesn't + // match the checksum or size repomd.xml lists. + ErrChecksumMismatch = errors.New("primary metadata does not match its repomd.xml checksum") + // ErrMalformedMetadata is returned for repodata that cannot be parsed. + ErrMalformedMetadata = errors.New("malformed RPM repository metadata") + // ErrUnsupportedCompression is returned when the primary metadata is + // compressed in a format other than zstd, xz or gzip. + ErrUnsupportedCompression = errors.New("unsupported primary metadata compression") +) + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads the primary metadata of one repository: the directory +// that holds repodata/, such as +// https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os. +// The release, variant and architecture are never inferred from a mirror +// root, and mirror lists and metalinks are not followed. +// +// Repository metadata doesn't name its distribution, so the distro to +// record on artifacts, as in the PURL qualifier, is given as a query +// parameter, which repositories don't use: .../x86_64/os?distro=fedora-43. +// +// The metadata is downloaded once per Registry and kept for its lifetime, +// so callers looking up many packages should reuse one Registry. +type Registry struct { + repoURL string + distro string + configErr error + client *core.Client + urls *URLs + + maxPrimarySize int64 + + mu sync.Mutex + packages map[string][]record +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + r := &Registry{client: client, maxPrimarySize: maxPrimarySize} + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + r.configErr = fmt.Errorf("%w: %q is not a repository URL", ErrInvalidConfig, baseURL) + } else { + r.distro = u.Query().Get("distro") + u.RawQuery, u.Fragment = "", "" + r.repoURL = strings.TrimSuffix(u.String(), "/") + } + r.urls = &URLs{repositoryURL: r.repoURL} + // As a PURL's repository_url carries it: the repository with the + // distro it was configured with. + if r.distro != "" { + r.urls.repositoryURL += "?distro=" + r.distro + } + return r +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// record is one package of the primary metadata. +type record struct { + name string + arch string + epoch string + version string // ver-rel + checksumType string + checksum string + summary string + packager string + url string + license string + sourceRPM string + buildTime time.Time + size int64 + installedSize int64 + href string // "" when the file is not under the repository + requires []core.Dependency +} + +// evr is the form versions are compared in. +func (rec record) evr() string { + return rec.epoch + ":" + rec.version +} + +// index reads the repository's metadata, once. Nothing is kept unless it +// was read and checked whole, so a failed read is tried again next time. +func (r *Registry) index(ctx context.Context) (map[string][]record, error) { + if r.configErr != nil { + return nil, r.configErr + } + r.mu.Lock() + defer r.mu.Unlock() + if r.packages != nil { + return r.packages, nil + } + + body, err := r.client.GetBody(ctx, r.repoURL+"/"+repomdPath) + if err != nil { + return nil, err + } + primary, err := parseRepomd(body) + if err != nil { + return nil, err + } + + // Decided before the download, so an unsupported format costs none. + if !supportedPrimary(primary.href) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedCompression, primary.href) + } + + packages := map[string][]record{} + seen := map[string]bool{} + primaryURL := r.repoURL + "/" + primary.href + err = r.client.Stream(ctx, primaryURL, func(body io.Reader) error { + return r.readPrimary(body, primary, primaryURL, func(rec record) { + key := rec.name + "\x00" + rec.evr() + "\x00" + rec.arch + if seen[key] { + return + } + seen[key] = true + packages[rec.name] = append(packages[rec.name], rec) + }) + }) + if err != nil { + return nil, err + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].evr(), recs[j].evr(), ecosystem) > 0 + }) + } + r.packages = packages + return packages, nil +} + +// primaryData is repomd.xml's entry for the primary metadata. +type primaryData struct { + href string + checksumType string // "" when repomd.xml lists no checksum + checksum string + size int64 +} + +func parseRepomd(data []byte) (*primaryData, error) { + var repomd struct { + Data []struct { + Type string `xml:"type,attr"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + Size int64 `xml:"size"` + } `xml:"data"` + } + if err := xml.Unmarshal(data, &repomd); err != nil { + return nil, fmt.Errorf("%w: repomd.xml: %w", ErrMalformedMetadata, err) + } + for _, d := range repomd.Data { + if d.Type != "primary" { + continue + } + if !safeHref(d.Location.Href) { + return nil, fmt.Errorf("%w: repomd.xml: primary location %q", ErrMalformedMetadata, d.Location.Href) + } + return &primaryData{ + href: d.Location.Href, + checksumType: strings.ToLower(strings.TrimSpace(d.Checksum.Type)), + checksum: strings.ToLower(strings.TrimSpace(d.Checksum.Value)), + size: d.Size, + }, nil + } + return nil, fmt.Errorf("%w: repomd.xml lists no primary metadata", ErrMalformedMetadata) +} + +func newHash(checksumType string) (hash.Hash, bool) { + switch checksumType { + case "sha256": + return sha256.New(), true + case "sha512": + return sha512.New(), true + case "sha384": + return sha512.New384(), true + case "sha224": + return sha256.New224(), true + case "sha", "sha1": + return sha1.New(), true //nolint:gosec // as listed by the repository + } + return nil, false +} + +// readPrimary decompresses and parses the primary metadata, checking the +// compressed bytes against the checksum and size repomd.xml lists. +func (r *Registry) readPrimary(body io.Reader, primary *primaryData, primaryURL string, fn func(record)) error { + var hasher hash.Hash + counted := &countingReader{r: body} + if primary.checksumType != "" { + h, ok := newHash(primary.checksumType) + if !ok { + return fmt.Errorf("%w: repomd.xml: checksum type %q", ErrMalformedMetadata, primary.checksumType) + } + hasher = h + counted.r = io.TeeReader(body, hasher) + } + check := func() error { + if hasher == nil { + return nil + } + got := hex.EncodeToString(hasher.Sum(nil)) + if got != primary.checksum || (primary.size > 0 && counted.n != primary.size) { + return fmt.Errorf("%w: %s is %d bytes with %s %s, repomd.xml lists %d bytes with %s", + ErrChecksumMismatch, primaryURL, counted.n, primary.checksumType, got, primary.size, primary.checksum) + } + return nil + } + + var decoded io.Reader + switch { + case strings.HasSuffix(primary.href, ".zst"): + zr, err := zstd.NewReader(counted, zstd.WithDecoderMaxWindow(maxZstdWindow), zstd.WithDecoderConcurrency(1)) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer zr.Close() + decoded = zr + case strings.HasSuffix(primary.href, ".xz"): + // Bounded as it decodes: a block asking for more is an error. + decoded = xzlimit.NewReader(counted, xzlimit.MaxDict) + case strings.HasSuffix(primary.href, ".gz"): + gz, err := gzip.NewReader(counted) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer func() { _ = gz.Close() }() + decoded = gz + default: + decoded = counted + } + + limited := &io.LimitedReader{R: decoded, N: r.maxPrimarySize + 1} + if err := parsePrimary(limited, fn); err != nil { + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + return err + } + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + return check() +} + +// supportedPrimary reports whether href names primary metadata the client +// can decode: zstd, xz, gzip or plain XML. +func supportedPrimary(href string) bool { + for _, ext := range []string{".zst", ".xz", ".gz", ".xml"} { + if strings.HasSuffix(href, ext) { + return true + } + } + return false +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// xmlPackage is a of primary.xml, with the fields the client uses. +type xmlPackage struct { + Type string `xml:"type,attr"` + Name string `xml:"name"` + Arch string `xml:"arch"` + Version struct { + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"version"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Summary string `xml:"summary"` + Packager string `xml:"packager"` + URL string `xml:"url"` + Time struct { + Build int64 `xml:"build,attr"` + } `xml:"time"` + Size struct { + Package int64 `xml:"package,attr"` + Installed int64 `xml:"installed,attr"` + } `xml:"size"` + Location struct { + Href string `xml:"href,attr"` + Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"` + } `xml:"location"` + Format struct { + License string `xml:"license"` + SourceRPM string `xml:"sourcerpm"` + Requires []struct { + Name string `xml:"name,attr"` + Flags string `xml:"flags,attr"` + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"requires>entry"` + } `xml:"format"` +} + +// parsePrimary reads primary.xml as a stream, calling fn for each rpm +// package. +func parsePrimary(r io.Reader, fn func(record)) error { + dec := xml.NewDecoder(r) + for { + tok, err := dec.Token() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg xmlPackage + if err := dec.DecodeElement(&pkg, &start); err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + if pkg.Type != "rpm" { + continue + } + if pkg.Name == "" || pkg.Version.Ver == "" { + return fmt.Errorf("%w: primary.xml: package without a name or version", ErrMalformedMetadata) + } + fn(pkg.record()) + } +} + +func (pkg *xmlPackage) record() record { + rec := record{ + name: pkg.Name, + arch: pkg.Arch, + epoch: pkg.Version.Epoch, + version: evr("", pkg.Version.Ver, pkg.Version.Rel), + checksumType: strings.ToLower(pkg.Checksum.Type), + checksum: strings.ToLower(strings.TrimSpace(pkg.Checksum.Value)), + summary: pkg.Summary, + packager: pkg.Packager, + url: pkg.URL, + license: pkg.Format.License, + sourceRPM: pkg.Format.SourceRPM, + size: pkg.Size.Package, + installedSize: pkg.Size.Installed, + } + if rec.epoch == "" { + rec.epoch = "0" + } + if pkg.Time.Build > 0 { + rec.buildTime = time.Unix(pkg.Time.Build, 0).UTC() + } + // xml:base points the file at another repository, which is not followed. + if pkg.Location.Base == "" && safeHref(pkg.Location.Href) { + rec.href = pkg.Location.Href + } + seen := map[string]bool{} + for _, req := range pkg.Format.Requires { + // File paths, sonames and capabilities such as rpmlib(...) name no + // package. + if req.Name == "" || strings.HasPrefix(req.Name, "/") || strings.Contains(req.Name, "(") || seen[req.Name] { + continue + } + seen[req.Name] = true + rec.requires = append(rec.requires, core.Dependency{ + Name: req.Name, + Requirements: requirement(req.Flags, req.Epoch, req.Ver, req.Rel), + Scope: core.Runtime, + }) + } + return rec +} + +// evr joins an epoch, version and release, leaving out an epoch of 0 and +// an empty release. +func evr(epoch, ver, rel string) string { + s := ver + if epoch != "" && epoch != "0" { + s = epoch + ":" + s + } + if rel != "" { + s += "-" + rel + } + return s +} + +// requirement renders a requires entry's version constraint. +func requirement(flags, epoch, ver, rel string) string { + ops := map[string]string{"EQ": "=", "GE": ">=", "LE": "<=", "GT": ">", "LT": "<"} + op, ok := ops[flags] + if !ok || ver == "" { + return "" + } + return op + " " + evr(epoch, ver, rel) +} + +// safeHref reports whether a location stays under the repository root: +// relative, without dot segments, escapes or URL syntax. +func safeHref(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.ContainsAny(p, `\?#%:`) { + return false + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + +// splitName strips a PURL vendor namespace: "fedora/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + packages, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.summary, + Homepage: latest.url, + Licenses: latest.license, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "epoch": latest.epoch, + "source_rpm": latest.sourceRPM, + }, + }, nil +} + +// FetchVersions returns one Version per epoch, version and release, newest +// first, with an artifact for each architecture. The version number is +// version-release, as in the rpm PURL type, where the epoch is a qualifier. +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + var versions []core.Version + last := "" + for _, rec := range recs { + if len(versions) == 0 || last != rec.evr() { + last = rec.evr() + versions = append(versions, core.Version{ + Number: rec.version, + PublishedAt: rec.buildTime, + Licenses: rec.license, + Metadata: map[string]any{ + "epoch": rec.epoch, + "source_rpm": rec.sourceRPM, + "packager": rec.packager, + "installed_size": rec.installedSize, + }, + }) + } + if artifact, ok := r.artifact(rec); ok { + v := &versions[len(versions)-1] + v.Artifacts = append(v.Artifacts, artifact) + } + } + return versions, nil +} + +func (r *Registry) artifact(rec record) (core.Artifact, bool) { + if rec.href == "" { + return core.Artifact{}, false + } + artifact := core.Artifact{ + URL: r.repoURL + "/" + rec.href, + Filename: path.Base(rec.href), + Size: rec.size, + Metadata: map[string]any{"arch": rec.arch, "epoch": rec.epoch}, + } + if r.distro != "" { + artifact.Metadata["distro"] = r.distro + } + if rec.checksum != "" && rec.checksumType != "" { + artifact.Integrity = rec.checksumType + "-" + rec.checksum + } + return artifact, true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version == version { + return rec.requires, nil + } + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].packager == "" { + return nil, nil + } + return []core.Maintainer{{Name: recs[0].packager}}, nil +} + +type URLs struct { + repositoryURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns "": a package's file is only known from the metadata. +func (u *URLs) Download(name, version string) string { + return "" +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + var qualifiers map[string]string + if u.repositoryURL != "" { + qualifiers = map[string]string{"repository_url": u.repositoryURL} + } + return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() +} diff --git a/internal/rpm/rpm_test.go b/internal/rpm/rpm_test.go new file mode 100644 index 0000000..b334681 --- /dev/null +++ b/internal/rpm/rpm_test.go @@ -0,0 +1,664 @@ +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/internal/core" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// Trimmed from Fedora 43 updates' primary.xml, with a second architecture, +// an older build and a package with an epoch added. +const primaryXML = ` + + + curl + x86_64 + + 235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a + A utility for getting files from remote servers + curl is a command line tool +for transferring data. + Fedora Project + https://curl.se/ + + + curl + i686 + + 335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b + + + + curl + x86_64 + + AB5b92fa + + + + bind + x86_64 + + 435b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5c + + + + bind + x86_64 + + 535b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5d + + + + tzdata + noarch + + 635b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5e + + + +` + +func compress(t *testing.T, ext, s string) []byte { + t.Helper() + var buf bytes.Buffer + var w interface { + Write([]byte) (int, error) + Close() error + } + var err error + switch ext { + case ".zst": + w, err = zstd.NewWriter(&buf) + case ".xz": + w, err = xz.NewWriter(&buf) + case ".gz": + w = gzip.NewWriter(&buf) + default: + return []byte(s) + } + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// repo is a fake RPM repository under /os/. +type repo struct { + mu sync.Mutex + files map[string][]byte + requests []string +} + +// newRepo serves primary, compressed as ext ("" for plain .xml), and a +// repomd.xml listing it with its sha256 and size. +func newRepo(t *testing.T, ext, primary string) (*repo, string) { + t.Helper() + r := &repo{files: map[string][]byte{}} + data := compress(t, ext, primary) + sum := sha256.Sum256(data) + href := "repodata/" + hex.EncodeToString(sum[:]) + "-primary.xml" + ext + r.files[href] = data + r.setRepomd("sha256", hex.EncodeToString(sum[:]), href, len(data)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + path := strings.TrimPrefix(req.URL.Path, "/os/") + r.mu.Lock() + r.requests = append(r.requests, path) + body, ok := r.files[path] + r.mu.Unlock() + if !ok { + http.NotFound(w, req) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return r, srv.URL + "/os" +} + +func (r *repo) setRepomd(sumType, sum, href string, size int) { + r.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(` + + 1794529563 + + 00 + + 1 + + + %s + + %d + 1 + + +`, sumType, sum, href, size)) +} + +func (r *repo) primaryHref() string { + for path := range r.files { + if strings.Contains(path, "-primary.xml") { + return path + } + } + return "" +} + +func numbers(versions []core.Version) []string { + var out []string + for _, v := range versions { + out = append(out, v.Number) + } + return out +} + +func TestFetchVersions(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + versions, err := r.FetchVersions(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"8.15.0-10.fc43", "8.15.0-4.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (newest first, by rpm version)", numbers(versions), want) + } + older := versions[1] + if !older.PublishedAt.Equal(time.Unix(1771515699, 0)) { + t.Errorf("published = %v, want the build time", older.PublishedAt) + } + if older.Licenses != "curl" || older.Metadata["epoch"] != "0" || older.Metadata["source_rpm"] != "curl-8.15.0-4.fc43.src.rpm" { + t.Errorf("version = %+v", older) + } + want := []core.Artifact{ + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.x86_64.rpm", + Filename: "curl-8.15.0-4.fc43.x86_64.rpm", + Integrity: "sha256-235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a", + Size: 234567, + Metadata: map[string]any{"arch": "x86_64", "distro": "fedora-43", "epoch": "0"}, + }, + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.i686.rpm", + Filename: "curl-8.15.0-4.fc43.i686.rpm", + Integrity: "sha256-335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b", + Metadata: map[string]any{"arch": "i686", "distro": "fedora-43", "epoch": "0"}, + }, + } + if !reflect.DeepEqual(older.Artifacts, want) { + t.Errorf("artifacts =\n %+v\nwant\n %+v", older.Artifacts, want) + } + // The checksum's own type names the integrity, lowercased. + if got := versions[0].Artifacts[0].Integrity; got != "sha512-ab5b92fa" { + t.Errorf("sha512 integrity = %q", got) + } + }) + } +} + +// The epoch orders versions but is not part of the version number, as in +// the rpm PURL type, where it is a qualifier. +func TestEpoch(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "bind") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"9.18.41-1.fc43", "9.20.1-1.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (epoch 32 before epoch 31)", numbers(versions), want) + } + if e := versions[0].Metadata["epoch"]; e != "32" { + t.Errorf("epoch = %v", e) + } + if e := versions[0].Artifacts[0].Metadata["epoch"]; e != "32" { + t.Errorf("artifact epoch = %v", e) + } +} + +// Without a distro in the URL none is recorded, and noarch packages record +// their own architecture. +func TestNoarchAndNoDistro(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base+"/", core.DefaultClient()).FetchVersions(context.Background(), "tzdata") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 1 { + t.Fatalf("FetchVersions = %+v, %v", versions, err) + } + if got, want := versions[0].Artifacts[0].Metadata, (map[string]any{"arch": "noarch", "epoch": "0"}); !reflect.DeepEqual(got, want) { + t.Errorf("metadata = %v, want %v", got, want) + } + if got, want := versions[0].Artifacts[0].URL, base+"/Packages/t/tzdata-2026a-1.fc43.noarch.rpm"; got != want { + t.Errorf("URL = %q, want %q", got, want) + } +} + +// Primary metadata is streamed, so the client's limit on buffered +// responses doesn't apply to it. +func TestPrimaryLargerThanResponseCap(t *testing.T) { + _, base := newRepo(t, "", primaryXML) + client := core.NewClient(core.WithMaxRetries(0)) + client.MaxResponseSize = 1024 // repomd.xml fits, primary.xml doesn't + if _, err := New(base, client).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestChecksumMismatch(t *testing.T) { + other := strings.Replace(primaryXML, "8.15.0", "8.15.1", 1) + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + rp, base := newRepo(t, ext, primaryXML) + href := rp.primaryHref() + good := rp.files[href] + rp.files[href] = compress(t, ext, other) + + r := New(base, core.DefaultClient()) + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("err = %v, want ErrChecksumMismatch", err) + } + // Nothing was kept: the next lookup reads the repository again. + rp.files[href] = good + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("after the repository was fixed: %v", err) + } + }) + } +} + +// The right hash with another size is rejected too. +func TestChecksumSizeMismatch(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + sum := sha256.Sum256([]byte(primaryXML)) + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), rp.primaryHref(), len(primaryXML)+1) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +func TestChecksumTypes(t *testing.T) { + sum512 := sha512.Sum512([]byte(primaryXML)) + for name, tt := range map[string]struct { + sumType, sum string + wantErr error + }{ + "sha512": {"sha512", hex.EncodeToString(sum512[:]), nil}, + "sha512 upper": {"SHA512", strings.ToUpper(hex.EncodeToString(sum512[:])), nil}, + "sha512 wrong": {"sha512", "00", ErrChecksumMismatch}, + "unknown type": {"md5", "00", ErrMalformedMetadata}, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd(tt.sumType, tt.sum, rp.primaryHref(), len(primaryXML)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, tt.wantErr) { + t.Errorf("err = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestInvalidConfiguration(t *testing.T) { + for name, url := range map[string]string{"no url": "", "no host": "/pub/fedora"} { + t.Run(name, func(t *testing.T) { + _, err := New(url, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("err = %v, want ErrInvalidConfig", err) + } + }) + } +} + +func TestMalformedRepomd(t *testing.T) { + for name, repomd := range map[string]string{ + "not xml": "", + "no primary": ``, + "href leaves the repository": ``, + "absolute href": ``, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(repomd) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } +} + +func TestUnsupportedCompression(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd("sha256", "00", "repodata/primary.xml.zck", 1) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrUnsupportedCompression) { + t.Errorf("err = %v, want ErrUnsupportedCompression", err) + } +} + +// A repomd.xml that lists a checksum but no size checks the checksum alone. +func TestChecksumWithoutSize(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + href := rp.primaryHref() + sum := sha256.Sum256(rp.files[href]) + repomd := `%s` + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, hex.EncodeToString(sum[:]))) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, "00")) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("wrong checksum: err = %v, want ErrChecksumMismatch", err) + } +} + +// A repomd.xml without a checksum for primary leaves it unchecked. +func TestNoChecksum(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(``) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestMalformedPrimary(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz"} { + t.Run("not"+ext, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + garbage := []byte(strings.Repeat("not compressed data ", 10)) + sum := sha256.Sum256(garbage) + rp.files["repodata/primary.xml"+ext] = garbage + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml"+ext, len(garbage)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } + t.Run("truncated xml", func(t *testing.T) { + _, base := newRepo(t, "", primaryXML[:len(primaryXML)/2]) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + t.Run("package without a name", func(t *testing.T) { + _, base := newRepo(t, "", `x86_64`) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) +} + +func TestDecompressedLimit(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + // Cut mid-document, the limit is still what the error names, + // not the XML it broke. + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) / 2 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("far over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } + }) + } +} + +// A zstd frame asking for a 256 MiB window, which the decoder would allow +// by default, is refused rather than allocated. +func TestZstdWindowRefused(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + // Magic, a frame header with no content size, a window descriptor of + // 2^28 bytes, and one last raw block holding a document. + frame := append([]byte{0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x90, 0x61, 0x00, 0x00}, "\n"...) + sum := sha256.Sum256(frame) + rp.files["repodata/primary.xml.zst"] = frame + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml.zst", len(frame)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } +} + +func TestNotFound(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "missing"); !errors.As(err, ¬Found) { + t.Errorf("missing package: err = %v, want NotFoundError", err) + } + if _, err := r.FetchDependencies(context.Background(), "curl", "1.0-1"); !errors.As(err, ¬Found) || notFound.Version != "1.0-1" { + t.Errorf("missing version: err = %v, want NotFoundError for 1.0-1", err) + } +} + +// A package whose file is outside the repository, by its href or by an +// xml:base pointing elsewhere, gets no artifact. Other package types are +// not read. +func TestUnsafeLocations(t *testing.T) { + primary := ` +upx86_64 +basedx86_64 +nonex86_64 +barex86_64 +barex86_64 +otherx86_64 +` + _, base := newRepo(t, "", primary) + r := New(base, core.DefaultClient()) + for _, name := range []string{"up", "based", "none"} { + versions, err := r.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 0 { + t.Errorf("%s: versions = %+v, %v; want one without artifacts", name, versions, err) + } + } + // No epoch means 0, no checksum means no integrity, and a package + // listed twice is recorded once. + bare, err := r.FetchVersions(context.Background(), "bare") + if err != nil || len(bare) != 1 || len(bare[0].Artifacts) != 1 { + t.Fatalf("bare: versions = %+v, %v; want one with one artifact", bare, err) + } + if a := bare[0].Artifacts[0]; a.Integrity != "" || a.Metadata["epoch"] != "0" || a.Filename != "bare-1-1.x86_64.rpm" || !strings.HasSuffix(a.URL, "/Packages/bare-1-1.x86_64.rpm") { + t.Errorf("bare artifact = %+v", a) + } + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "other"); !errors.As(err, ¬Found) { + t.Errorf("non-rpm package: err = %v, want NotFoundError", err) + } + if got, err := r.FetchMaintainers(context.Background(), "up"); err != nil || got != nil { + t.Errorf("maintainers = %+v, %v; want none without a packager", got, err) + } +} + +func TestFetchPackage(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + pkg, err := r.FetchPackage(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + // The newest build has no summary of its own in this fixture. + if pkg.Name != "curl" || pkg.Namespace != "fedora" || pkg.LatestVersion != "8.15.0-10.fc43" { + t.Errorf("package = %+v", pkg) + } + pkg, err = New(base, core.DefaultClient()).FetchPackage(context.Background(), "bind") + if err != nil || pkg.LatestVersion != "9.18.41-1.fc43" || pkg.Metadata["epoch"] != "32" { + t.Errorf("bind = %+v, %v", pkg, err) + } +} + +func TestFetchPackageDetails(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + pkg, err := New(base, core.DefaultClient()).FetchPackage(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Description != "A utility for getting files from remote servers" || pkg.Homepage != "https://curl.se/" || pkg.Licenses != "curl" { + t.Errorf("package = %+v", pkg) + } +} + +// Dependencies are the requirements that name packages: file paths, +// sonames and other capabilities are left out. +func TestFetchDependencies(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + deps, err := New(base, core.DefaultClient()).FetchDependencies(context.Background(), "curl", "8.15.0-4.fc43") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "libcurl", Requirements: ">= 8.15.0-4.fc43", Scope: core.Runtime}, + {Name: "openssl-libs", Requirements: "= 1:3.5.4", Scope: core.Runtime}, + {Name: "zlib", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +func TestRequirement(t *testing.T) { + for _, tt := range []struct { + flags, epoch, ver, rel, want string + }{ + {"EQ", "0", "1.0", "1", "= 1.0-1"}, + {"GE", "", "1.0", "", ">= 1.0"}, + {"LE", "2", "1.0", "1", "<= 2:1.0-1"}, + {"GT", "0", "1.0", "", "> 1.0"}, + {"LT", "0", "1.0", "", "< 1.0"}, + {"", "0", "", "", ""}, + {"EQ", "0", "", "", ""}, + {"XX", "0", "1.0", "", ""}, + } { + if got := requirement(tt.flags, tt.epoch, tt.ver, tt.rel); got != tt.want { + t.Errorf("requirement(%q, %q, %q, %q) = %q, want %q", tt.flags, tt.epoch, tt.ver, tt.rel, got, tt.want) + } + } +} + +func TestFetchMaintainers(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + got, err := New(base, core.DefaultClient()).FetchMaintainers(context.Background(), "curl") + if err != nil || !reflect.DeepEqual(got, []core.Maintainer{{Name: "Fedora Project"}}) { + t.Errorf("maintainers = %+v, %v", got, err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os/?distro=fedora-43", nil).URLs() + if got := u.Download("curl", "8.15.0-4.fc43"); got != "" { + t.Errorf("Download = %q, want none without the metadata", got) + } + // repository_url is the whole configured URL, distro included. + const repo = "?repository_url=https:%2F%2Fdl.fedoraproject.org%2Fpub%2Ffedora%2Flinux%2Freleases%2F43%2FEverything%2Fx86_64%2Fos" + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo+"%3Fdistro%3Dfedora-43"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } + u = New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", nil).URLs() + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo; got != want { + t.Errorf("PURL without a distro = %q, want %q", got, want) + } + if got, want := New("", nil).URLs().PURL("fedora/curl", "1-1"), "pkg:rpm/fedora/curl@1-1"; got != want { + t.Errorf("PURL without a repository = %q, want %q", got, want) + } +} + +// A PURL the registry builds leads back to the same repository and distro. +func TestPURLRoundTrip(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + reg, name, version, err := core.NewFromPURL(r.URLs().PURL("fedora/curl", "8.15.0-4.fc43"), core.DefaultClient()) + if err != nil || name != "fedora/curl" || version != "8.15.0-4.fc43" { + t.Fatalf("NewFromPURL = %q, %q, %v", name, version, err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 2 || versions[1].Artifacts[0].Metadata["distro"] != "fedora-43" { + t.Errorf("FetchVersions = %+v, %v", versions, err) + } +} + +// The metadata is read once per Registry. +func TestMetadataReadOnce(t *testing.T) { + rp, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + for range 3 { + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + } + if len(rp.requests) != 2 { + t.Errorf("requests = %v, want repomd.xml and primary once each", rp.requests) + } +} + +func TestSafeHref(t *testing.T) { + for in, ok := range map[string]bool{ + "Packages/c/curl-8.15.0-4.fc43.x86_64.rpm": true, + "": false, + "/etc/passwd": false, + "Packages/../../etc/passwd": false, + "Packages/x.rpm?query": false, + "Packages/x.rpm#frag": false, + `Packages\x.rpm`: false, + "Packages/%2e%2e/x.rpm": false, + "http://evil.example/x.rpm": false, + } { + if got := safeHref(in); got != ok { + t.Errorf("safeHref(%q) = %v, want %v", in, got, ok) + } + } +} diff --git a/internal/xzlimit/testdata/bookworm-updates-Packages.xz b/internal/xzlimit/testdata/bookworm-updates-Packages.xz new file mode 100644 index 0000000..2498e00 Binary files /dev/null and b/internal/xzlimit/testdata/bookworm-updates-Packages.xz differ diff --git a/internal/xzlimit/xzlimit.go b/internal/xzlimit/xzlimit.go new file mode 100644 index 0000000..33c7bed --- /dev/null +++ b/internal/xzlimit/xzlimit.go @@ -0,0 +1,459 @@ +// Package xzlimit decodes .xz files with a bound on the memory a file can +// make the decoder allocate. +// +// An .xz block header names the LZMA2 dictionary its data needs, up to +// 4 GiB, and github.com/ulikunitz/xz allocates what it names: its DictCap +// is a minimum, not a limit (ulikunitz/xz#84). The header cannot be checked +// ahead of decoding either, since where a block's data ends is only known +// by decoding it. So this package reads the .xz container itself, refuses +// a block whose dictionary is over the limit as it reaches it, and hands +// the block's data to ulikunitz/xz's LZMA2 decoder with exactly the +// dictionary the block asked for. +package xzlimit + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash" + "hash/crc32" + "hash/crc64" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// ErrMalformed is returned for a file that is not a well-formed .xz file +// this package can read, or whose blocks ask for too large a dictionary. +var ErrMalformed = errors.New("malformed xz file") + +// MaxDict is the dictionary size callers allow by default. xz -9 uses +// 64 MiB; Debian's indexes use 8 MiB. +const MaxDict = 64 << 20 + +const ( + headerSize = 12 + footerSize = 12 + lzma2Filter = 0x21 + maxVarint = 9 + + checkNone = 0x00 + checkCRC32 = 0x01 + checkCRC64 = 0x04 + checkSHA256 = 0x0A +) + +var ( + headerMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} + footerMagic = []byte{'Y', 'Z'} + crc64Table = crc64.MakeTable(crc64.ECMA) +) + +// Reader decodes an .xz file: one or more streams, each of blocks whose +// only filter is LZMA2, as xz writes them by default. +type Reader struct { + src *source + maxDict uint64 + err error + + streams int + inStream bool + checkType byte + flags [2]byte + records []record // the blocks of the current stream + + // The block being decoded. + block io.Reader + check hash.Hash + blockStart int64 // offset of the block header + dataStart int64 // offset of the block's data + declared blockSizes + produced int64 +} + +// blockSizes are the sizes a block header declares for its data, or -1 +// for one it leaves out. +type blockSizes struct { + compressed int64 + uncompressed int64 +} + +// record is what the index lists for a block. +type record struct { + unpadded int64 + uncompressed int64 +} + +// NewReader returns a Reader that refuses any block asking for a +// dictionary larger than maxDict bytes. +func NewReader(r io.Reader, maxDict uint64) *Reader { + return &Reader{src: &source{r: bufio.NewReader(r)}, maxDict: maxDict} +} + +// source counts the bytes read, which gives the size of each block. It is +// an io.ByteReader so the LZMA2 decoder reads no further than it decodes. +type source struct { + r *bufio.Reader + n int64 +} + +func (s *source) Read(p []byte) (int, error) { + n, err := s.r.Read(p) + s.n += int64(n) + return n, err +} + +func (s *source) ReadByte() (byte, error) { + b, err := s.r.ReadByte() + if err == nil { + s.n++ + } + return b, err +} + +// full reads exactly n bytes; the file ending inside them is malformed. +func (s *source) full(n int) ([]byte, error) { + buf := make([]byte, n) + if _, err := io.ReadFull(s, buf); err != nil { + return nil, early(err) + } + return buf, nil +} + +// early turns the end of the file, where more was expected, into an error. +func early(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + return err +} + +func (r *Reader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + for r.err == nil { + if r.block == nil { + r.err = r.next() + continue + } + n, err := r.block.Read(p) + if n > 0 { + _, _ = r.check.Write(p[:n]) + r.produced += int64(n) + return n, nil + } + if err == io.EOF { + err = r.finishBlock() + } + r.err = err + } + if r.err == io.EOF { + return 0, io.EOF + } + // Both are kept: a caller may look for the source's error, such as a + // size limit, as well as for a malformed file. + if !errors.Is(r.err, ErrMalformed) { + r.err = fmt.Errorf("%w: %w", ErrMalformed, r.err) + } + return 0, r.err +} + +// next reads up to the next block's data: a stream header, a block header, +// or the index and footer that end a stream. It returns io.EOF at the end +// of the file. +func (r *Reader) next() error { + if !r.inStream { + return r.startStream() + } + first, err := r.src.ReadByte() + if err != nil { + return early(err) + } + if first == 0 { + return r.endStream() + } + return r.startBlock(first) +} + +// startStream skips stream padding and reads a stream header. +func (r *Reader) startStream() error { + for { + peek, err := r.src.r.Peek(1) + if err == io.EOF && r.streams > 0 { + return io.EOF + } + if err != nil { + return early(err) + } + if peek[0] != 0 || r.streams == 0 { + break + } + // Stream padding comes in multiples of four zero bytes. + padding, err := r.src.full(4) + if err != nil { + return err + } + if !bytes.Equal(padding, []byte{0, 0, 0, 0}) { + return errors.New("bad stream padding") + } + } + header, err := r.src.full(headerSize) + if err != nil { + return err + } + if !bytes.Equal(header[:6], headerMagic) { + return errors.New("no stream header") + } + if crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:]) { + return errors.New("stream header checksum") + } + if header[6] != 0 { + return errors.New("unsupported stream flags") + } + switch header[7] { + case checkNone, checkCRC32, checkCRC64, checkSHA256: + default: + return fmt.Errorf("unsupported check type %#x", header[7]) + } + r.flags = [2]byte{header[6], header[7]} + r.checkType = header[7] + r.records = r.records[:0] + r.inStream = true + r.streams++ + return nil +} + +// startBlock reads the block header whose first byte is sizeByte and +// starts decoding the block's data. +func (r *Reader) startBlock(sizeByte byte) error { + r.blockStart = r.src.n - 1 + size := (int(sizeByte) + 1) * 4 + rest, err := r.src.full(size - 1) + if err != nil { + return err + } + header := append([]byte{sizeByte}, rest...) + if crc32.ChecksumIEEE(header[:size-4]) != binary.LittleEndian.Uint32(header[size-4:]) { + return errors.New("block header checksum") + } + dict, declared, err := blockFields(header[1 : size-4]) + if err != nil { + return err + } + if dict > r.maxDict { + return fmt.Errorf("block asks for a dictionary of %d bytes, more than %d", dict, r.maxDict) + } + // Exactly what the block asked for, never more. The smallest size the + // format can name, 4096, is also the decoder's minimum. + block, err := lzma.Reader2Config{DictCap: int(dict)}.NewReader2(r.src) + if err != nil { + return err + } + r.block = block + r.check = newCheck(r.checkType) + r.dataStart = r.blockStart + int64(size) + r.declared = declared + r.produced = 0 + return nil +} + +// blockFields reads a block header's fields, without its size byte and +// checksum, and returns the dictionary size of its LZMA2 filter and the +// sizes it declares for the block's data. +func blockFields(fields []byte) (uint64, blockSizes, error) { + sizes := blockSizes{compressed: -1, uncompressed: -1} + flags := fields[0] + if flags&0x3C != 0 { + return 0, sizes, errors.New("reserved block flags") + } + if flags&0x03 != 0 { + return 0, sizes, errors.New("unsupported filter chain: more than one filter") + } + pos := 1 + for _, field := range []struct { + present byte + size *int64 + }{{0x40, &sizes.compressed}, {0x80, &sizes.uncompressed}} { + if flags&field.present == 0 { + continue + } + v, n := uvarint(fields[pos:]) + if n == 0 { + return 0, sizes, errors.New("bad block header") + } + *field.size = int64(v) + pos += n + } + // One filter: LZMA2, with one byte of properties. + if len(fields)-pos < 3 || fields[pos] != lzma2Filter || fields[pos+1] != 1 { + return 0, sizes, errors.New("unsupported filter chain: not LZMA2") + } + bits := fields[pos+2] + if bits > 40 { + return 0, sizes, errors.New("bad LZMA2 properties") + } + for _, b := range fields[pos+3:] { + if b != 0 { + return 0, sizes, errors.New("bad block header padding") + } + } + return dictSize(bits), sizes, nil +} + +// dictSize decodes the LZMA2 dictionary size property, 0 to 40. +func dictSize(bits byte) uint64 { + if bits == 40 { + return 1<<32 - 1 + } + return uint64(2|bits&1) << (bits/2 + 11) +} + +func newCheck(checkType byte) hash.Hash { + switch checkType { + case checkCRC32: + return crc32.NewIEEE() + case checkCRC64: + return crc64.New(crc64Table) + case checkSHA256: + return sha256.New() + } + return noCheck{} +} + +// noCheck is the check of a stream that has none. +type noCheck struct{ hash.Hash } + +func (noCheck) Write(p []byte) (int, error) { return len(p), nil } +func (noCheck) Sum([]byte) []byte { return nil } + +// finishBlock reads what follows a block's data: its padding and check. +func (r *Reader) finishBlock() error { + if d := r.declared.compressed; d >= 0 && d != r.src.n-r.dataStart { + return fmt.Errorf("block header declares %d bytes of data, the block has %d", d, r.src.n-r.dataStart) + } + if d := r.declared.uncompressed; d >= 0 && d != r.produced { + return fmt.Errorf("block header declares %d bytes uncompressed, the block has %d", d, r.produced) + } + unpadded := r.src.n - r.blockStart + if padding := int((4 - unpadded%4) % 4); padding > 0 { + zeros, err := r.src.full(padding) + if err != nil { + return err + } + if !bytes.Equal(zeros, make([]byte, padding)) { + return errors.New("bad block padding") + } + } + want := r.check.Sum(nil) + // CRCs are stored little-endian, and hash.Hash gives them big-endian. + if r.checkType == checkCRC32 || r.checkType == checkCRC64 { + for i, j := 0, len(want)-1; i < j; i, j = i+1, j-1 { + want[i], want[j] = want[j], want[i] + } + } + got, err := r.src.full(len(want)) + if err != nil { + return err + } + if !bytes.Equal(got, want) { + return errors.New("block check mismatch") + } + r.records = append(r.records, record{unpadded: unpadded + int64(len(want)), uncompressed: r.produced}) + r.block = nil + return nil +} + +// endStream reads the index, whose indicator byte has been read, and the +// stream footer, and checks them against the blocks that were decoded. +func (r *Reader) endStream() error { + indexStart := r.src.n - 1 + crc := crc32.NewIEEE() + _, _ = crc.Write([]byte{0}) + in := io.TeeReader(r.src, crc) + readUvarint := func() (uint64, error) { + var buf [maxVarint]byte + for i := range buf { + if _, err := io.ReadFull(in, buf[i:i+1]); err != nil { + return 0, early(err) + } + if buf[i]&0x80 == 0 { + v, _ := uvarint(buf[:i+1]) + return v, nil + } + } + return 0, errors.New("bad index") + } + + count, err := readUvarint() + if err != nil { + return err + } + if count != uint64(len(r.records)) { + return fmt.Errorf("index lists %d blocks, the stream has %d", count, len(r.records)) + } + for _, rec := range r.records { + unpadded, err := readUvarint() + if err != nil { + return err + } + uncompressed, err := readUvarint() + if err != nil { + return err + } + if unpadded != uint64(rec.unpadded) || uncompressed != uint64(rec.uncompressed) { + return errors.New("index does not match the blocks") + } + } + if padding := int((4 - (r.src.n-indexStart)%4) % 4); padding > 0 { + zeros := make([]byte, padding) + if _, err := io.ReadFull(in, zeros); err != nil { + return early(err) + } + if !bytes.Equal(zeros, make([]byte, padding)) { + return errors.New("bad index padding") + } + } + sum := crc.Sum32() + stored, err := r.src.full(4) + if err != nil { + return err + } + if binary.LittleEndian.Uint32(stored) != sum { + return errors.New("index checksum") + } + indexSize := r.src.n - indexStart + + footer, err := r.src.full(footerSize) + if err != nil { + return err + } + switch { + case !bytes.Equal(footer[10:], footerMagic): + return errors.New("no stream footer") + case crc32.ChecksumIEEE(footer[4:10]) != binary.LittleEndian.Uint32(footer): + return errors.New("stream footer checksum") + case int64(binary.LittleEndian.Uint32(footer[4:8])+1)*4 != indexSize: + return errors.New("stream footer does not match the index") + case footer[8] != r.flags[0] || footer[9] != r.flags[1]: + return errors.New("stream footer does not match the header") + } + r.inStream = false + return nil +} + +// uvarint decodes an xz multibyte integer, returning 0 bytes read when it +// is malformed or truncated. +func uvarint(b []byte) (uint64, int) { + var v uint64 + for i := 0; i < len(b) && i < maxVarint; i++ { + v |= uint64(b[i]&0x7F) << (7 * i) + if b[i]&0x80 == 0 { + return v, i + 1 + } + } + return 0, 0 +} diff --git a/internal/xzlimit/xzlimit_test.go b/internal/xzlimit/xzlimit_test.go new file mode 100644 index 0000000..47818d3 --- /dev/null +++ b/internal/xzlimit/xzlimit_test.go @@ -0,0 +1,513 @@ +package xzlimit + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "io" + "os" + "strings" + "testing" + + "github.com/ulikunitz/xz" +) + +const sample = "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n\n" + +func compress(t *testing.T, config xz.WriterConfig, parts ...[]byte) []byte { + t.Helper() + var buf bytes.Buffer + w, err := config.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + for _, part := range parts { + if _, err := w.Write(part); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// noise is bytes that don't compress, which LZMA2 stores in stored chunks. +func noise(n int) []byte { + out := make([]byte, n) + state := uint32(1) + for i := range out { + state = state*1664525 + 1013904223 + out[i] = byte(state >> 24) + } + return out +} + +func decode(data []byte, maxDict uint64) ([]byte, error) { + return io.ReadAll(NewReader(bytes.NewReader(data), maxDict)) +} + +// reference decodes with the library's own reader. +func reference(t *testing.T, data []byte) []byte { + t.Helper() + r, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + out, err := io.ReadAll(r) + if err != nil { + t.Fatalf("the reference reader fails: %v", err) + } + return out +} + +func TestDecodesWhatTheLibraryDecodes(t *testing.T) { + debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") + if err != nil { + t.Fatal(err) + } + text := []byte(strings.Repeat(sample, 5000)) + mixed := [][]byte{text, noise(200_000), []byte(strings.Repeat("x", 300_000))} + for name, data := range map[string][]byte{ + "debian's own index": debian, + "one small block": compress(t, xz.WriterConfig{}, []byte(sample)), + "empty": compress(t, xz.WriterConfig{}), + "several blocks": compress(t, xz.WriterConfig{BlockSize: 64 << 10}, mixed...), + "crc32 check": compress(t, xz.WriterConfig{CheckSum: xz.CRC32, BlockSize: 64 << 10}, mixed...), + "sha256 check": compress(t, xz.WriterConfig{CheckSum: xz.SHA256, BlockSize: 64 << 10}, mixed...), + "no check": compress(t, xz.WriterConfig{NoCheckSum: true, BlockSize: 64 << 10}, mixed...), + "small dictionary": compress(t, xz.WriterConfig{DictCap: 4096}, text), + "two streams": append(compress(t, xz.WriterConfig{}, text), compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample))...), + "two padded streams": bytes.Join([][]byte{compress(t, xz.WriterConfig{}, text), make([]byte, 8), compress(t, xz.WriterConfig{}, []byte(sample)), make([]byte, 4)}, nil), + } { + t.Run(name, func(t *testing.T) { + want := reference(t, data) + got, err := decode(data, MaxDict) + if err != nil { + t.Fatalf("decode: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("decoded %d bytes, the library decodes %d; they differ", len(got), len(want)) + } + }) + } +} + +// TestReadsInSmallPieces checks nothing depends on how much a Read asks for. +func TestReadsInSmallPieces(t *testing.T) { + data := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 500)), noise(9000)) + want := reference(t, data) + r := NewReader(iotestOneByte{bytes.NewReader(data)}, MaxDict) + var got []byte + buf := make([]byte, 7) + for { + n, err := r.Read(buf) + got = append(got, buf[:n]...) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Read: %v", err) + } + } + if !bytes.Equal(got, want) { + t.Errorf("decoded %d bytes, want %d", len(got), len(want)) + } + if n, err := r.Read(nil); n != 0 || err != nil { + t.Errorf("Read(nil) = %d, %v", n, err) + } +} + +// iotestOneByte hands out one byte per Read. +type iotestOneByte struct{ r io.Reader } + +func (o iotestOneByte) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + return o.r.Read(p[:1]) +} + +// setDict rewrites the dictionary size of the block header at offset in +// data, fixing the header's checksum, so a small block claims a large +// dictionary. +func setDict(t *testing.T, data []byte, offset int, bits byte) []byte { + t.Helper() + out := bytes.Clone(data) + size := (int(out[offset]) + 1) * 4 + header := out[offset : offset+size] + i := bytes.Index(header, []byte{lzma2Filter, 0x01}) + if i < 0 { + t.Fatal("no LZMA2 filter in the block header") + } + header[i+2] = bits + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} + +// blockOffsets returns where each block header of a one-stream file starts, +// from the sizes its index lists. +func blockOffsets(t *testing.T, data []byte) []int { + t.Helper() + r := NewReader(bytes.NewReader(data), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil { + t.Fatal(err) + } + var offsets []int + offset := headerSize + for _, rec := range r.records { + offsets = append(offsets, offset) + offset += int((rec.unpadded + 3) &^ 3) + } + return offsets +} + +func TestDictionaryLimit(t *testing.T) { + small := compress(t, xz.WriterConfig{}, []byte(sample)) + // 128 MiB, past the limit. + if _, err := decode(setDict(t, small, headerSize, 30), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "dictionary") { + t.Errorf("128 MiB dictionary: err = %v, want ErrMalformed naming the dictionary", err) + } + // The limit itself is allowed: 64 MiB is property 28. + if dictSize(28) != MaxDict { + t.Fatalf("dictSize(28) = %d", dictSize(28)) + } + if _, err := decode(small, 8<<20); err != nil { + t.Errorf("a block at the limit: %v", err) + } + if _, err := decode(small, 8<<20-1); !errors.Is(err, ErrMalformed) { + t.Errorf("a block one byte over the limit: err = %v, want ErrMalformed", err) + } + + // A later block is held to the limit when the reader gets to it, after + // the blocks before it were decoded. + blocks := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 500))) + offsets := blockOffsets(t, blocks) + if len(offsets) < 3 { + t.Fatalf("the fixture has %d blocks, want several", len(offsets)) + } + got, err := decode(setDict(t, blocks, offsets[2], 38), MaxDict) + if !errors.Is(err, ErrMalformed) { + t.Errorf("large third block: err = %v, want ErrMalformed", err) + } + if len(got) != 2*(4<<10) { + t.Errorf("decoded %d bytes before the refused block, want the two blocks before it", len(got)) + } + // And one in a second stream. + two := bytes.Join([][]byte{small, make([]byte, 4), setDict(t, small, headerSize, 38)}, nil) + if got, err := decode(two, MaxDict); !errors.Is(err, ErrMalformed) || string(got) != sample { + t.Errorf("large block in a second stream: %q, %v", got, err) + } +} + +func TestDictSize(t *testing.T) { + for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { + if got := dictSize(bits); got != want { + t.Errorf("dictSize(%d) = %d, want %d", bits, got, want) + } + } +} + +// fixCRC recomputes the CRC32 that ends a header of size bytes at offset. +func fixCRC(data []byte, offset, size int) { + binary.LittleEndian.PutUint32(data[offset+size-4:], crc32.ChecksumIEEE(data[offset:offset+size-4])) +} + +func TestMalformedFiles(t *testing.T) { + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(strings.Repeat(sample, 20))) + blockHeaderSize := (int(valid[headerSize]) + 1) * 4 + patch := func(at int, b ...byte) []byte { + out := bytes.Clone(valid) + if at < 0 { + at += len(out) + } + copy(out[at:], b) + return out + } + // The block header: size, flags, filter id, properties size, properties. + header := func(edit func(h []byte)) []byte { + out := bytes.Clone(valid) + edit(out[headerSize : headerSize+blockHeaderSize]) + fixCRC(out, headerSize, blockHeaderSize) + return out + } + streamFlags := func(flag0, checkType byte) []byte { + out := patch(6, flag0, checkType) + binary.LittleEndian.PutUint32(out[8:], crc32.ChecksumIEEE(out[6:8])) + copy(out[len(out)-4:], []byte{flag0, checkType}) + binary.LittleEndian.PutUint32(out[len(out)-12:], crc32.ChecksumIEEE(out[len(out)-8:len(out)-2])) + return out + } + + for name, data := range map[string][]byte{ + "empty": nil, + "not xz": []byte(strings.Repeat("plain text ", 10)), + "header magic": patch(0, 0xFE), + "header checksum": patch(8, valid[8]^1), + "stream flags": streamFlags(1, checkCRC32), + "unsupported check": streamFlags(0, 0x02), + "block header checksum": patch(headerSize+blockHeaderSize-1, valid[headerSize+blockHeaderSize-1]^1), + "reserved block flags": header(func(h []byte) { h[1] = 0x04 }), + "two filters": header(func(h []byte) { h[1] = 0x01 }), + "filter not lzma2": header(func(h []byte) { h[2] = 0x03 }), + "properties size": header(func(h []byte) { h[3] = 0x02 }), + // 200 would shift the size to nothing and read as a tiny dictionary. + "properties past 40": header(func(h []byte) { h[4] = 200 }), + "block header padding": header(func(h []byte) { h[5] = 1 }), + "corrupt data": patch(headerSize+blockHeaderSize+20, valid[headerSize+blockHeaderSize+20]^0xFF), + "block check": patch(-12-8-4, valid[len(valid)-12-8-4]^1), + "index checksum": patch(-12-1, valid[len(valid)-12-1]^1), + "footer magic": patch(-2, 'Y', 'X'), + "footer checksum": patch(-12, valid[len(valid)-12]^1), + "garbage after the stream": append(bytes.Clone(valid), "garbage!"...), + "padding of two bytes": append(bytes.Clone(valid), 0, 0), + "padding with a stray byte": bytes.Join([][]byte{valid, {0, 1, 0, 0}, valid}, nil), + "padding before the stream": append(make([]byte, 4), valid...), + } { + t.Run(name, func(t *testing.T) { + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("err = %v, want ErrMalformed", err) + } + }) + } +} + +// The index and footer must describe the blocks that were decoded. +func TestIndexAndFooterAreChecked(t *testing.T) { + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(strings.Repeat(sample, 20))) + // A one-block index is 8 bytes when both sizes fit a byte or two: + // find it from the footer's backward size. + indexSize := (int(binary.LittleEndian.Uint32(valid[len(valid)-8:])) + 1) * 4 + indexStart := len(valid) - footerSize - indexSize + reindex := func(edit func(index []byte)) []byte { + out := bytes.Clone(valid) + index := out[indexStart : indexStart+indexSize] + edit(index) + binary.LittleEndian.PutUint32(index[indexSize-4:], crc32.ChecksumIEEE(index[:indexSize-4])) + return out + } + refooter := func(edit func(footer []byte)) []byte { + out := bytes.Clone(valid) + footer := out[len(out)-footerSize:] + edit(footer) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return out + } + for name, data := range map[string][]byte{ + "record count": reindex(func(index []byte) { index[1] = 2 }), + "no records": reindex(func(index []byte) { index[1] = 0 }), + "unpadded size": reindex(func(index []byte) { index[2]++ }), + "uncompressed size": reindex(func(index []byte) { index[indexSize-5-1]++ }), + "backward size": refooter(func(footer []byte) { footer[4]++ }), + "footer flags": refooter(func(footer []byte) { footer[9] = checkCRC64 }), + "footer reserved flags": refooter(func(footer []byte) { footer[8] = 1 }), + } { + t.Run(name, func(t *testing.T) { + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("err = %v, want ErrMalformed", err) + } + }) + } +} + +// Every truncation of a file is an error, never a short result. +func TestTruncatedFiles(t *testing.T) { + debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") + if err != nil { + t.Fatal(err) + } + blocks := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 300)), noise(5000)) + for name, data := range map[string][]byte{"debian": debian, "several blocks": blocks} { + step := 1 + if len(data) > 4000 { + step = 7 + } + for n := 0; n < len(data); n += step { + if _, err := decode(data[:n], MaxDict); !errors.Is(err, ErrMalformed) { + t.Fatalf("%s cut to %d of %d bytes: err = %v, want ErrMalformed", name, n, len(data), err) + } + } + } +} + +// An error from the source is still found in the error returned. +func TestSourceErrorIsKept(t *testing.T) { + valid := compress(t, xz.WriterConfig{}, noise(50_000)) + boom := errors.New("response too large") + for _, cut := range []int{0, 5, headerSize, headerSize + 3, 2000, len(valid) - 20, len(valid) - 3} { + r := NewReader(io.MultiReader(bytes.NewReader(valid[:cut]), failing{boom}), MaxDict) + if _, err := io.Copy(io.Discard, r); !errors.Is(err, boom) { + t.Errorf("source failing after %d bytes: err = %v, want it to wrap the source's error", cut, err) + } + } +} + +type failing struct{ err error } + +func (f failing) Read([]byte) (int, error) { return 0, f.err } + +func TestUnsupportedCheckIsNamed(t *testing.T) { + data := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample)) + data[7] = 0x02 + binary.LittleEndian.PutUint32(data[8:], crc32.ChecksumIEEE(data[6:8])) + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "unsupported check type") { + t.Errorf("err = %v, want ErrMalformed naming the check type", err) + } +} + +func TestUvarint(t *testing.T) { + for _, tt := range []struct { + in []byte + want uint64 + n int + }{ + {[]byte{0x00}, 0, 1}, + {[]byte{0x7F, 0xFF}, 127, 1}, + {[]byte{0x80, 0x01}, 128, 2}, + {[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}, 1<<63 - 1, 9}, + {[]byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00}, 0, 0}, // ten bytes + {[]byte{0x80}, 0, 0}, + {nil, 0, 0}, + } { + if got, n := uvarint(tt.in); got != tt.want || n != tt.n { + t.Errorf("uvarint(%x) = %d, %d; want %d, %d", tt.in, got, n, tt.want, tt.n) + } + } +} + +// An index whose record count is an overlong integer is refused, here in a +// stream with no blocks, where the count would otherwise read as zero. +func TestOverlongIndexCount(t *testing.T) { + some := compress(t, xz.WriterConfig{}, []byte(sample)) + // A stream with no blocks: the header, an index and a footer. + stream := func(index []byte) []byte { + for len(index)%4 != 0 { + index = append(index, 0) + } + index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index)) + footer := bytes.Clone(some[len(some)-footerSize:]) + binary.LittleEndian.PutUint32(footer[4:], uint32(len(index)/4-1)) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return bytes.Join([][]byte{some[:headerSize], index, footer}, nil) + } + if got, err := decode(stream([]byte{0x00, 0x00}), MaxDict); err != nil || len(got) != 0 { + t.Fatalf("a stream with no blocks: %q, %v", got, err) + } + overlong := append([]byte{0x00}, bytes.Repeat([]byte{0x80}, 9)...) + if _, err := decode(stream(overlong), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "bad index") { + t.Errorf("err = %v, want ErrMalformed for the index", err) + } +} + +// withBlockSizes rebuilds a one-block CRC32 file so its block header +// declares these sizes (nil leaves one out), with the index and footer +// rewritten for the longer header. +func withBlockSizes(t *testing.T, valid []byte, compressed, uncompressed *uint64) []byte { + t.Helper() + r := NewReader(bytes.NewReader(valid), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil || len(r.records) != 1 { + t.Fatalf("the fixture is not a one-block file: %v", err) + } + const checkSize = 4 + oldHeaderSize := (int(valid[headerSize]) + 1) * 4 + dataSize := int(r.records[0].unpadded) - oldHeaderSize - checkSize + data := valid[headerSize+oldHeaderSize : headerSize+oldHeaderSize+dataSize] + blockEnd := headerSize + int((r.records[0].unpadded+3)&^3) + check := valid[blockEnd-checkSize : blockEnd] + + appendUvarint := func(b []byte, v uint64) []byte { + for v >= 0x80 { + b = append(b, byte(v)|0x80) + v >>= 7 + } + return append(b, byte(v)) + } + header := []byte{0, 0} + if compressed != nil { + header[1] |= 0x40 + header = appendUvarint(header, *compressed) + } + if uncompressed != nil { + header[1] |= 0x80 + header = appendUvarint(header, *uncompressed) + } + header = append(header, lzma2Filter, 0x01, valid[headerSize+4]) + for len(header)%4 != 0 { + header = append(header, 0) + } + header[0] = byte(len(header)/4 + 1 - 1) + header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header)) + + block := bytes.Join([][]byte{header, data}, nil) + unpadded := len(block) + checkSize + for len(block)%4 != 0 { + block = append(block, 0) + } + block = append(block, check...) + + index := appendUvarint([]byte{0x00, 0x01}, uint64(unpadded)) + index = appendUvarint(index, uint64(r.records[0].uncompressed)) + for len(index)%4 != 0 { + index = append(index, 0) + } + index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index)) + footer := bytes.Clone(valid[len(valid)-footerSize:]) + binary.LittleEndian.PutUint32(footer[4:], uint32(len(index)/4-1)) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return bytes.Join([][]byte{valid[:headerSize], block, index, footer}, nil) +} + +// A block header may declare the sizes of its data; they must be the sizes +// the block turns out to have. +func TestDeclaredBlockSizes(t *testing.T) { + text := strings.Repeat(sample, 20) + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(text)) + r := NewReader(bytes.NewReader(valid), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil { + t.Fatal(err) + } + blockHeaderSize := (int(valid[headerSize]) + 1) * 4 + compressed := uint64(r.records[0].unpadded) - uint64(blockHeaderSize) - 4 + uncompressed := uint64(len(text)) + ptr := func(v uint64) *uint64 { return &v } + + for name, sizes := range map[string][2]*uint64{ + "both": {ptr(compressed), ptr(uncompressed)}, + "compressed only": {ptr(compressed), nil}, + "uncompressed only": {nil, ptr(uncompressed)}, + } { + data := withBlockSizes(t, valid, sizes[0], sizes[1]) + if got := reference(t, data); string(got) != text { + t.Fatalf("%s: the fixture does not decode with the library", name) + } + if got, err := decode(data, MaxDict); err != nil || string(got) != text { + t.Errorf("%s: decoded %d bytes, %v", name, len(got), err) + } + } + for name, sizes := range map[string][2]*uint64{ + "compressed too small": {ptr(compressed - 1), ptr(uncompressed)}, + "compressed too large": {ptr(compressed + 1), nil}, + "uncompressed zero": {nil, ptr(0)}, + "uncompressed too large": {ptr(compressed), ptr(uncompressed + 1)}, + } { + if _, err := decode(withBlockSizes(t, valid, sizes[0], sizes[1]), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "block header") { + t.Errorf("%s: err = %v, want ErrMalformed for the block header's size", name, err) + } + } +} + +// A declared size that never ends is named as the header's fault. +func TestUnterminatedBlockSize(t *testing.T) { + data := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample)) + size := (int(data[headerSize]) + 1) * 4 + header := data[headerSize : headerSize+size] + header[1] = 0x40 + for i := 2; i < size-4; i++ { + header[i] = 0x80 + } + fixCRC(data, headerSize, size) + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "bad block header") { + t.Errorf("err = %v, want ErrMalformed for the block header", err) + } +} diff --git a/registries_test.go b/registries_test.go index 81027fa..7f1c8b0 100644 --- a/registries_test.go +++ b/registries_test.go @@ -20,7 +20,7 @@ import ( func TestSupportedEcosystems(t *testing.T) { ecosystems := registries.SupportedEcosystems() - expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} + expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform"} sort.Strings(ecosystems) if len(ecosystems) != len(expected) { @@ -69,6 +69,8 @@ func TestNew(t *testing.T) { {"swift", true}, {"apm", true}, {"apk", true}, + {"deb", true}, + {"rpm", true}, {"unknown", true}, } @@ -117,6 +119,8 @@ func TestDefaultURL(t *testing.T) { {"swift", ""}, {"apm", ""}, {"apk", ""}, + {"deb", ""}, + {"rpm", ""}, } for _, tt := range tests {