From 5ca248c69ea99f2dc93792f12488773af57f750e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 15:39:38 +0100 Subject: [PATCH 1/7] Add a Debian repository client Read the Packages indexes of the suites, components and architectures given as query parameters on the archive URL. Each index is checked against the SHA256 its suite's InRelease or Release lists, tried as Packages.xz, Packages.gz and Packages past a 404, and decoded as a stream. The LZMA2 dictionary of every .xz block is checked before decoding, since the decoder allocates whatever a block asks for. Artifacts record arch and distro (the suite's codename without -updates or -security) for qualifier selection. Versions keep their epoch and compare with the deb scheme. Add Client.Stream for large bodies that aren't JSON. --- README.md | 4 + all/all.go | 3 +- client/client.go | 14 + client/client_test.go | 44 +++ fetch/deb_resolver_test.go | 62 ++++ go.mod | 1 + go.sum | 2 + internal/deb/deb.go | 539 +++++++++++++++++++++++++++++++ internal/deb/deb_test.go | 645 +++++++++++++++++++++++++++++++++++++ internal/deb/parse.go | 288 +++++++++++++++++ internal/deb/parse_test.go | 295 +++++++++++++++++ internal/deb/xzcheck.go | 193 +++++++++++ registries_test.go | 4 +- 13 files changed, 2092 insertions(+), 2 deletions(-) create mode 100644 fetch/deb_resolver_test.go create mode 100644 internal/deb/deb.go create mode 100644 internal/deb/deb_test.go create mode 100644 internal/deb/parse.go create mode 100644 internal/deb/parse_test.go create mode 100644 internal/deb/xzcheck.go diff --git a/README.md b/README.md index d953bfe..d09e2e8 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | +| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm` | ## Direct Registry Usage @@ -196,11 +197,14 @@ import _ "github.com/git-pkgs/registries/all" | Terraform | `terraform` | https://registry.terraform.io | | Swift | `swift` | URL required | | Alpine | `apk` | URL required | +| Debian | `deb` | URL required | | GitHub Actions | `githubactions` | https://api.github.com | | APM | None | URL required | The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index. +The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. + GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. ### Swift registries diff --git a/all/all.go b/all/all.go index 897535a..aad270b 100644 --- a/all/all.go +++ b/all/all.go @@ -9,7 +9,7 @@ // // // Now all ecosystems are available // ecosystems := registries.SupportedEcosystems() -// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] +// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] package all import ( @@ -21,6 +21,7 @@ import ( _ "github.com/git-pkgs/registries/internal/conda" _ "github.com/git-pkgs/registries/internal/cpan" _ "github.com/git-pkgs/registries/internal/cran" + _ "github.com/git-pkgs/registries/internal/deb" _ "github.com/git-pkgs/registries/internal/deno" _ "github.com/git-pkgs/registries/internal/dub" _ "github.com/git-pkgs/registries/internal/elm" diff --git a/client/client.go b/client/client.go index aedf379..88df2d1 100644 --- a/client/client.go +++ b/client/client.go @@ -121,6 +121,20 @@ func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decod return nil } +// Stream fetches a URL and passes the response body to fn, for large bodies +// that are not JSON, such as compressed indexes. Bodies are limited to +// MaxStreamSize: reading past it fails with ErrResponseTooLarge. Retries +// cover getting a response; an error from fn is returned as is. +func (c *Client) Stream(ctx context.Context, url string, fn func(io.Reader) error) error { + resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + limit := c.maxStreamSize() + return fn(&limitedReader{r: resp.Body, remaining: limit, limit: limit, url: url}) +} + func (c *Client) maxResponseSize() int64 { if c.MaxResponseSize > 0 { return c.MaxResponseSize diff --git a/client/client_test.go b/client/client_test.go index 0c377ea..cec5761 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -293,3 +293,47 @@ func TestHeadRetriesAndRateLimits(t *testing.T) { t.Errorf("rate limiter waits = %d, want 3", limiter.waits) } } + +func TestStream(t *testing.T) { + const doc = "Package: curl\nVersion: 7.88.1-10\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/missing" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(doc)) + })) + defer srv.Close() + + readAll := func(c *Client, url string) (string, error) { + var got []byte + err := c.Stream(context.Background(), url, func(r io.Reader) error { + var err error + got, err = io.ReadAll(r) + return err + }) + return string(got), err + } + + got, err := readAll(NewClient(WithMaxResponseSize(10), WithMaxRetries(0)), srv.URL) + if err != nil || got != doc { + t.Errorf("Stream = %q, %v; the buffered limit should not apply", got, err) + } + if _, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc)-1)), WithMaxRetries(0)), srv.URL); !errors.Is(err, ErrResponseTooLarge) { + t.Errorf("over the stream limit: err = %v, want ErrResponseTooLarge", err) + } + if got, err := readAll(NewClient(WithMaxStreamSize(int64(len(doc))), WithMaxRetries(0)), srv.URL); err != nil || got != doc { + t.Errorf("at the stream limit: %q, %v", got, err) + } + + var httpErr *HTTPError + _, err = readAll(NewClient(WithMaxRetries(0)), srv.URL+"/missing") + if !errors.As(err, &httpErr) || httpErr.StatusCode != http.StatusNotFound { + t.Errorf("missing: err = %v, want a 404 HTTPError", err) + } + + sentinel := errors.New("stop") + if err := NewClient().Stream(context.Background(), srv.URL, func(io.Reader) error { return sentinel }); !errors.Is(err, sentinel) { + t.Errorf("fn error: err = %v, want it returned as is", err) + } +} diff --git a/fetch/deb_resolver_test.go b/fetch/deb_resolver_test.go new file mode 100644 index 0000000..96e50cf --- /dev/null +++ b/fetch/deb_resolver_test.go @@ -0,0 +1,62 @@ +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/deb" + "github.com/git-pkgs/registries/safehttp" +) + +// debArchive serves bookworm amd64 and arm64 plain Packages indexes with no +// Release, over TLS: the resolver only hands out https download URLs. +func debArchive(t *testing.T) (string, *registries.Client) { + t.Helper() + indexes := map[string]string{ + "/debian/dists/bookworm/main/binary-amd64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n" + + "Filename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\nSize: 3\nSHA256: aa\n", + "/debian/dists/bookworm/main/binary-arm64/Packages": "Package: curl\nVersion: 7.88.1-10\nArchitecture: arm64\n" + + "Filename: pool/main/c/curl/curl_7.88.1-10_arm64.deb\nSize: 4\nSHA256: bb\n", + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := indexes[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + client := registries.DefaultClient() + client.HTTPClient = server.Client() + return server.URL + "/debian", client +} + +func TestResolveDebByQualifiers(t *testing.T) { + base, client := debArchive(t) + reg, err := registries.New("deb", base+"?suites=bookworm&architectures=amd64,arm64", client) + if err != nil { + t.Fatal(err) + } + r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true})) + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10", + ResolveOptions{Qualifiers: map[string]string{"arch": "arm64", "distro": "bookworm"}}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if want := base + "/pool/main/c/curl/curl_7.88.1-10_arm64.deb"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 { + t.Errorf("info = %+v, want %s", info, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "deb", "debian/curl", "7.88.1-10", + ResolveOptions{Qualifiers: map[string]string{"arch": "amd64", "distro": "trixie"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/go.mod b/go.mod index 4e0d676..9d83d32 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rubyist/circuitbreaker v2.2.1+incompatible + github.com/ulikunitz/xz v0.5.17 go.yaml.in/yaml/v3 v3.0.6 ) diff --git a/go.sum b/go.sum index 756e1e8..30e87b0 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6 github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/deb/deb.go b/internal/deb/deb.go new file mode 100644 index 0000000..391d26a --- /dev/null +++ b/internal/deb/deb.go @@ -0,0 +1,539 @@ +// Package deb provides a registry client for Debian repositories, read from +// their Packages indexes. +package deb + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/url" + "path" + "sort" + "strings" + "sync" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/vers" + "github.com/ulikunitz/xz" +) + +const ( + DefaultURL = "" + ecosystem = "deb" + + // maxIndexSize bounds one decompressed Packages index. noble universe + // amd64 is about 73 MB. + maxIndexSize = 512 << 20 + // maxXZSize bounds a Packages.xz, which is read whole so its blocks can + // be checked before decoding. sid main amd64 is about 11 MB. + maxXZSize = 128 << 20 + + archAll = "all" +) + +var ( + // ErrInvalidConfig is returned when the repository URL names no + // archive, suites or architectures, or asks for source packages. + ErrInvalidConfig = errors.New("invalid Debian repository configuration") + // ErrNoIndex is returned when no configured suite, component and + // architecture has a Packages index. + ErrNoIndex = errors.New("no Debian Packages index found") + // ErrChecksumMismatch is returned when an index doesn't match the + // SHA256 its suite's Release lists. + ErrChecksumMismatch = errors.New("index does not match its Debian Release checksum") + // ErrMalformedIndex is returned for Release files and indexes that + // cannot be parsed. + ErrMalformedIndex = errors.New("malformed Debian index") +) + +// indexVariants are tried in this order, moving on after a 404. +var indexVariants = []string{"Packages.xz", "Packages.gz", "Packages"} + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads the Packages indexes of one archive. The URL is the +// archive root, with the suites, components and architectures to read as +// query parameters, which archives don't use: +// +// https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64 +// +// components defaults to main. An archive under another root, such as +// deb.debian.org/debian-security, is a Registry of its own. +// +// The indexes are downloaded once per Registry and kept for its lifetime, +// so callers looking up many packages should reuse one Registry. +type Registry struct { + archiveURL string + suites []string + components []string + architectures []string + configErr error + client *core.Client + urls *URLs + + maxIndexSize int64 + maxXZSize int64 + + mu sync.Mutex + packages map[string][]record +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + r := &Registry{client: client, maxIndexSize: maxIndexSize, maxXZSize: maxXZSize} + r.configErr = r.configure(baseURL) + r.urls = &URLs{archiveURL: r.archiveURL} + return r +} + +func (r *Registry) configure(baseURL string) error { + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("%w: %q is not an archive URL", ErrInvalidConfig, baseURL) + } + q := u.Query() + u.RawQuery, u.Fragment = "", "" + r.archiveURL = strings.TrimSuffix(u.String(), "/") + r.suites = splitList(q.Get("suites")) + r.components = splitList(q.Get("components")) + r.architectures = splitList(q.Get("architectures")) + if len(r.components) == 0 { + r.components = []string{"main"} + } + switch { + case len(r.suites) == 0: + return fmt.Errorf("%w: no suites in %q", ErrInvalidConfig, baseURL) + case len(r.architectures) == 0: + return fmt.Errorf("%w: no architectures in %q", ErrInvalidConfig, baseURL) + } + for _, list := range [][]string{r.suites, r.components, r.architectures} { + for _, item := range list { + if !safeSegment(item) { + return fmt.Errorf("%w: %q", ErrInvalidConfig, item) + } + } + } + for _, arch := range r.architectures { + if arch == "source" { + return fmt.Errorf("%w: source packages are not supported", ErrInvalidConfig) + } + } + return nil +} + +func splitList(s string) []string { + var out []string + for _, item := range strings.Split(s, ",") { + if item = strings.TrimSpace(item); item != "" { + out = append(out, item) + } + } + return out +} + +// safeSegment refuses suite, component and architecture names that would +// leave the dists/ path they are placed in. +func safeSegment(s string) bool { + return s != "." && s != ".." && !strings.ContainsAny(s, `/\?#%`) +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// index reads every configured index, once. Nothing is kept unless all of +// them were read and checked, so a failed read is tried again next time. +func (r *Registry) index(ctx context.Context) (map[string][]record, error) { + if r.configErr != nil { + return nil, r.configErr + } + r.mu.Lock() + defer r.mu.Unlock() + if r.packages != nil { + return r.packages, nil + } + + packages := map[string][]record{} + seen := map[string]bool{} + add := func(rec record) { + key := rec.name + "\x00" + rec.version + "\x00" + rec.arch + if seen[key] { + return + } + seen[key] = true + packages[rec.name] = append(packages[rec.name], rec) + } + read := 0 + for _, suite := range r.suites { + n, err := r.readSuite(ctx, suite, add) + if err != nil { + return nil, err + } + read += n + } + if read == 0 { + return nil, fmt.Errorf("%w: under %s for suites %v", ErrNoIndex, r.archiveURL, r.suites) + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].version, recs[j].version, ecosystem) > 0 + }) + } + r.packages = packages + return packages, nil +} + +// readSuite reads a suite's indexes and returns how many it found. +func (r *Registry) readSuite(ctx context.Context, suite string, add func(record)) (int, error) { + rel, err := r.fetchRelease(ctx, suite) + if err != nil { + return 0, err + } + codename := suite + if rel != nil && rel.codename != "" { + codename = rel.codename + } + + read := 0 + for _, component := range r.components { + dirs := make([]string, 0, len(r.architectures)+1) + for _, arch := range r.architectures { + dirs = append(dirs, component+"/binary-"+arch) + } + // Architecture: all packages are in every binary- index, + // and in binary-all where the Release lists one. + if rel != nil && rel.lists(component+"/binary-"+archAll) { + dirs = append(dirs, component+"/binary-"+archAll) + } + for _, dir := range dirs { + found, err := r.readIndex(ctx, suite, dir, rel, func(rec record) { + rec.suite, rec.component, rec.distro = suite, component, distro(codename) + add(rec) + }) + if err != nil { + return 0, err + } + if found { + read++ + } + } + } + return read, nil +} + +// fetchRelease reads InRelease, falling back to Release. It returns nil +// when the suite has neither, and its indexes are read unchecked. +func (r *Registry) fetchRelease(ctx context.Context, suite string) (*release, error) { + for _, name := range []string{"InRelease", "Release"} { + body, err := r.client.GetBody(ctx, r.archiveURL+"/dists/"+suite+"/"+name) + if isNotFound(err) { + continue + } + if err != nil { + return nil, err + } + return parseRelease(body) + } + return nil, nil +} + +// lists reports whether the Release has an index variant for dir. +func (rel *release) lists(dir string) bool { + for _, variant := range indexVariants { + if _, ok := rel.files[dir+"/"+variant]; ok { + return true + } + } + return false +} + +// readIndex reads the first variant of dir's index that is served. With a +// Release, only variants it lists are tried, and each is checked against +// its entry. It reports whether an index was read. +func (r *Registry) readIndex(ctx context.Context, suite, dir string, rel *release, fn func(record)) (bool, error) { + for _, variant := range indexVariants { + var sum *fileSum + if rel != nil && len(rel.files) > 0 { + entry, ok := rel.files[dir+"/"+variant] + if !ok { + continue + } + sum = &entry + } + indexURL := r.archiveURL + "/dists/" + suite + "/" + dir + "/" + variant + var recs []record + err := r.client.Stream(ctx, indexURL, func(body io.Reader) error { + var err error + recs, err = r.parseIndex(body, variant, indexURL, sum) + return err + }) + if isNotFound(err) { + continue + } + if err != nil { + return false, err + } + for _, rec := range recs { + fn(rec) + } + return true, nil + } + return false, nil +} + +// parseIndex decompresses and parses one index, checking the compressed +// bytes against sum when there is one. +func (r *Registry) parseIndex(body io.Reader, variant, indexURL string, sum *fileSum) ([]record, error) { + hasher := sha256.New() + counted := &countingReader{r: io.TeeReader(body, hasher)} + + var decoded io.Reader + switch variant { + case "Packages.xz": + data, err := io.ReadAll(io.LimitReader(counted, r.maxXZSize+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > r.maxXZSize { + return nil, fmt.Errorf("%w: %s is larger than %d bytes", ErrMalformedIndex, indexURL, r.maxXZSize) + } + // Checked before decoding, so a bad file never reaches the decoder. + if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { + return nil, err + } + if err := checkXZ(data, maxXZDict); err != nil { + return nil, err + } + xr, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + decoded = xr + case "Packages.gz": + gz, err := gzip.NewReader(counted) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + defer func() { _ = gz.Close() }() + decoded = gz + default: + decoded = counted + } + + limited := &io.LimitedReader{R: decoded, N: r.maxIndexSize + 1} + var recs []record + if err := parseStanzas(limited, func(rec record) { recs = append(recs, rec) }); err != nil { + return nil, err + } + if limited.N == 0 { + return nil, fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedIndex, indexURL, r.maxIndexSize) + } + // The gzip reader and the parser read to the end of the body. + if variant != "Packages.xz" { + if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { + return nil, err + } + } + return recs, nil +} + +func checkSum(h hash.Hash, size int64, sum *fileSum, indexURL string) error { + if sum == nil { + return nil + } + if got := hex.EncodeToString(h.Sum(nil)); got != sum.sha256 || size != sum.size { + return fmt.Errorf("%w: %s is %d bytes with SHA256 %s, Release lists %d bytes with %s", + ErrChecksumMismatch, indexURL, size, got, sum.size, sum.sha256) + } + return nil +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +func isNotFound(err error) bool { + var httpErr *core.HTTPError + return errors.As(err, &httpErr) && httpErr.IsNotFound() +} + +// splitName strips a PURL vendor namespace: "debian/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + packages, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.description, + Homepage: latest.homepage, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "source": latest.source, + "section": latest.section, + }, + }, nil +} + +// FetchVersions returns one Version per version number, newest first, with +// an artifact for each architecture and suite it was found in. +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + var versions []core.Version + for _, rec := range recs { + if len(versions) == 0 || versions[len(versions)-1].Number != rec.version { + versions = append(versions, core.Version{ + Number: rec.version, + Metadata: map[string]any{ + "source": rec.source, + "section": rec.section, + "maintainer": rec.maintainer, + "installed_size": rec.installedSize, + }, + }) + } + if artifact, ok := r.artifact(rec); ok { + v := &versions[len(versions)-1] + v.Artifacts = append(v.Artifacts, artifact) + } + } + return versions, nil +} + +func (r *Registry) artifact(rec record) (core.Artifact, bool) { + if !safePoolPath(rec.filename) { + return core.Artifact{}, false + } + artifact := core.Artifact{ + URL: r.archiveURL + "/" + rec.filename, + Filename: path.Base(rec.filename), + Size: rec.size, + Metadata: map[string]any{ + "arch": rec.arch, + "distro": rec.distro, + "suite": rec.suite, + "component": rec.component, + }, + } + if rec.sha256 != "" { + artifact.Integrity = "sha256-" + strings.ToLower(rec.sha256) + } + return artifact, true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version != version { + continue + } + var deps []core.Dependency + for _, d := range parseDependencies(rec.preDepends, rec.depends) { + deps = append(deps, core.Dependency{Name: d.name, Requirements: d.requirement, Scope: core.Runtime}) + } + return deps, nil + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].maintainer == "" { + return nil, nil + } + return []core.Maintainer{parseMaintainer(recs[0].maintainer)}, nil +} + +// parseMaintainer parses "Name "; any other value is kept whole as the name. +func parseMaintainer(value string) core.Maintainer { + if open := strings.LastIndex(value, "<"); open >= 0 && strings.HasSuffix(value, ">") { + name := strings.TrimSpace(value[:open]) + email := value[open+1 : len(value)-1] + if email != "" && strings.Contains(email, "@") && !strings.ContainsAny(email, "<> ") { + return core.Maintainer{Name: name, Email: email} + } + } + return core.Maintainer{Name: strings.TrimSpace(value)} +} + +type URLs struct { + archiveURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns "": a package's file is only known from the index. +func (u *URLs) Download(name, version string) string { + return "" +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + var qualifiers map[string]string + if u.archiveURL != "" { + qualifiers = map[string]string{"repository_url": u.archiveURL} + } + return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() +} diff --git a/internal/deb/deb_test.go b/internal/deb/deb_test.go new file mode 100644 index 0000000..b3187e0 --- /dev/null +++ b/internal/deb/deb_test.go @@ -0,0 +1,645 @@ +package deb + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/registries/internal/core" + "github.com/ulikunitz/xz" +) + +// Trimmed from deb.debian.org's bookworm main/binary-amd64 and +// bookworm-updates indexes. +const bookwormAMD64 = `Package: curl +Version: 7.88.1-10+deb12u5 +Installed-Size: 500 +Maintainer: Alessandro Ghedini +Architecture: amd64 +Depends: libc6 (>= 2.34), libcurl4 (= 7.88.1-10+deb12u5), zlib1g (>= 1:1.1.4) +Description: command line tool for transferring data with URL syntax + curl is a command line tool for transferring data with URL syntax. +Homepage: https://curl.se/ +Section: web +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u5_amd64.deb +Size: 315860 +SHA256: 1b2f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f80 + +Package: openssh-client +Source: openssh +Version: 1:9.2p1-2+deb12u3 +Maintainer: Debian OpenSSH Maintainers +Architecture: amd64 +Pre-Depends: libc6 (>= 2.34) +Depends: adduser, passwd, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>= 1:1.1.4), libselinux1 [linux-any] +Tag: implemented-in::c, interface::commandline, interface::shell, + network::client, protocol::sftp +Description: secure shell (SSH) client, for secure access to remote machines +Section: net +Filename: pool/main/o/openssh/openssh-client_9.2p1-2+deb12u3_amd64.deb +Size: 991056 +SHA256: 2c3f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f81 + +Package: ca-certificates +Version: 20230311 +Maintainer: Julien Cristau +Architecture: all +Depends: openssl (>= 1.1.1), debconf (>= 0.5) | debconf-2.0 +Description: Common CA certificates +Filename: pool/main/c/ca-certificates/ca-certificates_20230311_all.deb +Size: 153656 +SHA256: 3d4f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f82 +` + +const bookwormARM64 = `Package: curl +Version: 7.88.1-10+deb12u5 +Maintainer: Alessandro Ghedini +Architecture: arm64 +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u5_arm64.deb +Size: 301024 +SHA256: 4e5f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f83 +` + +const updatesAMD64 = `Package: curl +Version: 7.88.1-10+deb12u14 +Maintainer: Alessandro Ghedini +Architecture: amd64 +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u14_amd64.deb +Size: 316112 +SHA256: 5f6f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f84 +` + +func xzData(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w, err := xz.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func gzData(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// archive is a fake Debian archive. served are the files it answers with; +// listed are entries its Release files name, served or not. +type archive struct { + t *testing.T + mu sync.Mutex + served map[string][]byte + listed map[string][]byte + requests []string +} + +func newArchive(t *testing.T) *archive { + return &archive{t: t, served: map[string][]byte{}, listed: map[string][]byte{}} +} + +// index adds one Packages index of suite, as variant ("Packages.xz", +// "Packages.gz" or "Packages"), served and listed. +func (a *archive) index(suite, dir, variant, stanzas string) { + data := []byte(stanzas) + switch variant { + case "Packages.xz": + data = xzData(a.t, stanzas) + case "Packages.gz": + data = gzData(a.t, stanzas) + } + path := "dists/" + suite + "/" + dir + "/" + variant + a.served[path] = data + a.listed[path] = data +} + +// release writes the suite's Release (or InRelease, clearsigned) over the +// files listed under it. +func (a *archive) release(suite, codename string, signed bool) { + prefix := "dists/" + suite + "/" + var paths []string + for p := range a.listed { + if strings.HasPrefix(p, prefix) { + paths = append(paths, p) + } + } + sort.Strings(paths) + var b strings.Builder + fmt.Fprintf(&b, "Origin: Debian\nSuite: %s\nCodename: %s\nArchitectures: all amd64 arm64\nComponents: main\nMD5Sum:\n", suite, codename) + for _, p := range paths { + fmt.Fprintf(&b, " 00000000000000000000000000000000 %8d %s\n", len(a.listed[p]), strings.TrimPrefix(p, prefix)) + } + b.WriteString("SHA256:\n") + for _, p := range paths { + sum := sha256.Sum256(a.listed[p]) + fmt.Fprintf(&b, " %s %8d %s\n", hex.EncodeToString(sum[:]), len(a.listed[p]), strings.TrimPrefix(p, prefix)) + } + if !signed { + a.served[prefix+"Release"] = []byte(b.String()) + return + } + a.served[prefix+"InRelease"] = []byte("-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + b.String() + + "-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCAAdFiEE\n-----END PGP SIGNATURE-----\n") +} + +func (a *archive) server() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/debian/") + a.mu.Lock() + a.requests = append(a.requests, path) + data, ok := a.served[path] + a.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(data) + })) + a.t.Cleanup(srv.Close) + return srv +} + +// standard is bookworm (InRelease, xz, amd64 and arm64) plus +// bookworm-updates (plain Release, gz only). +func standard(t *testing.T) (*archive, string) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64) + a.release("bookworm", "bookworm", true) + a.index("bookworm-updates", "main/binary-amd64", "Packages.gz", updatesAMD64) + a.release("bookworm-updates", "bookworm-updates", false) + srv := a.server() + return a, srv.URL + "/debian" +} + +func newRegistry(base, query string) *Registry { + return New(base+"?"+query, core.DefaultClient()) +} + +func numbers(versions []core.Version) []string { + var out []string + for _, v := range versions { + out = append(out, v.Number) + } + return out +} + +func TestFetchVersions(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&components=main&architectures=amd64") + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"7.88.1-10+deb12u14", "7.88.1-10+deb12u5"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (newest first, by deb version)", numbers(versions), want) + } + + newest := versions[0].Artifacts + if len(newest) != 1 { + t.Fatalf("artifacts = %+v, want one", newest) + } + got := newest[0] + want := core.Artifact{ + URL: base + "/pool/main/c/curl/curl_7.88.1-10+deb12u14_amd64.deb", + Filename: "curl_7.88.1-10+deb12u14_amd64.deb", + Integrity: "sha256-5f6f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f84", + Size: 316112, + Metadata: map[string]any{"arch": "amd64", "distro": "bookworm", "suite": "bookworm-updates", "component": "main"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("artifact =\n %+v\nwant\n %+v", got, want) + } + if d := versions[1].Artifacts[0].Metadata["distro"]; d != "bookworm" { + t.Errorf("bookworm artifact distro = %v", d) + } +} + +func TestFetchVersionsAcrossArchitectures(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + var archs []any + for _, a := range versions[0].Artifacts { + archs = append(archs, a.Metadata["arch"]) + } + if want := []any{"amd64", "arm64"}; !reflect.DeepEqual(archs, want) { + t.Errorf("artifact archs = %v, want %v", archs, want) + } +} + +// Architecture: all packages appear in every binary- index and in +// binary-all where the Release lists one; they are recorded once, with +// their own architecture. +func TestArchitectureAll(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64+"\n"+caCertificatesOnly) + a.index("bookworm", "main/binary-all", "Packages.xz", caCertificatesOnly) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + versions, err := r.FetchVersions(context.Background(), "ca-certificates") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if arts := versions[0].Artifacts; len(arts) != 1 || arts[0].Metadata["arch"] != "all" { + t.Errorf("artifacts = %+v, want one, arch all", arts) + } + if !a.requested("dists/bookworm/main/binary-all/Packages.xz") { + t.Errorf("binary-all was not read: %v", a.requests) + } +} + +const caCertificatesOnly = `Package: ca-certificates +Version: 20230311 +Architecture: all +Filename: pool/main/c/ca-certificates/ca-certificates_20230311_all.deb +Size: 153656 +SHA256: 3d4f0f3c2a4bdbe8b1e3f9a1b6d7c9e0f1a2b3c4d5e6f708192a3b4c5d6e7f82 +` + +func (a *archive) requested(path string) bool { + a.mu.Lock() + defer a.mu.Unlock() + for _, p := range a.requests { + if p == path { + return true + } + } + return false +} + +// deb.debian.org lists a plain Packages it answers 404 for, and other +// archives list no .xz: the variants are tried in order past a 404. +func TestIndexVariantFallback(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.listed["dists/bookworm/main/binary-amd64/Packages"] = []byte(bookwormAMD64) + delete(a.served, "dists/bookworm/main/binary-amd64/Packages.xz") + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + delete(a.served, "dists/bookworm/main/binary-amd64/Packages.gz") + a.served["dists/bookworm/main/binary-amd64/Packages"] = []byte(bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } +} + +// Variants a Release doesn't list are not requested. +func TestIndexVariantsNotListedAreSkipped(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + a.release("bookworm", "bookworm", true) + a.served["dists/bookworm/main/binary-amd64/Packages.xz"] = []byte("not an index") + base := a.server().URL + "/debian" + + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if a.requested("dists/bookworm/main/binary-amd64/Packages.xz") { + t.Error("requested a variant the Release doesn't list") + } +} + +// Without a Release, as some Artifactory setups serve, the index is read +// without a checksum to check against. +func TestNoRelease(t *testing.T) { + a := newArchive(t) + a.index("stable", "main/binary-amd64", "Packages.gz", bookwormAMD64) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=stable&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if d := versions[0].Artifacts[0].Metadata["distro"]; d != "stable" { + t.Errorf("distro = %v, want the suite name without a Release", d) + } +} + +func TestChecksumMismatch(t *testing.T) { + for _, variant := range []string{"Packages.xz", "Packages.gz", "Packages"} { + t.Run(variant, func(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", variant, bookwormAMD64) + a.release("bookworm", "bookworm", true) + path := "dists/bookworm/main/binary-amd64/" + variant + switch variant { + case "Packages.xz": + a.served[path] = xzData(t, updatesAMD64) + case "Packages.gz": + a.served[path] = gzData(t, updatesAMD64) + default: + a.served[path] = []byte(updatesAMD64) + } + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64") + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("err = %v, want ErrChecksumMismatch", err) + } + // Nothing was kept: the next lookup reads the archive again. + a.served[path] = a.listed[path] + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("after the archive was fixed: %v", err) + } + }) + } +} + +func TestInvalidConfiguration(t *testing.T) { + _, base := standard(t) + for name, url := range map[string]string{ + "no url": "", + "no host": "/debian?suites=bookworm&architectures=amd64", + "suite path": base + "?suites=../../etc&architectures=amd64", + "component path": base + "?suites=bookworm&components=a/b&architectures=amd64", + "architecture path": base + "?suites=bookworm&architectures=..", + "no suites": base + "?architectures=amd64", + "no architectures": base + "?suites=bookworm", + "source": base + "?suites=bookworm&architectures=source", + } { + t.Run(name, func(t *testing.T) { + _, err := New(url, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("err = %v, want ErrInvalidConfig", err) + } + }) + } +} + +func TestNoIndexFound(t *testing.T) { + a := newArchive(t) + base := a.server().URL + "/debian" + _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrNoIndex) { + t.Errorf("err = %v, want ErrNoIndex", err) + } +} + +func TestNotFound(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "missing"); !errors.As(err, ¬Found) { + t.Errorf("missing package: err = %v, want NotFoundError", err) + } + if _, err := r.FetchDependencies(context.Background(), "curl", "1.0-1"); !errors.As(err, ¬Found) || notFound.Version != "1.0-1" { + t.Errorf("missing version: err = %v, want NotFoundError for 1.0-1", err) + } +} + +// The epoch is part of Version but never of Filename. +func TestEpochVersion(t *testing.T) { + _, base := standard(t) + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "debian/openssh-client") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + if v := versions[0]; v.Number != "1:9.2p1-2+deb12u3" || v.Artifacts[0].Filename != "openssh-client_9.2p1-2+deb12u3_amd64.deb" { + t.Errorf("version = %q, filename = %q", v.Number, v.Artifacts[0].Filename) + } +} + +func TestFetchPackage(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&architectures=amd64") + pkg, err := r.FetchPackage(context.Background(), "debian/curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Name != "curl" || pkg.Namespace != "debian" || pkg.LatestVersion != "7.88.1-10+deb12u14" { + t.Errorf("name = %q, namespace = %q, latest = %q", pkg.Name, pkg.Namespace, pkg.LatestVersion) + } + pkg, err = r.FetchPackage(context.Background(), "openssh-client") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Description != "secure shell (SSH) client, for secure access to remote machines" || pkg.Metadata["source"] != "openssh" || pkg.Metadata["section"] != "net" { + t.Errorf("package = %+v", pkg) + } +} + +func TestFetchDependencies(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + deps, err := r.FetchDependencies(context.Background(), "openssh-client", "1:9.2p1-2+deb12u3") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "libc6", Requirements: ">= 2.34", Scope: core.Runtime}, + {Name: "adduser", Scope: core.Runtime}, + {Name: "passwd", Scope: core.Runtime}, + {Name: "libssl3", Requirements: ">= 3.0.13", Scope: core.Runtime}, + {Name: "zlib1g", Requirements: ">= 1:1.1.4", Scope: core.Runtime}, + {Name: "libselinux1", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +func TestFetchMaintainers(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + got, err := r.FetchMaintainers(context.Background(), "curl") + if err != nil || !reflect.DeepEqual(got, []core.Maintainer{{Name: "Alessandro Ghedini", Email: "ghedo@debian.org"}}) { + t.Errorf("maintainers = %+v, %v", got, err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://deb.debian.org/debian/?suites=bookworm&architectures=amd64", nil).URLs() + if got := u.Download("curl", "7.88.1-10"); got != "" { + t.Errorf("Download = %q, want none without the index", got) + } + if got, want := u.PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } + if got, want := u.PURL("debian/curl", "1:9.2p1-2"), "pkg:deb/debian/curl@1:9.2p1-2?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian"; got != want { + t.Errorf("PURL with epoch = %q, want %q", got, want) + } +} + +// An index is read once per Registry. +func TestIndexReadOnce(t *testing.T) { + a, base := standard(t) + r := newRegistry(base, "suites=bookworm&architectures=amd64") + for range 3 { + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + } + n := 0 + for _, p := range a.requests { + if strings.HasSuffix(p, "/Packages.xz") { + n++ + } + } + if n != 1 { + t.Errorf("index requested %d times, want once", n) + } +} + +func TestDecompressedIndexLimit(t *testing.T) { + for _, variant := range []string{"Packages.xz", "Packages.gz", "Packages"} { + t.Run(variant, func(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", variant, bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxIndexSize = int64(len(bookwormAMD64)) - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("over the limit: err = %v, want ErrMalformedIndex", err) + } + r = newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxIndexSize = int64(len(bookwormAMD64)) + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } + }) + } +} + +func TestXZSizeLimit(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + size := int64(len(a.served["dists/bookworm/main/binary-amd64/Packages.xz"])) + + r := newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxXZSize = size - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("over the limit: err = %v, want ErrMalformedIndex", err) + } + r = newRegistry(base, "suites=bookworm&architectures=amd64") + r.maxXZSize = size + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } +} + +// An .xz whose blocks ask for too large a dictionary is refused before it +// is decoded, even when the Release lists it. +func TestXZDictionaryRefused(t *testing.T) { + a := newArchive(t) + path := "dists/bookworm/main/binary-amd64/Packages.xz" + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.served[path] = setXZDict(t, a.served[path], 30) // 128 MiB + a.listed[path] = a.served[path] + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } +} + +// A suite named by its alias, such as stable, records the Release's +// codename. +func TestDistroFromRelease(t *testing.T) { + a := newArchive(t) + a.index("stable", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.release("stable", "bookworm", true) + base := a.server().URL + "/debian" + versions, err := newRegistry(base, "suites=stable&architectures=amd64").FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatal(err) + } + if d := versions[0].Artifacts[0].Metadata["distro"]; d != "bookworm" { + t.Errorf("distro = %v, want the Release's codename", d) + } +} + +// A Release with no SHA256 entries, only MD5Sum, checks nothing and lists +// nothing, so every variant is tried. +func TestReleaseWithoutSHA256(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.gz", bookwormAMD64) + a.served["dists/bookworm/Release"] = []byte("Codename: bookworm\nMD5Sum:\n 0123 12 main/binary-amd64/Packages.gz\n") + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +// Same size, other content: only the hash tells them apart. +func TestChecksumHashMismatch(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages", bookwormAMD64) + a.release("bookworm", "bookworm", true) + a.served["dists/bookworm/main/binary-amd64/Packages"] = []byte(strings.Replace(bookwormAMD64, "Size: 315860", "Size: 315861", 1)) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +// A Filename leaving the archive root gets no artifact. +func TestUnsafeFilename(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", "Package: evil\nVersion: 1\nArchitecture: amd64\nFilename: ../../etc/passwd\n") + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + versions, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "evil") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 0 { + t.Errorf("versions = %+v, %v; want one without artifacts", versions, err) + } + // It has no Maintainer either. + r := newRegistry(base, "suites=bookworm&architectures=amd64") + if got, err := r.FetchMaintainers(context.Background(), "evil"); err != nil || got != nil { + t.Errorf("maintainers = %+v, %v; want none", got, err) + } +} + +// A Release whose SHA256 entry has the right hash but another size is +// rejected too. +func TestChecksumSizeMismatch(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages", bookwormAMD64) + a.release("bookworm", "bookworm", false) + rel := string(a.served["dists/bookworm/Release"]) + a.served["dists/bookworm/Release"] = []byte(strings.ReplaceAll(rel, + fmt.Sprintf(" %8d main/binary-amd64/Packages\n", len(bookwormAMD64)), + fmt.Sprintf(" %8d main/binary-amd64/Packages\n", len(bookwormAMD64)+1))) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64").FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} diff --git a/internal/deb/parse.go b/internal/deb/parse.go new file mode 100644 index 0000000..40861fc --- /dev/null +++ b/internal/deb/parse.go @@ -0,0 +1,288 @@ +package deb + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" +) + +// maxLineSize bounds one index line. bookworm main has a 75,649-byte +// Provides: line. +const maxLineSize = 1 << 20 + +// record is one stanza of a Packages index, with the fields the client uses. +type record struct { + name string + version string + arch string + source string + maintainer string + description string + homepage string + section string + filename string + sha256 string + size int64 + installedSize int64 + depends string + preDepends string + + // Where the stanza was read. + suite string + component string + distro string +} + +// parseStanzas reads a Packages index as a stream, calling fn for each +// stanza. Field names are matched case-insensitively and folded lines are +// joined, as deb822(5) asks of readers. +func parseStanzas(r io.Reader, fn func(record)) error { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxLineSize) + var rec record + var field string // the field a continuation line belongs to + inStanza := false + lineNo := 0 + + finish := func() error { + if !inStanza { + return nil + } + if rec.name == "" || rec.version == "" { + return fmt.Errorf("%w: stanza ending at line %d has no Package or Version", ErrMalformedIndex, lineNo) + } + fn(rec) + rec, field, inStanza = record{}, "", false + return nil + } + + for scanner.Scan() { + lineNo++ + line := scanner.Text() + if strings.TrimSpace(line) == "" { + if err := finish(); err != nil { + return err + } + continue + } + if line[0] == ' ' || line[0] == '\t' { + if !inStanza { + return fmt.Errorf("%w: line %d continues no field", ErrMalformedIndex, lineNo) + } + rec.continueField(field, strings.TrimSpace(line)) + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + return fmt.Errorf("%w: line %d is not a field", ErrMalformedIndex, lineNo) + } + inStanza = true + field = strings.ToLower(key) + if err := rec.set(field, strings.TrimSpace(value)); err != nil { + return fmt.Errorf("%w: line %d: %w", ErrMalformedIndex, lineNo, err) + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + return finish() +} + +func (rec *record) set(field, value string) error { + var err error + switch field { + case "package": + rec.name = value + case "version": + rec.version = value + case "architecture": + rec.arch = value + case "source": + rec.source = value + case "maintainer": + rec.maintainer = value + case "description": + rec.description = value + case "homepage": + rec.homepage = value + case "section": + rec.section = value + case "filename": + rec.filename = value + case "sha256": + rec.sha256 = value + case "size": + rec.size, err = strconv.ParseInt(value, 10, 64) + case "installed-size": + rec.installedSize, err = strconv.ParseInt(value, 10, 64) + case "depends": + rec.depends = value + case "pre-depends": + rec.preDepends = value + } + return err +} + +// continueField appends a folded line to the relationship fields. Other +// multi-line fields, such as Description, keep their first line. +func (rec *record) continueField(field, value string) { + switch field { + case "depends": + rec.depends = joinFolded(rec.depends, value) + case "pre-depends": + rec.preDepends = joinFolded(rec.preDepends, value) + } +} + +func joinFolded(s, more string) string { + if s == "" { + return more + } + return s + " " + more +} + +// fileSum is a Release file's SHA256 entry for one index file. +type fileSum struct { + sha256 string + size int64 +} + +// release is the part of a Release or InRelease file the client uses. +type release struct { + codename string + suite string + // files maps a path under dists// to its SHA256 entry. + files map[string]fileSum +} + +// parseRelease reads a Release file, or an InRelease file, whose +// clearsigned text is read without checking the signature. +func parseRelease(data []byte) (*release, error) { + rel := &release{files: map[string]fileSum{}} + scanner := bufio.NewScanner(bytes.NewReader(clearsignedText(data))) + scanner.Buffer(nil, maxLineSize) + field := "" + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + if line[0] == ' ' || line[0] == '\t' { + if field != "sha256" { + continue + } + parts := strings.Fields(line) + if len(parts) != 3 { + return nil, fmt.Errorf("%w: Release SHA256 entry %q", ErrMalformedIndex, line) + } + size, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: Release SHA256 entry %q", ErrMalformedIndex, line) + } + rel.files[parts[2]] = fileSum{sha256: strings.ToLower(parts[0]), size: size} + continue + } + key, value, _ := strings.Cut(line, ":") + field = strings.ToLower(key) + switch field { + case "codename": + rel.codename = strings.TrimSpace(value) + case "suite": + rel.suite = strings.TrimSpace(value) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + return rel, nil +} + +// clearsignedText returns the signed text of an OpenPGP clearsigned +// message, with dash escapes removed, or data unchanged if it is not one. +func clearsignedText(data []byte) []byte { + const begin, sig = "-----BEGIN PGP SIGNED MESSAGE-----", "-----BEGIN PGP SIGNATURE-----" + if !bytes.HasPrefix(data, []byte(begin)) { + return data + } + // Armor headers such as Hash: end at the first blank line. + _, body, ok := bytes.Cut(data, []byte("\n\n")) + if !ok { + return nil + } + if i := bytes.Index(body, []byte("\n"+sig)); i >= 0 { + body = body[:i+1] + } + lines := bytes.SplitAfter(body, []byte("\n")) + for i, line := range lines { + lines[i] = bytes.TrimPrefix(line, []byte("- ")) + } + return bytes.Join(lines, nil) +} + +// distro is the codename a suite's packages are recorded under: +// bookworm-updates and bookworm-security are bookworm. +func distro(codename string) string { + name, _, _ := strings.Cut(codename, "-") + return name +} + +// dependency is one relationship from Depends or Pre-Depends. +type dependency struct { + name string + requirement string +} + +// parseDependencies reads relationship fields. Of alternatives ("a | b") +// the first is kept, as apt tries it first; architecture qualifiers +// (zlib1g:any), architecture restrictions ([linux-any]) and build profiles +// () are dropped. +func parseDependencies(fields ...string) []dependency { + var deps []dependency + for _, field := range fields { + for _, group := range strings.Split(field, ",") { + first, _, _ := strings.Cut(group, "|") + first = cutBetween(cutBetween(first, "[", "]"), "<", ">") + name, requirement, _ := strings.Cut(first, "(") + name = strings.TrimSpace(name) + name, _, _ = strings.Cut(name, ":") + if name == "" { + continue + } + requirement = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(requirement), ")")) + deps = append(deps, dependency{name: name, requirement: requirement}) + } + } + return deps +} + +// cutBetween removes every open...close span from s. +func cutBetween(s, open, close string) string { + for { + i := strings.Index(s, open) + if i < 0 { + return s + } + j := strings.Index(s[i:], close) + if j < 0 { + return s[:i] + } + s = s[:i] + s[i+j+len(close):] + } +} + +// safePoolPath reports whether a Filename stays under the archive root: +// relative, without dot segments, escapes or URL syntax. +func safePoolPath(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.ContainsAny(p, `\?#%:`) { + return false + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go new file mode 100644 index 0000000..93ddef2 --- /dev/null +++ b/internal/deb/parse_test.go @@ -0,0 +1,295 @@ +package deb + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "reflect" + "strings" + "testing" +) + +func TestParseStanzas(t *testing.T) { + long := strings.Repeat("libfoo, ", 10_000) // longer than bufio's 64 KiB default + index := "package: Foo\nVERSION: 1.0-1\nArchitecture: amd64\n" + + "Depends: libc6 (>= 2.34),\n libssl3 (>= 3)\n" + + "Provides: " + long + "\n" + + "Description: first line\n second line\n" + + "Filename: pool/f/foo_1.0-1_amd64.deb\n \t\n" + + "Package: bar\nVersion: 2\nArchitecture: all\n" + var got []record + err := parseStanzas(strings.NewReader(index), func(rec record) { got = append(got, rec) }) + if err != nil { + t.Fatalf("parseStanzas: %v", err) + } + if len(got) != 2 { + t.Fatalf("records = %+v, want two", got) + } + foo := got[0] + if foo.name != "Foo" || foo.version != "1.0-1" || foo.arch != "amd64" || foo.filename != "pool/f/foo_1.0-1_amd64.deb" { + t.Errorf("foo = %+v", foo) + } + if foo.depends != "libc6 (>= 2.34), libssl3 (>= 3)" { + t.Errorf("folded Depends = %q", foo.depends) + } + if foo.description != "first line" { + t.Errorf("description = %q, want its first line", foo.description) + } + if got[1].name != "bar" || got[1].arch != "all" { + t.Errorf("bar = %+v", got[1]) + } +} + +func TestParseStanzasMalformed(t *testing.T) { + for name, index := range map[string]string{ + "no colon": "Package: foo\nVersion 1\n", + "no package": "Version: 1\nArchitecture: amd64\n", + "no version": "Package: foo\n\nPackage: bar\nVersion: 1\n", + "leading continuation": " continued\nPackage: foo\nVersion: 1\n", + } { + t.Run(name, func(t *testing.T) { + err := parseStanzas(strings.NewReader(index), func(record) {}) + if !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } + }) + } +} + +func TestParseRelease(t *testing.T) { + data := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + + "Suite: oldstable-updates\nCodename: bookworm-updates\n" + + "Architectures: all amd64\nMD5Sum:\n 0123 12 main/binary-amd64/Packages.xz\n 4567 3 main/binary-arm64/Packages\n" + + "SHA256:\n abcd 12 main/binary-amd64/Packages.xz\n ef01 0 main/binary-all/Packages\n" + + "- Dash-Escaped: kept\n" + + "-----BEGIN PGP SIGNATURE-----\n\niQIz\n-----END PGP SIGNATURE-----\n" + + // Text after the signature isn't signed, and is not read. + "Codename: unsigned\nSHA256:\n 9999 1 main/binary-amd64/Packages\n" + rel, err := parseRelease([]byte(data)) + if err != nil { + t.Fatalf("parseRelease: %v", err) + } + if rel.codename != "bookworm-updates" { + t.Errorf("codename = %q", rel.codename) + } + want := map[string]fileSum{ + "main/binary-amd64/Packages.xz": {sha256: "abcd", size: 12}, + "main/binary-all/Packages": {sha256: "ef01", size: 0}, + } + if !reflect.DeepEqual(rel.files, want) { + t.Errorf("files = %+v, want %+v", rel.files, want) + } +} + +func TestParseReleaseMalformedEntry(t *testing.T) { + for _, entry := range []string{" abcd notasize main/binary-amd64/Packages", " abcd 12 main/binary-amd64/Packages extra", " abcd 12"} { + if _, err := parseRelease([]byte("SHA256:\n" + entry + "\n")); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("%q: err = %v, want ErrMalformedIndex", entry, err) + } + } +} + +func TestDistroFromCodename(t *testing.T) { + for in, want := range map[string]string{ + "bookworm": "bookworm", + "bookworm-updates": "bookworm", + "bookworm-security": "bookworm", + "jammy-updates": "jammy", + } { + if got := distro(in); got != want { + t.Errorf("distro(%q) = %q, want %q", in, got, want) + } + } +} + +func TestParseDependencies(t *testing.T) { + got := parseDependencies("libc6 (>= 2.34)", "adduser, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>=1:1.1.4), libselinux1 [linux-any], libfoo , ") + want := []dependency{ + {"libc6", ">= 2.34"}, {"adduser", ""}, {"libssl3", ">= 3.0.13"}, + {"zlib1g", ">=1:1.1.4"}, {"libselinux1", ""}, {"libfoo", ""}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseDependencies =\n %+v\nwant\n %+v", got, want) + } +} + +func TestPoolPath(t *testing.T) { + for in, ok := range map[string]bool{ + "pool/main/c/curl/curl_7.88.1-10_amd64.deb": true, + "": false, + "/etc/passwd": false, + "pool/../../etc/passwd": false, + "pool/x.deb?query": false, + "pool/x.deb#frag": false, + `pool\x.deb`: false, + "pool/%2e%2e/x.deb": false, + "http://evil.example/x.deb": false, + } { + if got := safePoolPath(in); got != ok { + t.Errorf("safePoolPath(%q) = %v, want %v", in, got, ok) + } + } +} + +// setXZDict rewrites the dictionary size of every LZMA2 block header in +// data, fixing the header CRC, so a small file claims a large dictionary. +func setXZDict(t *testing.T, data []byte, bits byte) []byte { + t.Helper() + out := bytes.Clone(data) + // A file written by xzData holds one stream with one block, whose + // header starts after the 12-byte stream header. + const start = 12 + size := (int(out[start]) + 1) * 4 + header := out[start : start+size] + i := bytes.Index(header, []byte{0x21, 0x01}) + if i < 0 { + t.Fatal("no LZMA2 filter in the block header") + } + header[i+2] = bits + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} + +func TestCheckXZDictionary(t *testing.T) { + small := xzData(t, bookwormAMD64) + if err := checkXZ(small, maxXZDict); err != nil { + t.Fatalf("checkXZ(8 MiB dictionary): %v", err) + } + // 2 GiB, past the 64 MiB limit. + if err := checkXZ(setXZDict(t, small, 38), maxXZDict); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("2 GiB dictionary: err = %v, want ErrMalformedIndex", err) + } + // A large dictionary in a second stream, after padding, is found too. + two := append(append(bytes.Clone(small), 0, 0, 0, 0), setXZDict(t, small, 38)...) + if err := checkXZ(two, maxXZDict); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("second stream: err = %v, want ErrMalformedIndex", err) + } + if err := checkXZ(append(append(bytes.Clone(small), 0, 0, 0, 0), small...), maxXZDict); err != nil { + t.Errorf("two small streams: %v", err) + } + for name, data := range map[string][]byte{ + "empty": nil, + "truncated": small[:len(small)-3], + "garbage": []byte(strings.Repeat("x", 64)), + } { + if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("%s: err = %v, want ErrMalformedIndex", name, err) + } + } +} + +func TestXZDictSize(t *testing.T) { + for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { + if got := lzma2DictSize(bits); got != want { + t.Errorf("lzma2DictSize(%d) = %d, want %d", bits, got, want) + } + } +} + +// xzVarint encodes an xz multibyte integer. +func xzVarint(v uint64) []byte { + var out []byte + for v >= 0x80 { + out = append(out, byte(v)|0x80) + v >>= 7 + } + return append(out, byte(v)) +} + +// xzBlockHeader builds a block header from its flags byte and the bytes +// after it (sizes and filter flags), padded and with a CRC32 slot. +func xzBlockHeader(flags byte, body ...[]byte) []byte { + h := []byte{0, flags} + for _, b := range body { + h = append(h, b...) + } + for (len(h)+4)%4 != 0 { + h = append(h, 0) + } + h = append(h, 0, 0, 0, 0) + h[0] = byte(len(h)/4 - 1) + return h +} + +func lzma2Props(bits byte) []byte { return []byte{lzma2Filter, 0x01, bits} } + +// xzStream builds one stream whose blocks hold only their headers. Nothing +// here is meant to decode; checkXZ reads only the container. +func xzStream(headers ...[]byte) []byte { + out := append(bytes.Clone(xzHeaderMagic), 0, 1, 0, 0, 0, 0) + index := []byte{0} + index = append(index, xzVarint(uint64(len(headers)))...) + for _, h := range headers { + out = append(out, h...) + index = append(index, xzVarint(uint64(len(h)))...) + index = append(index, xzVarint(1)...) + } + for len(index)%4 != 0 { + index = append(index, 0) + } + index = append(index, 0, 0, 0, 0) + out = append(out, index...) + footer := make([]byte, 12) + binary.LittleEndian.PutUint32(footer[4:8], uint32(len(index)/4-1)) + copy(footer[10:], xzFooterMagic) + return append(out, footer...) +} + +func TestCheckXZContainer(t *testing.T) { + small := lzma2Props(22) + large := lzma2Props(38) + valid := xzStream(xzBlockHeader(0x00, small)) + if err := checkXZ(valid, maxXZDict); err != nil { + t.Fatalf("valid synthetic stream: %v", err) + } + + patch := func(data []byte, at int, b ...byte) []byte { + out := bytes.Clone(data) + if at < 0 { + at += len(out) + } + copy(out[at:], b) + return out + } + indexStart := len(valid) - 12 - 8 // one record: indicator, count, two sizes, padding, CRC + tenByteOne := []byte{0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00} + + for name, data := range map[string][]byte{ + "footer magic": patch(valid, -2, 'Y', 'X'), + "header magic": patch(valid, 0, 0xFE), + "index indicator": patch(valid, indexStart, 1), + "index past the start": patch(valid, -8, 0xFF, 0xFF, 0xFF, 0x0F), + "huge record count": xzStreamCount(t, valid, 1<<40), + "zero header size": xzStream(append([]byte{0}, xzBlockHeader(0x00, small)[1:]...)), + "header past block": xzStream(append([]byte{0x40}, xzBlockHeader(0x00, small)[1:]...)), + // Read as filter flags, these sizes would name a one-byte filter + // that isn't LZMA2 and hide the one after them. + "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), large)), + // The same with each size on its own: past the size, the filter + // flags claim 33 bytes of properties, which aren't there. + "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), + "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), + "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, large)), + "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), + "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), + "large second block": xzStream(xzBlockHeader(0x00, small), xzBlockHeader(0x00, large)), + } { + t.Run(name, func(t *testing.T) { + if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } + }) + } +} + +// xzStreamCount rewrites the record count of a one-block stream's index. +func xzStreamCount(t *testing.T, stream []byte, count uint64) []byte { + t.Helper() + indexStart := len(stream) - 12 - 8 + index := append([]byte{0}, xzVarint(count)...) + if len(index) > 8 { + t.Fatal("count too long for the index") + } + return append(append(bytes.Clone(stream[:indexStart]), append(index, make([]byte, 8-len(index))...)...), stream[len(stream)-12:]...) +} diff --git a/internal/deb/xzcheck.go b/internal/deb/xzcheck.go new file mode 100644 index 0000000..5d115bd --- /dev/null +++ b/internal/deb/xzcheck.go @@ -0,0 +1,193 @@ +package deb + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +// maxXZDict bounds the LZMA2 dictionary an index may ask for. The decoder +// allocates the dictionary a block header names, up to 4 GiB, and its own +// DictCap does not lower that (ulikunitz/xz#84). Debian uses 8 MiB. +const maxXZDict = 64 << 20 + +const ( + xzHeaderSize = 12 + xzFooterSize = 12 + lzma2Filter = 0x21 + maxVarint = 9 +) + +var ( + xzHeaderMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} + xzFooterMagic = []byte{'Y', 'Z'} +) + +// checkXZ checks every block of every stream in an .xz file for an LZMA2 +// dictionary larger than maxDict, before the file is decoded. Streams are +// found from the end, through each stream's footer and index, so blocks +// need not be decoded to be found. +func checkXZ(data []byte, maxDict uint64) error { + end := len(data) + streams := 0 + for { + // Stream padding is a multiple of four zero bytes. + for end >= 4 && bytes.Equal(data[end-4:end], []byte{0, 0, 0, 0}) { + end -= 4 + } + if end == 0 { + break + } + start, err := checkXZStream(data[:end], maxDict) + if err != nil { + return fmt.Errorf("%w: xz: %w", ErrMalformedIndex, err) + } + end = start + streams++ + } + if streams == 0 { + return fmt.Errorf("%w: xz: no stream", ErrMalformedIndex) + } + return nil +} + +// checkXZStream checks the stream that ends data and returns its offset. +func checkXZStream(data []byte, maxDict uint64) (int, error) { + if len(data) < xzHeaderSize+xzFooterSize { + return 0, fmt.Errorf("stream too short") + } + footer := data[len(data)-xzFooterSize:] + if !bytes.Equal(footer[10:], xzFooterMagic) { + return 0, fmt.Errorf("no stream footer") + } + indexSize := (int(binary.LittleEndian.Uint32(footer[4:8])) + 1) * 4 + indexStart := len(data) - xzFooterSize - indexSize + if indexStart < 0 { + return 0, fmt.Errorf("index outside the stream") + } + blocks, err := xzIndexBlocks(data[indexStart:len(data)-xzFooterSize], indexStart) + if err != nil { + return 0, err + } + total := 0 + for _, size := range blocks { + total += size + } + start := indexStart - total - xzHeaderSize + if start < 0 || !bytes.Equal(data[start:start+len(xzHeaderMagic)], xzHeaderMagic) { + return 0, fmt.Errorf("no stream header") + } + offset := start + xzHeaderSize + for _, size := range blocks { + if err := checkXZBlockHeader(data[offset:offset+size], maxDict); err != nil { + return 0, err + } + offset += size + } + return start, nil +} + +// xzIndexBlocks reads an xz index and returns the space each block takes, +// its unpadded size rounded up to four bytes. No block is larger than +// limit, the bytes before the index. +func xzIndexBlocks(index []byte, limit int) ([]int, error) { + if len(index) == 0 || index[0] != 0 { + return nil, fmt.Errorf("no index") + } + pos := 1 + count, n := readVarint(index[pos:]) + if n == 0 { + return nil, fmt.Errorf("bad index") + } + pos += n + // Each record is at least two bytes, which bounds count. + if count > uint64(len(index)) { + return nil, fmt.Errorf("bad index") + } + blocks := make([]int, 0, count) + for range count { + unpadded, n := readVarint(index[pos:]) + // Bounded so the sizes below cannot overflow int. + if n == 0 || unpadded > uint64(limit) { + return nil, fmt.Errorf("bad index record") + } + pos += n + if _, n = readVarint(index[pos:]); n == 0 { + return nil, fmt.Errorf("bad index record") + } + pos += n + blocks = append(blocks, int((unpadded+3)&^3)) + } + return blocks, nil +} + +// checkXZBlockHeader reads the filters of the block that starts block. +func checkXZBlockHeader(block []byte, maxDict uint64) error { + if len(block) == 0 || block[0] == 0 { + return fmt.Errorf("no block header") + } + size := (int(block[0]) + 1) * 4 + if size > len(block) { + return fmt.Errorf("block header past the block") + } + header := block[1 : size-4] // without the size byte and the CRC32 + if len(header) == 0 { + return fmt.Errorf("short block header") + } + flags := header[0] + pos := 1 + for _, present := range []byte{0x40, 0x80} { // compressed and uncompressed sizes + if flags&present != 0 { + _, n := readVarint(header[pos:]) + if n == 0 { + return fmt.Errorf("bad block header") + } + pos += n + } + } + for range int(flags&0x03) + 1 { + id, n := readVarint(header[pos:]) + if n == 0 { + return fmt.Errorf("bad filter flags") + } + pos += n + propsSize, n := readVarint(header[pos:]) + if n == 0 || uint64(len(header)-pos-n) < propsSize { + return fmt.Errorf("bad filter flags") + } + pos += n + props := header[pos : pos+int(propsSize)] + pos += int(propsSize) + if id != lzma2Filter { + continue + } + if len(props) != 1 || props[0] > 40 { + return fmt.Errorf("bad LZMA2 properties") + } + if dict := lzma2DictSize(props[0]); dict > maxDict { + return fmt.Errorf("LZMA2 dictionary of %d bytes is larger than %d", dict, maxDict) + } + } + return nil +} + +// lzma2DictSize decodes the LZMA2 dictionary size property, 0 to 40. +func lzma2DictSize(bits byte) uint64 { + if bits == 40 { + return 1<<32 - 1 + } + return uint64(2|bits&1) << (bits/2 + 11) +} + +// readVarint reads an xz multibyte integer, returning 0 bytes read when it +// is malformed or truncated. +func readVarint(b []byte) (uint64, int) { + var v uint64 + for i := 0; i < len(b) && i < maxVarint; i++ { + v |= uint64(b[i]&0x7F) << (7 * i) + if b[i]&0x80 == 0 { + return v, i + 1 + } + } + return 0, 0 +} diff --git a/registries_test.go b/registries_test.go index 81027fa..f041108 100644 --- a/registries_test.go +++ b/registries_test.go @@ -20,7 +20,7 @@ import ( func TestSupportedEcosystems(t *testing.T) { ecosystems := registries.SupportedEcosystems() - expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} + expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} sort.Strings(ecosystems) if len(ecosystems) != len(expected) { @@ -69,6 +69,7 @@ func TestNew(t *testing.T) { {"swift", true}, {"apm", true}, {"apk", true}, + {"deb", true}, {"unknown", true}, } @@ -117,6 +118,7 @@ func TestDefaultURL(t *testing.T) { {"swift", ""}, {"apm", ""}, {"apk", ""}, + {"deb", ""}, } for _, tt := range tests { From d0272625d90dbb48555cb90f627ef38d69d68e1e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 15:52:46 +0100 Subject: [PATCH 2/7] Carry the archive configuration in the deb PURL repository_url held only the archive root, so a PURL the registry built could not be read back: the suites and architectures were missing. It is now the whole configured URL. Keep the resolver test, which needs httptest.NewTLSServer, out of the TinyGo build. --- README.md | 4 ++-- fetch/deb_resolver_test.go | 2 ++ internal/deb/deb.go | 19 +++++++++++++++---- internal/deb/deb_test.go | 29 ++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d09e2e8..c597594 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | -| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm` | +| Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` | ## Direct Registry Usage @@ -203,7 +203,7 @@ import _ "github.com/git-pkgs/registries/all" The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index. -The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. +The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded. GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. diff --git a/fetch/deb_resolver_test.go b/fetch/deb_resolver_test.go index 96e50cf..0d9a678 100644 --- a/fetch/deb_resolver_test.go +++ b/fetch/deb_resolver_test.go @@ -1,3 +1,5 @@ +//go:build !tinygo + package fetch import ( diff --git a/internal/deb/deb.go b/internal/deb/deb.go index 391d26a..d56b013 100644 --- a/internal/deb/deb.go +++ b/internal/deb/deb.go @@ -95,10 +95,21 @@ func New(baseURL string, client *core.Client) *Registry { } r := &Registry{client: client, maxIndexSize: maxIndexSize, maxXZSize: maxXZSize} r.configErr = r.configure(baseURL) - r.urls = &URLs{archiveURL: r.archiveURL} + r.urls = &URLs{repositoryURL: r.repositoryURL()} return r } +// repositoryURL is the URL that configures this Registry, as a PURL's +// repository_url carries it: the archive root with what to read from it. +func (r *Registry) repositoryURL() string { + if r.archiveURL == "" { + return "" + } + return r.archiveURL + "?suites=" + strings.Join(r.suites, ",") + + "&components=" + strings.Join(r.components, ",") + + "&architectures=" + strings.Join(r.architectures, ",") +} + func (r *Registry) configure(baseURL string) error { u, err := url.Parse(baseURL) if err != nil || u.Scheme == "" || u.Host == "" { @@ -513,7 +524,7 @@ func parseMaintainer(value string) core.Maintainer { } type URLs struct { - archiveURL string + repositoryURL string } func (u *URLs) Registry(name, version string) string { @@ -532,8 +543,8 @@ func (u *URLs) Documentation(name, version string) string { func (u *URLs) PURL(name, version string) string { namespace, pkg := splitName(name) var qualifiers map[string]string - if u.archiveURL != "" { - qualifiers = map[string]string{"repository_url": u.archiveURL} + if u.repositoryURL != "" { + qualifiers = map[string]string{"repository_url": u.repositoryURL} } return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() } diff --git a/internal/deb/deb_test.go b/internal/deb/deb_test.go index b3187e0..3ee9fdd 100644 --- a/internal/deb/deb_test.go +++ b/internal/deb/deb_test.go @@ -483,18 +483,41 @@ func TestFetchMaintainers(t *testing.T) { } func TestURLs(t *testing.T) { - u := New("https://deb.debian.org/debian/?suites=bookworm&architectures=amd64", nil).URLs() + u := New("https://deb.debian.org/debian/?architectures=amd64&suites=bookworm,bookworm-updates", nil).URLs() if got := u.Download("curl", "7.88.1-10"); got != "" { t.Errorf("Download = %q, want none without the index", got) } - if got, want := u.PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian"; got != want { + // repository_url is the whole configured URL, so the PURL names what + // to read. + const repo = "?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian%3Fsuites%3Dbookworm%2Cbookworm-updates%26components%3Dmain%26architectures%3Damd64" + if got, want := u.PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10"+repo; got != want { t.Errorf("PURL = %q, want %q", got, want) } - if got, want := u.PURL("debian/curl", "1:9.2p1-2"), "pkg:deb/debian/curl@1:9.2p1-2?repository_url=https:%2F%2Fdeb.debian.org%2Fdebian"; got != want { + if got, want := u.PURL("debian/curl", "1:9.2p1-2"), "pkg:deb/debian/curl@1:9.2p1-2"+repo; got != want { t.Errorf("PURL with epoch = %q, want %q", got, want) } } +func TestPURLWithoutArchive(t *testing.T) { + if got, want := New("", nil).URLs().PURL("debian/curl", "7.88.1-10"), "pkg:deb/debian/curl@7.88.1-10"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } +} + +// A PURL the registry builds leads back to the same archive and indexes. +func TestPURLRoundTrip(t *testing.T) { + _, base := standard(t) + r := newRegistry(base, "suites=bookworm,bookworm-updates&architectures=amd64") + reg, name, version, err := core.NewFromPURL(r.URLs().PURL("debian/curl", "7.88.1-10+deb12u14"), core.DefaultClient()) + if err != nil || name != "debian/curl" || version != "7.88.1-10+deb12u14" { + t.Fatalf("NewFromPURL = %q, %q, %v", name, version, err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 2 || versions[0].Number != version { + t.Errorf("FetchVersions = %v, %v", numbers(versions), err) + } +} + // An index is read once per Registry. func TestIndexReadOnce(t *testing.T) { a, base := standard(t) From 7e7d550bd2416ff0b68be5bbefe99027963d7be3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 15:54:00 +0100 Subject: [PATCH 3/7] Add an RPM repository client Read repodata/repomd.xml, follow its primary entry, check that file against the checksum and size listed there, and decode .zst, .xz, .gz or plain XML as a stream. A zstd frame may ask for a window of at most 128 MiB, and .xz blocks are checked as in the Debian client, whose check moves to internal/xzcheck for both. Artifacts record arch, epoch and the distro given as a query parameter on the repository URL, since repository metadata names none. The version number is version-release, as in the rpm PURL type; versions are ordered by epoch, version and release. --- README.md | 4 + all/all.go | 3 +- fetch/rpm_resolver_test.go | 67 +++ go.mod | 1 + go.sum | 2 + internal/deb/deb.go | 5 +- internal/deb/parse_test.go | 143 ------ internal/rpm/rpm.go | 657 ++++++++++++++++++++++++++ internal/rpm/rpm_test.go | 679 +++++++++++++++++++++++++++ internal/{deb => xzcheck}/xzcheck.go | 70 +-- internal/xzcheck/xzcheck_test.go | 192 ++++++++ registries_test.go | 4 +- 12 files changed, 1649 insertions(+), 178 deletions(-) create mode 100644 fetch/rpm_resolver_test.go create mode 100644 internal/rpm/rpm.go create mode 100644 internal/rpm/rpm_test.go rename internal/{deb => xzcheck}/xzcheck.go (65%) create mode 100644 internal/xzcheck/xzcheck_test.go diff --git a/README.md b/README.md index c597594..d19d9e5 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | GitHub Actions | `pkg:githubactions/actions/checkout@v4` | | Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | | Debian | `pkg:deb/debian/curl@7.88.1-10+deb12u14?arch=amd64&distro=bookworm&repository_url=https://deb.debian.org/debian%3Fsuites%3Dbookworm%2Cbookworm-updates%26architectures%3Damd64` | +| RPM | `pkg:rpm/fedora/curl@8.15.0-4.fc43?arch=x86_64&distro=fedora-43&repository_url=https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os%3Fdistro%3Dfedora-43` | ## Direct Registry Usage @@ -198,6 +199,7 @@ import _ "github.com/git-pkgs/registries/all" | Swift | `swift` | URL required | | Alpine | `apk` | URL required | | Debian | `deb` | URL required | +| RPM | `rpm` | URL required | | GitHub Actions | `githubactions` | https://api.github.com | | APM | None | URL required | @@ -205,6 +207,8 @@ The Alpine URL is one repository architecture directory, such as `https://dl-cdn The Debian URL is an archive root with the suites, components and architectures to read as query parameters, which archives don't use: `https://deb.debian.org/debian?suites=bookworm,bookworm-updates&components=main&architectures=amd64`. `components` defaults to `main`. An archive under another root, such as `https://deb.debian.org/debian-security` with suite `bookworm-security`, is a registry of its own. The client reads `dists//InRelease`, falling back to `Release`, and checks each `Packages` index against its SHA256 there; a suite with neither is read unchecked. It tries `Packages.xz`, `Packages.gz` and `Packages` in that order, of those the Release lists, and decodes them as a stream. The signature on `InRelease` is not checked. Each artifact records `arch` and `distro`, the suite's codename without `-updates` or `-security`, for the `arch` and `distro` PURL qualifiers. Versions keep their epoch (`1:9.2p1-2+deb12u3`) and compare with the `deb` scheme. Source packages (`architectures=source`) are not supported. In a PURL, `repository_url` is this whole URL, query included, so its `?`, `=`, `&` and `,` are percent-encoded. +The RPM URL is one repository, the directory that holds `repodata/`, such as `https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os`. The client doesn't pick a release, variant or architecture from a mirror root, and doesn't follow mirror lists or metalinks. It reads `repodata/repomd.xml`, follows its `primary` entry, checks that file against the checksum and size listed there, and decodes `.zst`, `.xz`, `.gz` or plain XML as a stream. Repository metadata doesn't name its distribution, so the `distro` to record on artifacts is given as a query parameter: `.../x86_64/os?distro=fedora-43`. Each artifact records `arch`, `epoch` and that `distro`, with the package's own checksum as integrity (`sha256-...`). The version number is `version-release`, as in the `rpm` PURL type, where the epoch is a qualifier; versions are ordered by epoch, version and release. Dependencies are the requirements that name packages, leaving out file paths, sonames and other capabilities. In a PURL, `repository_url` is the whole URL, `distro` parameter included, so its `?` and `=` are percent-encoded. + GitHub Actions are git repositories, so `FetchPackage` reads `/repos/{owner}/{repo}` and `FetchVersions` lists tags from the GitHub REST API. Unauthenticated requests are limited to 60 per hour. A subdirectory action such as `pkg:githubactions/github/codeql-action/init@v3` resolves to the `github/codeql-action` repository. `URLs().Download` returns the `/archive/{ref}.tar.gz` source tarball for any tag, branch or commit SHA. GitHub doesn't guarantee these tarballs are byte-stable, so versions carry no `Integrity`. Dependencies and maintainers return an error wrapping `errors.ErrUnsupported`. For GitHub Enterprise Server, pass the `/api/v3` API root as the base URL. ### Swift registries diff --git a/all/all.go b/all/all.go index aad270b..770a157 100644 --- a/all/all.go +++ b/all/all.go @@ -9,7 +9,7 @@ // // // Now all ecosystems are available // ecosystems := registries.SupportedEcosystems() -// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] +// // ["apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform"] package all import ( @@ -41,6 +41,7 @@ import ( _ "github.com/git-pkgs/registries/internal/packagist" _ "github.com/git-pkgs/registries/internal/pub" _ "github.com/git-pkgs/registries/internal/pypi" + _ "github.com/git-pkgs/registries/internal/rpm" _ "github.com/git-pkgs/registries/internal/rubygems" _ "github.com/git-pkgs/registries/internal/swift" _ "github.com/git-pkgs/registries/internal/terraform" diff --git a/fetch/rpm_resolver_test.go b/fetch/rpm_resolver_test.go new file mode 100644 index 0000000..6911bdc --- /dev/null +++ b/fetch/rpm_resolver_test.go @@ -0,0 +1,67 @@ +//go:build !tinygo + +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/rpm" + "github.com/git-pkgs/registries/safehttp" +) + +// rpmRepository serves a repository with one version of curl for two +// architectures, over TLS: the resolver only hands out https download URLs. +func rpmRepository(t *testing.T) (string, *registries.Client) { + t.Helper() + files := map[string]string{ + "/os/repodata/repomd.xml": ``, + "/os/repodata/primary.xml": ` +curlx86_64 + aa +curlaarch64 + bb +`, + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + client := registries.DefaultClient() + client.HTTPClient = server.Client() + return server.URL + "/os", client +} + +func TestResolveRPMByQualifiers(t *testing.T) { + base, client := rpmRepository(t) + reg, err := registries.New("rpm", base+"?distro=fedora-43", client) + if err != nil { + t.Fatal(err) + } + r := NewResolver(WithAddressPolicy(safehttp.Options{AllowLoopback: true})) + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64", "distro": "fedora-43"}}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if want := base + "/Packages/c/curl-8.15.0-4.fc43.aarch64.rpm"; info.URL != want || info.Integrity != "sha256-bb" || info.Size != 4 { + t.Errorf("info = %+v, want %s", info, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "rpm", "fedora/curl", "8.15.0-4.fc43", + ResolveOptions{Qualifiers: map[string]string{"arch": "x86_64", "distro": "fedora-42"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("other distro: err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/go.mod b/go.mod index 9d83d32..9a14bd8 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/spdx v0.3.6 github.com/git-pkgs/vers v0.7.2 + github.com/klauspost/compress v1.20.3 github.com/opencontainers/go-digest v1.0.0 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rubyist/circuitbreaker v2.2.1+incompatible diff --git a/go.sum b/go.sum index 30e87b0..f6e12a2 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= github.com/github/go-spdx/v2 v2.8.0 h1:WKCnF2Ee/x+zwSIEeQdm6+be0H6PiCNLQmSZ5CFmxMI= github.com/github/go-spdx/v2 v2.8.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= diff --git a/internal/deb/deb.go b/internal/deb/deb.go index d56b013..b4998d9 100644 --- a/internal/deb/deb.go +++ b/internal/deb/deb.go @@ -20,6 +20,7 @@ import ( "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzcheck" "github.com/git-pkgs/vers" "github.com/ulikunitz/xz" ) @@ -329,8 +330,8 @@ func (r *Registry) parseIndex(body io.Reader, variant, indexURL string, sum *fil if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { return nil, err } - if err := checkXZ(data, maxXZDict); err != nil { - return nil, err + if err := xzcheck.Check(data, xzcheck.MaxDict); err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) } xr, err := xz.NewReader(bytes.NewReader(data)) if err != nil { diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go index 93ddef2..2b3282a 100644 --- a/internal/deb/parse_test.go +++ b/internal/deb/parse_test.go @@ -150,146 +150,3 @@ func setXZDict(t *testing.T, data []byte, bits byte) []byte { binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) return out } - -func TestCheckXZDictionary(t *testing.T) { - small := xzData(t, bookwormAMD64) - if err := checkXZ(small, maxXZDict); err != nil { - t.Fatalf("checkXZ(8 MiB dictionary): %v", err) - } - // 2 GiB, past the 64 MiB limit. - if err := checkXZ(setXZDict(t, small, 38), maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("2 GiB dictionary: err = %v, want ErrMalformedIndex", err) - } - // A large dictionary in a second stream, after padding, is found too. - two := append(append(bytes.Clone(small), 0, 0, 0, 0), setXZDict(t, small, 38)...) - if err := checkXZ(two, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("second stream: err = %v, want ErrMalformedIndex", err) - } - if err := checkXZ(append(append(bytes.Clone(small), 0, 0, 0, 0), small...), maxXZDict); err != nil { - t.Errorf("two small streams: %v", err) - } - for name, data := range map[string][]byte{ - "empty": nil, - "truncated": small[:len(small)-3], - "garbage": []byte(strings.Repeat("x", 64)), - } { - if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("%s: err = %v, want ErrMalformedIndex", name, err) - } - } -} - -func TestXZDictSize(t *testing.T) { - for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { - if got := lzma2DictSize(bits); got != want { - t.Errorf("lzma2DictSize(%d) = %d, want %d", bits, got, want) - } - } -} - -// xzVarint encodes an xz multibyte integer. -func xzVarint(v uint64) []byte { - var out []byte - for v >= 0x80 { - out = append(out, byte(v)|0x80) - v >>= 7 - } - return append(out, byte(v)) -} - -// xzBlockHeader builds a block header from its flags byte and the bytes -// after it (sizes and filter flags), padded and with a CRC32 slot. -func xzBlockHeader(flags byte, body ...[]byte) []byte { - h := []byte{0, flags} - for _, b := range body { - h = append(h, b...) - } - for (len(h)+4)%4 != 0 { - h = append(h, 0) - } - h = append(h, 0, 0, 0, 0) - h[0] = byte(len(h)/4 - 1) - return h -} - -func lzma2Props(bits byte) []byte { return []byte{lzma2Filter, 0x01, bits} } - -// xzStream builds one stream whose blocks hold only their headers. Nothing -// here is meant to decode; checkXZ reads only the container. -func xzStream(headers ...[]byte) []byte { - out := append(bytes.Clone(xzHeaderMagic), 0, 1, 0, 0, 0, 0) - index := []byte{0} - index = append(index, xzVarint(uint64(len(headers)))...) - for _, h := range headers { - out = append(out, h...) - index = append(index, xzVarint(uint64(len(h)))...) - index = append(index, xzVarint(1)...) - } - for len(index)%4 != 0 { - index = append(index, 0) - } - index = append(index, 0, 0, 0, 0) - out = append(out, index...) - footer := make([]byte, 12) - binary.LittleEndian.PutUint32(footer[4:8], uint32(len(index)/4-1)) - copy(footer[10:], xzFooterMagic) - return append(out, footer...) -} - -func TestCheckXZContainer(t *testing.T) { - small := lzma2Props(22) - large := lzma2Props(38) - valid := xzStream(xzBlockHeader(0x00, small)) - if err := checkXZ(valid, maxXZDict); err != nil { - t.Fatalf("valid synthetic stream: %v", err) - } - - patch := func(data []byte, at int, b ...byte) []byte { - out := bytes.Clone(data) - if at < 0 { - at += len(out) - } - copy(out[at:], b) - return out - } - indexStart := len(valid) - 12 - 8 // one record: indicator, count, two sizes, padding, CRC - tenByteOne := []byte{0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00} - - for name, data := range map[string][]byte{ - "footer magic": patch(valid, -2, 'Y', 'X'), - "header magic": patch(valid, 0, 0xFE), - "index indicator": patch(valid, indexStart, 1), - "index past the start": patch(valid, -8, 0xFF, 0xFF, 0xFF, 0x0F), - "huge record count": xzStreamCount(t, valid, 1<<40), - "zero header size": xzStream(append([]byte{0}, xzBlockHeader(0x00, small)[1:]...)), - "header past block": xzStream(append([]byte{0x40}, xzBlockHeader(0x00, small)[1:]...)), - // Read as filter flags, these sizes would name a one-byte filter - // that isn't LZMA2 and hide the one after them. - "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), large)), - // The same with each size on its own: past the size, the filter - // flags claim 33 bytes of properties, which aren't there. - "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), - "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), - "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, large)), - "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), - "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), - "large second block": xzStream(xzBlockHeader(0x00, small), xzBlockHeader(0x00, large)), - } { - t.Run(name, func(t *testing.T) { - if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("err = %v, want ErrMalformedIndex", err) - } - }) - } -} - -// xzStreamCount rewrites the record count of a one-block stream's index. -func xzStreamCount(t *testing.T, stream []byte, count uint64) []byte { - t.Helper() - indexStart := len(stream) - 12 - 8 - index := append([]byte{0}, xzVarint(count)...) - if len(index) > 8 { - t.Fatal("count too long for the index") - } - return append(append(bytes.Clone(stream[:indexStart]), append(index, make([]byte, 8-len(index))...)...), stream[len(stream)-12:]...) -} diff --git a/internal/rpm/rpm.go b/internal/rpm/rpm.go new file mode 100644 index 0000000..1dac501 --- /dev/null +++ b/internal/rpm/rpm.go @@ -0,0 +1,657 @@ +// Package rpm provides a registry client for RPM repositories, read from +// their repodata. +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha1" //nolint:gosec // repomd.xml checksums of old repositories + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/url" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzcheck" + "github.com/git-pkgs/vers" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +const ( + DefaultURL = "" + ecosystem = "rpm" + + repomdPath = "repodata/repomd.xml" + + // maxPrimarySize bounds the decompressed primary metadata. Fedora 44 + // Everything x86_64 is about 185 MB. + maxPrimarySize = 1 << 30 + // maxXZSize bounds a primary.xml.xz, which is read whole so its blocks + // can be checked before decoding. + maxXZSize = 128 << 20 + // maxZstdWindow bounds the window a zstd frame may ask the decoder to + // allocate. zstd --ultra -22 uses 128 MiB. + maxZstdWindow = 128 << 20 +) + +var ( + // ErrInvalidConfig is returned when the repository URL is missing or + // is not a URL. + ErrInvalidConfig = errors.New("invalid RPM repository configuration") + // ErrChecksumMismatch is returned when the primary metadata doesn't + // match the checksum or size repomd.xml lists. + ErrChecksumMismatch = errors.New("primary metadata does not match its repomd.xml checksum") + // ErrMalformedMetadata is returned for repodata that cannot be parsed. + ErrMalformedMetadata = errors.New("malformed RPM repository metadata") + // ErrUnsupportedCompression is returned when the primary metadata is + // compressed in a format other than zstd, xz or gzip. + ErrUnsupportedCompression = errors.New("unsupported primary metadata compression") +) + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads the primary metadata of one repository: the directory +// that holds repodata/, such as +// https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os. +// The release, variant and architecture are never inferred from a mirror +// root, and mirror lists and metalinks are not followed. +// +// Repository metadata doesn't name its distribution, so the distro to +// record on artifacts, as in the PURL qualifier, is given as a query +// parameter, which repositories don't use: .../x86_64/os?distro=fedora-43. +// +// The metadata is downloaded once per Registry and kept for its lifetime, +// so callers looking up many packages should reuse one Registry. +type Registry struct { + repoURL string + distro string + configErr error + client *core.Client + urls *URLs + + maxPrimarySize int64 + maxXZSize int64 + + mu sync.Mutex + packages map[string][]record +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + r := &Registry{client: client, maxPrimarySize: maxPrimarySize, maxXZSize: maxXZSize} + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + r.configErr = fmt.Errorf("%w: %q is not a repository URL", ErrInvalidConfig, baseURL) + } else { + r.distro = u.Query().Get("distro") + u.RawQuery, u.Fragment = "", "" + r.repoURL = strings.TrimSuffix(u.String(), "/") + } + r.urls = &URLs{repositoryURL: r.repoURL} + // As a PURL's repository_url carries it: the repository with the + // distro it was configured with. + if r.distro != "" { + r.urls.repositoryURL += "?distro=" + r.distro + } + return r +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// record is one package of the primary metadata. +type record struct { + name string + arch string + epoch string + version string // ver-rel + checksumType string + checksum string + summary string + packager string + url string + license string + sourceRPM string + buildTime time.Time + size int64 + installedSize int64 + href string // "" when the file is not under the repository + requires []core.Dependency +} + +// evr is the form versions are compared in. +func (rec record) evr() string { + return rec.epoch + ":" + rec.version +} + +// index reads the repository's metadata, once. Nothing is kept unless it +// was read and checked whole, so a failed read is tried again next time. +func (r *Registry) index(ctx context.Context) (map[string][]record, error) { + if r.configErr != nil { + return nil, r.configErr + } + r.mu.Lock() + defer r.mu.Unlock() + if r.packages != nil { + return r.packages, nil + } + + body, err := r.client.GetBody(ctx, r.repoURL+"/"+repomdPath) + if err != nil { + return nil, err + } + primary, err := parseRepomd(body) + if err != nil { + return nil, err + } + + // Decided before the download, so an unsupported format costs none. + if !supportedPrimary(primary.href) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedCompression, primary.href) + } + + packages := map[string][]record{} + seen := map[string]bool{} + primaryURL := r.repoURL + "/" + primary.href + err = r.client.Stream(ctx, primaryURL, func(body io.Reader) error { + return r.readPrimary(body, primary, primaryURL, func(rec record) { + key := rec.name + "\x00" + rec.evr() + "\x00" + rec.arch + if seen[key] { + return + } + seen[key] = true + packages[rec.name] = append(packages[rec.name], rec) + }) + }) + if err != nil { + return nil, err + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].evr(), recs[j].evr(), ecosystem) > 0 + }) + } + r.packages = packages + return packages, nil +} + +// primaryData is repomd.xml's entry for the primary metadata. +type primaryData struct { + href string + checksumType string // "" when repomd.xml lists no checksum + checksum string + size int64 +} + +func parseRepomd(data []byte) (*primaryData, error) { + var repomd struct { + Data []struct { + Type string `xml:"type,attr"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + Size int64 `xml:"size"` + } `xml:"data"` + } + if err := xml.Unmarshal(data, &repomd); err != nil { + return nil, fmt.Errorf("%w: repomd.xml: %w", ErrMalformedMetadata, err) + } + for _, d := range repomd.Data { + if d.Type != "primary" { + continue + } + if !safeHref(d.Location.Href) { + return nil, fmt.Errorf("%w: repomd.xml: primary location %q", ErrMalformedMetadata, d.Location.Href) + } + return &primaryData{ + href: d.Location.Href, + checksumType: strings.ToLower(strings.TrimSpace(d.Checksum.Type)), + checksum: strings.ToLower(strings.TrimSpace(d.Checksum.Value)), + size: d.Size, + }, nil + } + return nil, fmt.Errorf("%w: repomd.xml lists no primary metadata", ErrMalformedMetadata) +} + +func newHash(checksumType string) (hash.Hash, bool) { + switch checksumType { + case "sha256": + return sha256.New(), true + case "sha512": + return sha512.New(), true + case "sha384": + return sha512.New384(), true + case "sha224": + return sha256.New224(), true + case "sha", "sha1": + return sha1.New(), true //nolint:gosec // as listed by the repository + } + return nil, false +} + +// readPrimary decompresses and parses the primary metadata, checking the +// compressed bytes against the checksum and size repomd.xml lists. +func (r *Registry) readPrimary(body io.Reader, primary *primaryData, primaryURL string, fn func(record)) error { + var hasher hash.Hash + counted := &countingReader{r: body} + if primary.checksumType != "" { + h, ok := newHash(primary.checksumType) + if !ok { + return fmt.Errorf("%w: repomd.xml: checksum type %q", ErrMalformedMetadata, primary.checksumType) + } + hasher = h + counted.r = io.TeeReader(body, hasher) + } + check := func() error { + if hasher == nil { + return nil + } + got := hex.EncodeToString(hasher.Sum(nil)) + if got != primary.checksum || (primary.size > 0 && counted.n != primary.size) { + return fmt.Errorf("%w: %s is %d bytes with %s %s, repomd.xml lists %d bytes with %s", + ErrChecksumMismatch, primaryURL, counted.n, primary.checksumType, got, primary.size, primary.checksum) + } + return nil + } + + var decoded io.Reader + buffered := false + switch { + case strings.HasSuffix(primary.href, ".zst"): + zr, err := zstd.NewReader(counted, zstd.WithDecoderMaxWindow(maxZstdWindow), zstd.WithDecoderConcurrency(1)) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer zr.Close() + decoded = zr + case strings.HasSuffix(primary.href, ".xz"): + data, err := io.ReadAll(io.LimitReader(counted, r.maxXZSize+1)) + if err != nil { + return err + } + if int64(len(data)) > r.maxXZSize { + return fmt.Errorf("%w: %s is larger than %d bytes", ErrMalformedMetadata, primaryURL, r.maxXZSize) + } + // Checked before decoding, so a bad file never reaches the decoder. + if err := check(); err != nil { + return err + } + if err := xzcheck.Check(data, xzcheck.MaxDict); err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + xr, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + decoded, buffered = xr, true + case strings.HasSuffix(primary.href, ".gz"): + gz, err := gzip.NewReader(counted) + if err != nil { + return fmt.Errorf("%w: %w", ErrMalformedMetadata, err) + } + defer func() { _ = gz.Close() }() + decoded = gz + default: + decoded = counted + } + + limited := &io.LimitedReader{R: decoded, N: r.maxPrimarySize + 1} + if err := parsePrimary(limited, fn); err != nil { + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + return err + } + if limited.N == 0 { + return fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedMetadata, primaryURL, r.maxPrimarySize) + } + if buffered { + return nil + } + return check() +} + +// supportedPrimary reports whether href names primary metadata the client +// can decode: zstd, xz, gzip or plain XML. +func supportedPrimary(href string) bool { + for _, ext := range []string{".zst", ".xz", ".gz", ".xml"} { + if strings.HasSuffix(href, ext) { + return true + } + } + return false +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +// xmlPackage is a of primary.xml, with the fields the client uses. +type xmlPackage struct { + Type string `xml:"type,attr"` + Name string `xml:"name"` + Arch string `xml:"arch"` + Version struct { + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"version"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Summary string `xml:"summary"` + Packager string `xml:"packager"` + URL string `xml:"url"` + Time struct { + Build int64 `xml:"build,attr"` + } `xml:"time"` + Size struct { + Package int64 `xml:"package,attr"` + Installed int64 `xml:"installed,attr"` + } `xml:"size"` + Location struct { + Href string `xml:"href,attr"` + Base string `xml:"http://www.w3.org/XML/1998/namespace base,attr"` + } `xml:"location"` + Format struct { + License string `xml:"license"` + SourceRPM string `xml:"sourcerpm"` + Requires []struct { + Name string `xml:"name,attr"` + Flags string `xml:"flags,attr"` + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` + } `xml:"requires>entry"` + } `xml:"format"` +} + +// parsePrimary reads primary.xml as a stream, calling fn for each rpm +// package. +func parsePrimary(r io.Reader, fn func(record)) error { + dec := xml.NewDecoder(r) + for { + tok, err := dec.Token() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg xmlPackage + if err := dec.DecodeElement(&pkg, &start); err != nil { + return fmt.Errorf("%w: primary.xml: %w", ErrMalformedMetadata, err) + } + if pkg.Type != "rpm" { + continue + } + if pkg.Name == "" || pkg.Version.Ver == "" { + return fmt.Errorf("%w: primary.xml: package without a name or version", ErrMalformedMetadata) + } + fn(pkg.record()) + } +} + +func (pkg *xmlPackage) record() record { + rec := record{ + name: pkg.Name, + arch: pkg.Arch, + epoch: pkg.Version.Epoch, + version: evr("", pkg.Version.Ver, pkg.Version.Rel), + checksumType: strings.ToLower(pkg.Checksum.Type), + checksum: strings.ToLower(strings.TrimSpace(pkg.Checksum.Value)), + summary: pkg.Summary, + packager: pkg.Packager, + url: pkg.URL, + license: pkg.Format.License, + sourceRPM: pkg.Format.SourceRPM, + size: pkg.Size.Package, + installedSize: pkg.Size.Installed, + } + if rec.epoch == "" { + rec.epoch = "0" + } + if pkg.Time.Build > 0 { + rec.buildTime = time.Unix(pkg.Time.Build, 0).UTC() + } + // xml:base points the file at another repository, which is not followed. + if pkg.Location.Base == "" && safeHref(pkg.Location.Href) { + rec.href = pkg.Location.Href + } + seen := map[string]bool{} + for _, req := range pkg.Format.Requires { + // File paths, sonames and capabilities such as rpmlib(...) name no + // package. + if req.Name == "" || strings.HasPrefix(req.Name, "/") || strings.Contains(req.Name, "(") || seen[req.Name] { + continue + } + seen[req.Name] = true + rec.requires = append(rec.requires, core.Dependency{ + Name: req.Name, + Requirements: requirement(req.Flags, req.Epoch, req.Ver, req.Rel), + Scope: core.Runtime, + }) + } + return rec +} + +// evr joins an epoch, version and release, leaving out an epoch of 0 and +// an empty release. +func evr(epoch, ver, rel string) string { + s := ver + if epoch != "" && epoch != "0" { + s = epoch + ":" + s + } + if rel != "" { + s += "-" + rel + } + return s +} + +// requirement renders a requires entry's version constraint. +func requirement(flags, epoch, ver, rel string) string { + ops := map[string]string{"EQ": "=", "GE": ">=", "LE": "<=", "GT": ">", "LT": "<"} + op, ok := ops[flags] + if !ok || ver == "" { + return "" + } + return op + " " + evr(epoch, ver, rel) +} + +// safeHref reports whether a location stays under the repository root: +// relative, without dot segments, escapes or URL syntax. +func safeHref(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.ContainsAny(p, `\?#%:`) { + return false + } + for _, segment := range strings.Split(p, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + +// splitName strips a PURL vendor namespace: "fedora/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + packages, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.summary, + Homepage: latest.url, + Licenses: latest.license, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "epoch": latest.epoch, + "source_rpm": latest.sourceRPM, + }, + }, nil +} + +// FetchVersions returns one Version per epoch, version and release, newest +// first, with an artifact for each architecture. The version number is +// version-release, as in the rpm PURL type, where the epoch is a qualifier. +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + var versions []core.Version + last := "" + for _, rec := range recs { + if len(versions) == 0 || last != rec.evr() { + last = rec.evr() + versions = append(versions, core.Version{ + Number: rec.version, + PublishedAt: rec.buildTime, + Licenses: rec.license, + Metadata: map[string]any{ + "epoch": rec.epoch, + "source_rpm": rec.sourceRPM, + "packager": rec.packager, + "installed_size": rec.installedSize, + }, + }) + } + if artifact, ok := r.artifact(rec); ok { + v := &versions[len(versions)-1] + v.Artifacts = append(v.Artifacts, artifact) + } + } + return versions, nil +} + +func (r *Registry) artifact(rec record) (core.Artifact, bool) { + if rec.href == "" { + return core.Artifact{}, false + } + artifact := core.Artifact{ + URL: r.repoURL + "/" + rec.href, + Filename: path.Base(rec.href), + Size: rec.size, + Metadata: map[string]any{"arch": rec.arch, "epoch": rec.epoch}, + } + if r.distro != "" { + artifact.Metadata["distro"] = r.distro + } + if rec.checksum != "" && rec.checksumType != "" { + artifact.Integrity = rec.checksumType + "-" + rec.checksum + } + return artifact, true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version == version { + return rec.requires, nil + } + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].packager == "" { + return nil, nil + } + return []core.Maintainer{{Name: recs[0].packager}}, nil +} + +type URLs struct { + repositoryURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns "": a package's file is only known from the metadata. +func (u *URLs) Download(name, version string) string { + return "" +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + var qualifiers map[string]string + if u.repositoryURL != "" { + qualifiers = map[string]string{"repository_url": u.repositoryURL} + } + return purl.New(ecosystem, namespace, pkg, version, qualifiers).String() +} diff --git a/internal/rpm/rpm_test.go b/internal/rpm/rpm_test.go new file mode 100644 index 0000000..87c6e15 --- /dev/null +++ b/internal/rpm/rpm_test.go @@ -0,0 +1,679 @@ +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/internal/core" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// Trimmed from Fedora 43 updates' primary.xml, with a second architecture, +// an older build and a package with an epoch added. +const primaryXML = ` + + + curl + x86_64 + + 235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a + A utility for getting files from remote servers + curl is a command line tool +for transferring data. + Fedora Project + https://curl.se/ + + + curl + i686 + + 335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b + + + + curl + x86_64 + + AB5b92fa + + + + bind + x86_64 + + 435b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5c + + + + bind + x86_64 + + 535b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5d + + + + tzdata + noarch + + 635b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5e + + + +` + +func compress(t *testing.T, ext, s string) []byte { + t.Helper() + var buf bytes.Buffer + var w interface { + Write([]byte) (int, error) + Close() error + } + var err error + switch ext { + case ".zst": + w, err = zstd.NewWriter(&buf) + case ".xz": + w, err = xz.NewWriter(&buf) + case ".gz": + w = gzip.NewWriter(&buf) + default: + return []byte(s) + } + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// repo is a fake RPM repository under /os/. +type repo struct { + mu sync.Mutex + files map[string][]byte + requests []string +} + +// newRepo serves primary, compressed as ext ("" for plain .xml), and a +// repomd.xml listing it with its sha256 and size. +func newRepo(t *testing.T, ext, primary string) (*repo, string) { + t.Helper() + r := &repo{files: map[string][]byte{}} + data := compress(t, ext, primary) + sum := sha256.Sum256(data) + href := "repodata/" + hex.EncodeToString(sum[:]) + "-primary.xml" + ext + r.files[href] = data + r.setRepomd("sha256", hex.EncodeToString(sum[:]), href, len(data)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + path := strings.TrimPrefix(req.URL.Path, "/os/") + r.mu.Lock() + r.requests = append(r.requests, path) + body, ok := r.files[path] + r.mu.Unlock() + if !ok { + http.NotFound(w, req) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return r, srv.URL + "/os" +} + +func (r *repo) setRepomd(sumType, sum, href string, size int) { + r.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(` + + 1794529563 + + 00 + + 1 + + + %s + + %d + 1 + + +`, sumType, sum, href, size)) +} + +func (r *repo) primaryHref() string { + for path := range r.files { + if strings.Contains(path, "-primary.xml") { + return path + } + } + return "" +} + +func numbers(versions []core.Version) []string { + var out []string + for _, v := range versions { + out = append(out, v.Number) + } + return out +} + +func TestFetchVersions(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + versions, err := r.FetchVersions(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"8.15.0-10.fc43", "8.15.0-4.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (newest first, by rpm version)", numbers(versions), want) + } + older := versions[1] + if !older.PublishedAt.Equal(time.Unix(1771515699, 0)) { + t.Errorf("published = %v, want the build time", older.PublishedAt) + } + if older.Licenses != "curl" || older.Metadata["epoch"] != "0" || older.Metadata["source_rpm"] != "curl-8.15.0-4.fc43.src.rpm" { + t.Errorf("version = %+v", older) + } + want := []core.Artifact{ + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.x86_64.rpm", + Filename: "curl-8.15.0-4.fc43.x86_64.rpm", + Integrity: "sha256-235b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5a", + Size: 234567, + Metadata: map[string]any{"arch": "x86_64", "distro": "fedora-43", "epoch": "0"}, + }, + { + URL: base + "/Packages/c/curl-8.15.0-4.fc43.i686.rpm", + Filename: "curl-8.15.0-4.fc43.i686.rpm", + Integrity: "sha256-335b92fa8916229313a33eef861f4fd6453555164f5c3d3dae199a8853f6db5b", + Metadata: map[string]any{"arch": "i686", "distro": "fedora-43", "epoch": "0"}, + }, + } + if !reflect.DeepEqual(older.Artifacts, want) { + t.Errorf("artifacts =\n %+v\nwant\n %+v", older.Artifacts, want) + } + // The checksum's own type names the integrity, lowercased. + if got := versions[0].Artifacts[0].Integrity; got != "sha512-ab5b92fa" { + t.Errorf("sha512 integrity = %q", got) + } + }) + } +} + +// The epoch orders versions but is not part of the version number, as in +// the rpm PURL type, where it is a qualifier. +func TestEpoch(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "bind") + if err != nil { + t.Fatalf("FetchVersions: %v", err) + } + if want := []string{"9.18.41-1.fc43", "9.20.1-1.fc43"}; !reflect.DeepEqual(numbers(versions), want) { + t.Fatalf("versions = %v, want %v (epoch 32 before epoch 31)", numbers(versions), want) + } + if e := versions[0].Metadata["epoch"]; e != "32" { + t.Errorf("epoch = %v", e) + } + if e := versions[0].Artifacts[0].Metadata["epoch"]; e != "32" { + t.Errorf("artifact epoch = %v", e) + } +} + +// Without a distro in the URL none is recorded, and noarch packages record +// their own architecture. +func TestNoarchAndNoDistro(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + versions, err := New(base+"/", core.DefaultClient()).FetchVersions(context.Background(), "tzdata") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 1 { + t.Fatalf("FetchVersions = %+v, %v", versions, err) + } + if got, want := versions[0].Artifacts[0].Metadata, (map[string]any{"arch": "noarch", "epoch": "0"}); !reflect.DeepEqual(got, want) { + t.Errorf("metadata = %v, want %v", got, want) + } + if got, want := versions[0].Artifacts[0].URL, base+"/Packages/t/tzdata-2026a-1.fc43.noarch.rpm"; got != want { + t.Errorf("URL = %q, want %q", got, want) + } +} + +// Primary metadata is streamed, so the client's limit on buffered +// responses doesn't apply to it. +func TestPrimaryLargerThanResponseCap(t *testing.T) { + _, base := newRepo(t, "", primaryXML) + client := core.NewClient(core.WithMaxRetries(0)) + client.MaxResponseSize = 1024 // repomd.xml fits, primary.xml doesn't + if _, err := New(base, client).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestChecksumMismatch(t *testing.T) { + other := strings.Replace(primaryXML, "8.15.0", "8.15.1", 1) + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + rp, base := newRepo(t, ext, primaryXML) + href := rp.primaryHref() + good := rp.files[href] + rp.files[href] = compress(t, ext, other) + + r := New(base, core.DefaultClient()) + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("err = %v, want ErrChecksumMismatch", err) + } + // Nothing was kept: the next lookup reads the repository again. + rp.files[href] = good + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("after the repository was fixed: %v", err) + } + }) + } +} + +// The right hash with another size is rejected too. +func TestChecksumSizeMismatch(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + sum := sha256.Sum256([]byte(primaryXML)) + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), rp.primaryHref(), len(primaryXML)+1) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("err = %v, want ErrChecksumMismatch", err) + } +} + +func TestChecksumTypes(t *testing.T) { + sum512 := sha512.Sum512([]byte(primaryXML)) + for name, tt := range map[string]struct { + sumType, sum string + wantErr error + }{ + "sha512": {"sha512", hex.EncodeToString(sum512[:]), nil}, + "sha512 upper": {"SHA512", strings.ToUpper(hex.EncodeToString(sum512[:])), nil}, + "sha512 wrong": {"sha512", "00", ErrChecksumMismatch}, + "unknown type": {"md5", "00", ErrMalformedMetadata}, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd(tt.sumType, tt.sum, rp.primaryHref(), len(primaryXML)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, tt.wantErr) { + t.Errorf("err = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestInvalidConfiguration(t *testing.T) { + for name, url := range map[string]string{"no url": "", "no host": "/pub/fedora"} { + t.Run(name, func(t *testing.T) { + _, err := New(url, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("err = %v, want ErrInvalidConfig", err) + } + }) + } +} + +func TestMalformedRepomd(t *testing.T) { + for name, repomd := range map[string]string{ + "not xml": "", + "no primary": ``, + "href leaves the repository": ``, + "absolute href": ``, + } { + t.Run(name, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(repomd) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } +} + +func TestUnsupportedCompression(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + rp.setRepomd("sha256", "00", "repodata/primary.xml.zck", 1) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrUnsupportedCompression) { + t.Errorf("err = %v, want ErrUnsupportedCompression", err) + } +} + +// A repomd.xml that lists a checksum but no size checks the checksum alone. +func TestChecksumWithoutSize(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + href := rp.primaryHref() + sum := sha256.Sum256(rp.files[href]) + repomd := `%s` + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, hex.EncodeToString(sum[:]))) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } + rp.files["repodata/repomd.xml"] = []byte(fmt.Sprintf(repomd, "00")) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrChecksumMismatch) { + t.Errorf("wrong checksum: err = %v, want ErrChecksumMismatch", err) + } +} + +// A repomd.xml without a checksum for primary leaves it unchecked. +func TestNoChecksum(t *testing.T) { + rp, base := newRepo(t, ".gz", primaryXML) + rp.files["repodata/repomd.xml"] = []byte(``) + if _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} + +func TestMalformedPrimary(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz"} { + t.Run("not"+ext, func(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + garbage := []byte(strings.Repeat("not compressed data ", 10)) + sum := sha256.Sum256(garbage) + rp.files["repodata/primary.xml"+ext] = garbage + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml"+ext, len(garbage)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + } + t.Run("truncated xml", func(t *testing.T) { + _, base := newRepo(t, "", primaryXML[:len(primaryXML)/2]) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) + t.Run("package without a name", func(t *testing.T) { + _, base := newRepo(t, "", `x86_64`) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } + }) +} + +func TestDecompressedLimit(t *testing.T) { + for _, ext := range []string{".zst", ".xz", ".gz", ""} { + t.Run("primary.xml"+ext, func(t *testing.T) { + _, base := newRepo(t, ext, primaryXML) + r := New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + // Cut mid-document, the limit is still what the error names, + // not the XML it broke. + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) / 2 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) || !strings.Contains(err.Error(), "decompresses to more than") { + t.Errorf("far over the limit: err = %v, want ErrMalformedMetadata naming the limit", err) + } + r = New(base, core.DefaultClient()) + r.maxPrimarySize = int64(len(primaryXML)) + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } + }) + } +} + +func TestXZSizeLimit(t *testing.T) { + rp, base := newRepo(t, ".xz", primaryXML) + size := int64(len(rp.files[rp.primaryHref()])) + r := New(base, core.DefaultClient()) + r.maxXZSize = size - 1 + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("over the limit: err = %v, want ErrMalformedMetadata", err) + } + r = New(base, core.DefaultClient()) + r.maxXZSize = size + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("at the limit: %v", err) + } +} + +// A zstd frame asking for a 256 MiB window, which the decoder would allow +// by default, is refused rather than allocated. +func TestZstdWindowRefused(t *testing.T) { + rp, base := newRepo(t, "", primaryXML) + // Magic, a frame header with no content size, a window descriptor of + // 2^28 bytes, and one last raw block holding a document. + frame := append([]byte{0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x90, 0x61, 0x00, 0x00}, "\n"...) + sum := sha256.Sum256(frame) + rp.files["repodata/primary.xml.zst"] = frame + rp.setRepomd("sha256", hex.EncodeToString(sum[:]), "repodata/primary.xml.zst", len(frame)) + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedMetadata) { + t.Errorf("err = %v, want ErrMalformedMetadata", err) + } +} + +func TestNotFound(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "missing"); !errors.As(err, ¬Found) { + t.Errorf("missing package: err = %v, want NotFoundError", err) + } + if _, err := r.FetchDependencies(context.Background(), "curl", "1.0-1"); !errors.As(err, ¬Found) || notFound.Version != "1.0-1" { + t.Errorf("missing version: err = %v, want NotFoundError for 1.0-1", err) + } +} + +// A package whose file is outside the repository, by its href or by an +// xml:base pointing elsewhere, gets no artifact. Other package types are +// not read. +func TestUnsafeLocations(t *testing.T) { + primary := ` +upx86_64 +basedx86_64 +nonex86_64 +barex86_64 +barex86_64 +otherx86_64 +` + _, base := newRepo(t, "", primary) + r := New(base, core.DefaultClient()) + for _, name := range []string{"up", "based", "none"} { + versions, err := r.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 0 { + t.Errorf("%s: versions = %+v, %v; want one without artifacts", name, versions, err) + } + } + // No epoch means 0, no checksum means no integrity, and a package + // listed twice is recorded once. + bare, err := r.FetchVersions(context.Background(), "bare") + if err != nil || len(bare) != 1 || len(bare[0].Artifacts) != 1 { + t.Fatalf("bare: versions = %+v, %v; want one with one artifact", bare, err) + } + if a := bare[0].Artifacts[0]; a.Integrity != "" || a.Metadata["epoch"] != "0" || a.Filename != "bare-1-1.x86_64.rpm" || !strings.HasSuffix(a.URL, "/Packages/bare-1-1.x86_64.rpm") { + t.Errorf("bare artifact = %+v", a) + } + var notFound *core.NotFoundError + if _, err := r.FetchVersions(context.Background(), "other"); !errors.As(err, ¬Found) { + t.Errorf("non-rpm package: err = %v, want NotFoundError", err) + } + if got, err := r.FetchMaintainers(context.Background(), "up"); err != nil || got != nil { + t.Errorf("maintainers = %+v, %v; want none without a packager", got, err) + } +} + +func TestFetchPackage(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + pkg, err := r.FetchPackage(context.Background(), "fedora/curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + // The newest build has no summary of its own in this fixture. + if pkg.Name != "curl" || pkg.Namespace != "fedora" || pkg.LatestVersion != "8.15.0-10.fc43" { + t.Errorf("package = %+v", pkg) + } + pkg, err = New(base, core.DefaultClient()).FetchPackage(context.Background(), "bind") + if err != nil || pkg.LatestVersion != "9.18.41-1.fc43" || pkg.Metadata["epoch"] != "32" { + t.Errorf("bind = %+v, %v", pkg, err) + } +} + +func TestFetchPackageDetails(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + pkg, err := New(base, core.DefaultClient()).FetchPackage(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchPackage: %v", err) + } + if pkg.Description != "A utility for getting files from remote servers" || pkg.Homepage != "https://curl.se/" || pkg.Licenses != "curl" { + t.Errorf("package = %+v", pkg) + } +} + +// Dependencies are the requirements that name packages: file paths, +// sonames and other capabilities are left out. +func TestFetchDependencies(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + deps, err := New(base, core.DefaultClient()).FetchDependencies(context.Background(), "curl", "8.15.0-4.fc43") + if err != nil { + t.Fatalf("FetchDependencies: %v", err) + } + want := []core.Dependency{ + {Name: "libcurl", Requirements: ">= 8.15.0-4.fc43", Scope: core.Runtime}, + {Name: "openssl-libs", Requirements: "= 1:3.5.4", Scope: core.Runtime}, + {Name: "zlib", Scope: core.Runtime}, + } + if !reflect.DeepEqual(deps, want) { + t.Errorf("deps =\n %+v\nwant\n %+v", deps, want) + } +} + +func TestRequirement(t *testing.T) { + for _, tt := range []struct { + flags, epoch, ver, rel, want string + }{ + {"EQ", "0", "1.0", "1", "= 1.0-1"}, + {"GE", "", "1.0", "", ">= 1.0"}, + {"LE", "2", "1.0", "1", "<= 2:1.0-1"}, + {"GT", "0", "1.0", "", "> 1.0"}, + {"LT", "0", "1.0", "", "< 1.0"}, + {"", "0", "", "", ""}, + {"EQ", "0", "", "", ""}, + {"XX", "0", "1.0", "", ""}, + } { + if got := requirement(tt.flags, tt.epoch, tt.ver, tt.rel); got != tt.want { + t.Errorf("requirement(%q, %q, %q, %q) = %q, want %q", tt.flags, tt.epoch, tt.ver, tt.rel, got, tt.want) + } + } +} + +func TestFetchMaintainers(t *testing.T) { + single := primaryXML[:strings.Index(primaryXML, "")+len("")] + "\n" + _, base := newRepo(t, ".gz", single) + got, err := New(base, core.DefaultClient()).FetchMaintainers(context.Background(), "curl") + if err != nil || !reflect.DeepEqual(got, []core.Maintainer{{Name: "Fedora Project"}}) { + t.Errorf("maintainers = %+v, %v", got, err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os/?distro=fedora-43", nil).URLs() + if got := u.Download("curl", "8.15.0-4.fc43"); got != "" { + t.Errorf("Download = %q, want none without the metadata", got) + } + // repository_url is the whole configured URL, distro included. + const repo = "?repository_url=https:%2F%2Fdl.fedoraproject.org%2Fpub%2Ffedora%2Flinux%2Freleases%2F43%2FEverything%2Fx86_64%2Fos" + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo+"%3Fdistro%3Dfedora-43"; got != want { + t.Errorf("PURL = %q, want %q", got, want) + } + u = New("https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", nil).URLs() + if got, want := u.PURL("fedora/curl", "8.15.0-4.fc43"), "pkg:rpm/fedora/curl@8.15.0-4.fc43"+repo; got != want { + t.Errorf("PURL without a distro = %q, want %q", got, want) + } + if got, want := New("", nil).URLs().PURL("fedora/curl", "1-1"), "pkg:rpm/fedora/curl@1-1"; got != want { + t.Errorf("PURL without a repository = %q, want %q", got, want) + } +} + +// A PURL the registry builds leads back to the same repository and distro. +func TestPURLRoundTrip(t *testing.T) { + _, base := newRepo(t, ".zst", primaryXML) + r := New(base+"?distro=fedora-43", core.DefaultClient()) + reg, name, version, err := core.NewFromPURL(r.URLs().PURL("fedora/curl", "8.15.0-4.fc43"), core.DefaultClient()) + if err != nil || name != "fedora/curl" || version != "8.15.0-4.fc43" { + t.Fatalf("NewFromPURL = %q, %q, %v", name, version, err) + } + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil || len(versions) != 2 || versions[1].Artifacts[0].Metadata["distro"] != "fedora-43" { + t.Errorf("FetchVersions = %+v, %v", versions, err) + } +} + +// The metadata is read once per Registry. +func TestMetadataReadOnce(t *testing.T) { + rp, base := newRepo(t, ".zst", primaryXML) + r := New(base, core.DefaultClient()) + for range 3 { + if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + } + if len(rp.requests) != 2 { + t.Errorf("requests = %v, want repomd.xml and primary once each", rp.requests) + } +} + +func TestSafeHref(t *testing.T) { + for in, ok := range map[string]bool{ + "Packages/c/curl-8.15.0-4.fc43.x86_64.rpm": true, + "": false, + "/etc/passwd": false, + "Packages/../../etc/passwd": false, + "Packages/x.rpm?query": false, + "Packages/x.rpm#frag": false, + `Packages\x.rpm`: false, + "Packages/%2e%2e/x.rpm": false, + "http://evil.example/x.rpm": false, + } { + if got := safeHref(in); got != ok { + t.Errorf("safeHref(%q) = %v, want %v", in, got, ok) + } + } +} diff --git a/internal/deb/xzcheck.go b/internal/xzcheck/xzcheck.go similarity index 65% rename from internal/deb/xzcheck.go rename to internal/xzcheck/xzcheck.go index 5d115bd..0ee5964 100644 --- a/internal/deb/xzcheck.go +++ b/internal/xzcheck/xzcheck.go @@ -1,33 +1,41 @@ -package deb +// Package xzcheck bounds the memory an .xz file can make its decoder +// allocate, by reading the file's container before it is decoded. +package xzcheck import ( "bytes" "encoding/binary" + "errors" "fmt" ) -// maxXZDict bounds the LZMA2 dictionary an index may ask for. The decoder -// allocates the dictionary a block header names, up to 4 GiB, and its own -// DictCap does not lower that (ulikunitz/xz#84). Debian uses 8 MiB. -const maxXZDict = 64 << 20 +// ErrMalformed is returned for a file that is not a well-formed .xz +// container, or whose blocks ask for too large a dictionary. +var ErrMalformed = errors.New("malformed xz file") + +// MaxDict is the LZMA2 dictionary size callers allow by default. The +// ulikunitz/xz decoder allocates the dictionary a block header names, up to +// 4 GiB, and its own DictCap does not lower that (ulikunitz/xz#84). xz -9 +// uses 64 MiB; Debian's indexes use 8 MiB. +const MaxDict = 64 << 20 const ( - xzHeaderSize = 12 - xzFooterSize = 12 - lzma2Filter = 0x21 - maxVarint = 9 + headerSize = 12 + footerSize = 12 + lzma2Filter = 0x21 + maxVarint = 9 ) var ( - xzHeaderMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} - xzFooterMagic = []byte{'Y', 'Z'} + headerMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} + footerMagic = []byte{'Y', 'Z'} ) -// checkXZ checks every block of every stream in an .xz file for an LZMA2 +// Check checks every block of every stream in an .xz file for an LZMA2 // dictionary larger than maxDict, before the file is decoded. Streams are // found from the end, through each stream's footer and index, so blocks // need not be decoded to be found. -func checkXZ(data []byte, maxDict uint64) error { +func Check(data []byte, maxDict uint64) error { end := len(data) streams := 0 for { @@ -38,34 +46,34 @@ func checkXZ(data []byte, maxDict uint64) error { if end == 0 { break } - start, err := checkXZStream(data[:end], maxDict) + start, err := checkStream(data[:end], maxDict) if err != nil { - return fmt.Errorf("%w: xz: %w", ErrMalformedIndex, err) + return fmt.Errorf("%w: %w", ErrMalformed, err) } end = start streams++ } if streams == 0 { - return fmt.Errorf("%w: xz: no stream", ErrMalformedIndex) + return fmt.Errorf("%w: no stream", ErrMalformed) } return nil } -// checkXZStream checks the stream that ends data and returns its offset. -func checkXZStream(data []byte, maxDict uint64) (int, error) { - if len(data) < xzHeaderSize+xzFooterSize { +// checkStream checks the stream that ends data and returns its offset. +func checkStream(data []byte, maxDict uint64) (int, error) { + if len(data) < headerSize+footerSize { return 0, fmt.Errorf("stream too short") } - footer := data[len(data)-xzFooterSize:] - if !bytes.Equal(footer[10:], xzFooterMagic) { + footer := data[len(data)-footerSize:] + if !bytes.Equal(footer[10:], footerMagic) { return 0, fmt.Errorf("no stream footer") } indexSize := (int(binary.LittleEndian.Uint32(footer[4:8])) + 1) * 4 - indexStart := len(data) - xzFooterSize - indexSize + indexStart := len(data) - footerSize - indexSize if indexStart < 0 { return 0, fmt.Errorf("index outside the stream") } - blocks, err := xzIndexBlocks(data[indexStart:len(data)-xzFooterSize], indexStart) + blocks, err := indexBlocks(data[indexStart:len(data)-footerSize], indexStart) if err != nil { return 0, err } @@ -73,13 +81,13 @@ func checkXZStream(data []byte, maxDict uint64) (int, error) { for _, size := range blocks { total += size } - start := indexStart - total - xzHeaderSize - if start < 0 || !bytes.Equal(data[start:start+len(xzHeaderMagic)], xzHeaderMagic) { + start := indexStart - total - headerSize + if start < 0 || !bytes.Equal(data[start:start+len(headerMagic)], headerMagic) { return 0, fmt.Errorf("no stream header") } - offset := start + xzHeaderSize + offset := start + headerSize for _, size := range blocks { - if err := checkXZBlockHeader(data[offset:offset+size], maxDict); err != nil { + if err := checkBlockHeader(data[offset:offset+size], maxDict); err != nil { return 0, err } offset += size @@ -87,10 +95,10 @@ func checkXZStream(data []byte, maxDict uint64) (int, error) { return start, nil } -// xzIndexBlocks reads an xz index and returns the space each block takes, +// indexBlocks reads an xz index and returns the space each block takes, // its unpadded size rounded up to four bytes. No block is larger than // limit, the bytes before the index. -func xzIndexBlocks(index []byte, limit int) ([]int, error) { +func indexBlocks(index []byte, limit int) ([]int, error) { if len(index) == 0 || index[0] != 0 { return nil, fmt.Errorf("no index") } @@ -121,8 +129,8 @@ func xzIndexBlocks(index []byte, limit int) ([]int, error) { return blocks, nil } -// checkXZBlockHeader reads the filters of the block that starts block. -func checkXZBlockHeader(block []byte, maxDict uint64) error { +// checkBlockHeader reads the filters of the block that starts block. +func checkBlockHeader(block []byte, maxDict uint64) error { if len(block) == 0 || block[0] == 0 { return fmt.Errorf("no block header") } diff --git a/internal/xzcheck/xzcheck_test.go b/internal/xzcheck/xzcheck_test.go new file mode 100644 index 0000000..aed1ab8 --- /dev/null +++ b/internal/xzcheck/xzcheck_test.go @@ -0,0 +1,192 @@ +package xzcheck + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "strings" + "testing" + + "github.com/ulikunitz/xz" +) + +func xzData(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w, err := xz.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// setXZDict rewrites the dictionary size of every LZMA2 block header in +// data, fixing the header CRC, so a small file claims a large dictionary. +func setXZDict(t *testing.T, data []byte, bits byte) []byte { + t.Helper() + out := bytes.Clone(data) + // A file written by xzData holds one stream with one block, whose + // header starts after the 12-byte stream header. + const start = 12 + size := (int(out[start]) + 1) * 4 + header := out[start : start+size] + i := bytes.Index(header, []byte{0x21, 0x01}) + if i < 0 { + t.Fatal("no LZMA2 filter in the block header") + } + header[i+2] = bits + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} + +const sample = "Package: curl\nVersion: 7.88.1-10\n" + +func TestCheckXZDictionary(t *testing.T) { + small := xzData(t, sample) + if err := Check(small, MaxDict); err != nil { + t.Fatalf("Check(8 MiB dictionary): %v", err) + } + // 2 GiB, past the 64 MiB limit. + if err := Check(setXZDict(t, small, 38), MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("2 GiB dictionary: err = %v, want ErrMalformed", err) + } + // A large dictionary in a second stream, after padding, is found too. + two := append(append(bytes.Clone(small), 0, 0, 0, 0), setXZDict(t, small, 38)...) + if err := Check(two, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("second stream: err = %v, want ErrMalformed", err) + } + if err := Check(append(append(bytes.Clone(small), 0, 0, 0, 0), small...), MaxDict); err != nil { + t.Errorf("two small streams: %v", err) + } + for name, data := range map[string][]byte{ + "empty": nil, + "truncated": small[:len(small)-3], + "garbage": []byte(strings.Repeat("x", 64)), + } { + if err := Check(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("%s: err = %v, want ErrMalformed", name, err) + } + } +} + +func TestXZDictSize(t *testing.T) { + for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { + if got := lzma2DictSize(bits); got != want { + t.Errorf("lzma2DictSize(%d) = %d, want %d", bits, got, want) + } + } +} + +// xzVarint encodes an xz multibyte integer. +func xzVarint(v uint64) []byte { + var out []byte + for v >= 0x80 { + out = append(out, byte(v)|0x80) + v >>= 7 + } + return append(out, byte(v)) +} + +// xzBlockHeader builds a block header from its flags byte and the bytes +// after it (sizes and filter flags), padded and with a CRC32 slot. +func xzBlockHeader(flags byte, body ...[]byte) []byte { + h := []byte{0, flags} + for _, b := range body { + h = append(h, b...) + } + for (len(h)+4)%4 != 0 { + h = append(h, 0) + } + h = append(h, 0, 0, 0, 0) + h[0] = byte(len(h)/4 - 1) + return h +} + +func lzma2Props(bits byte) []byte { return []byte{lzma2Filter, 0x01, bits} } + +// xzStream builds one stream whose blocks hold only their headers. Nothing +// here is meant to decode; checkXZ reads only the container. +func xzStream(headers ...[]byte) []byte { + out := append(bytes.Clone(headerMagic), 0, 1, 0, 0, 0, 0) + index := []byte{0} + index = append(index, xzVarint(uint64(len(headers)))...) + for _, h := range headers { + out = append(out, h...) + index = append(index, xzVarint(uint64(len(h)))...) + index = append(index, xzVarint(1)...) + } + for len(index)%4 != 0 { + index = append(index, 0) + } + index = append(index, 0, 0, 0, 0) + out = append(out, index...) + footer := make([]byte, 12) + binary.LittleEndian.PutUint32(footer[4:8], uint32(len(index)/4-1)) + copy(footer[10:], footerMagic) + return append(out, footer...) +} + +func TestCheckXZContainer(t *testing.T) { + small := lzma2Props(22) + large := lzma2Props(38) + valid := xzStream(xzBlockHeader(0x00, small)) + if err := Check(valid, MaxDict); err != nil { + t.Fatalf("valid synthetic stream: %v", err) + } + + patch := func(data []byte, at int, b ...byte) []byte { + out := bytes.Clone(data) + if at < 0 { + at += len(out) + } + copy(out[at:], b) + return out + } + indexStart := len(valid) - 12 - 8 // one record: indicator, count, two sizes, padding, CRC + tenByteOne := []byte{0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00} + + for name, data := range map[string][]byte{ + "footer magic": patch(valid, -2, 'Y', 'X'), + "header magic": patch(valid, 0, 0xFE), + "index indicator": patch(valid, indexStart, 1), + "index past the start": patch(valid, -8, 0xFF, 0xFF, 0xFF, 0x0F), + "huge record count": xzStreamCount(t, valid, 1<<40), + "zero header size": xzStream(append([]byte{0}, xzBlockHeader(0x00, small)[1:]...)), + "header past block": xzStream(append([]byte{0x40}, xzBlockHeader(0x00, small)[1:]...)), + // Read as filter flags, these sizes would name a one-byte filter + // that isn't LZMA2 and hide the one after them. + "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), large)), + // The same with each size on its own: past the size, the filter + // flags claim 33 bytes of properties, which aren't there. + "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), + "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), + "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, large)), + "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), + "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), + "large second block": xzStream(xzBlockHeader(0x00, small), xzBlockHeader(0x00, large)), + } { + t.Run(name, func(t *testing.T) { + if err := Check(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("err = %v, want ErrMalformed", err) + } + }) + } +} + +// xzStreamCount rewrites the record count of a one-block stream's index. +func xzStreamCount(t *testing.T, stream []byte, count uint64) []byte { + t.Helper() + indexStart := len(stream) - 12 - 8 + index := append([]byte{0}, xzVarint(count)...) + if len(index) > 8 { + t.Fatal("count too long for the index") + } + return append(append(bytes.Clone(stream[:indexStart]), append(index, make([]byte, 8-len(index))...)...), stream[len(stream)-12:]...) +} diff --git a/registries_test.go b/registries_test.go index f041108..7f1c8b0 100644 --- a/registries_test.go +++ b/registries_test.go @@ -20,7 +20,7 @@ import ( func TestSupportedEcosystems(t *testing.T) { ecosystems := registries.SupportedEcosystems() - expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} + expected := []string{"apk", "apm", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deb", "deno", "dub", "elm", "gem", "githubactions", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "rpm", "swift", "terraform"} sort.Strings(ecosystems) if len(ecosystems) != len(expected) { @@ -70,6 +70,7 @@ func TestNew(t *testing.T) { {"apm", true}, {"apk", true}, {"deb", true}, + {"rpm", true}, {"unknown", true}, } @@ -119,6 +120,7 @@ func TestDefaultURL(t *testing.T) { {"apm", ""}, {"apk", ""}, {"deb", ""}, + {"rpm", ""}, } for _, tt := range tests { From b19b320618b84934d40dc38a35fe419e5a5125fd Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 16:52:56 +0100 Subject: [PATCH 4/7] Walk .xz files as the decoder does, and four index fixes The dictionary check found blocks through each stream's index, which the decoder only compares with the blocks after it has read their headers and allocated their dictionaries. An index that left a block out hid it. The check now reads the file front to back: each block header, then the block's LZMA2 chunks by the sizes in their headers. It no longer takes sizes from the footer, which could overflow a 32-bit int. - The same version in two releases is recorded for each, so either distro resolves; -updates repeating a version still adds nothing. - An index the Release lists but the archive doesn't serve fails the read instead of leaving its packages out for the Registry's lifetime. - A clearsigned Release with CRLF line endings is read, and one without its header separator or signature is an error, not an empty Release. - Dependencies keep << and >> requirements, which were cut as build profiles. --- internal/deb/deb.go | 15 +- internal/deb/deb_test.go | 65 +++++ internal/deb/parse.go | 59 +++-- internal/deb/parse_test.go | 237 ++++++++++++++---- .../deb/testdata/bookworm-updates-Packages.xz | Bin 0 -> 6924 bytes internal/deb/xzcheck.go | 219 +++++++++------- 6 files changed, 434 insertions(+), 161 deletions(-) create mode 100644 internal/deb/testdata/bookworm-updates-Packages.xz diff --git a/internal/deb/deb.go b/internal/deb/deb.go index d56b013..8fd63fe 100644 --- a/internal/deb/deb.go +++ b/internal/deb/deb.go @@ -183,8 +183,11 @@ func (r *Registry) index(ctx context.Context) (map[string][]record, error) { packages := map[string][]record{} seen := map[string]bool{} + // A version is recorded once per release: bookworm-updates repeating + // one, or binary-all repeating an Architecture: all package, adds + // nothing, while the same version in trixie is another artifact. add := func(rec record) { - key := rec.name + "\x00" + rec.version + "\x00" + rec.arch + key := rec.name + "\x00" + rec.version + "\x00" + rec.arch + "\x00" + rec.distro if seen[key] { return } @@ -277,8 +280,12 @@ func (rel *release) lists(dir string) bool { // readIndex reads the first variant of dir's index that is served. With a // Release, only variants it lists are tried, and each is checked against -// its entry. It reports whether an index was read. +// its entry. It reports whether an index was read: a directory the Release +// doesn't list, or without a Release one the archive doesn't serve, has +// none. A directory the Release lists must be served, or a mirror caught +// mid-sync would leave its packages out for the Registry's lifetime. func (r *Registry) readIndex(ctx context.Context, suite, dir string, rel *release, fn func(record)) (bool, error) { + listed := false for _, variant := range indexVariants { var sum *fileSum if rel != nil && len(rel.files) > 0 { @@ -287,6 +294,7 @@ func (r *Registry) readIndex(ctx context.Context, suite, dir string, rel *releas continue } sum = &entry + listed = true } indexURL := r.archiveURL + "/dists/" + suite + "/" + dir + "/" + variant var recs []record @@ -306,6 +314,9 @@ func (r *Registry) readIndex(ctx context.Context, suite, dir string, rel *releas } return true, nil } + if listed { + return false, fmt.Errorf("%w: %s/dists/%s/%s is listed in the Release but not served", ErrNoIndex, r.archiveURL, suite, dir) + } return false, nil } diff --git a/internal/deb/deb_test.go b/internal/deb/deb_test.go index 3ee9fdd..57d933f 100644 --- a/internal/deb/deb_test.go +++ b/internal/deb/deb_test.go @@ -666,3 +666,68 @@ func TestChecksumSizeMismatch(t *testing.T) { t.Errorf("err = %v, want ErrChecksumMismatch", err) } } + +// The same version in two releases is one version with an artifact for +// each, so either distro resolves; within a release, -updates repeating a +// version adds nothing. +func TestSameVersionInTwoReleases(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("bookworm", "bookworm", true) + a.index("bookworm-updates", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("bookworm-updates", "bookworm-updates", true) + a.index("trixie", "main/binary-amd64", "Packages.xz", caCertificatesOnly) + a.release("trixie", "trixie", true) + base := a.server().URL + "/debian" + + versions, err := newRegistry(base, "suites=bookworm,bookworm-updates,trixie&architectures=amd64").FetchVersions(context.Background(), "ca-certificates") + if err != nil || len(versions) != 1 { + t.Fatalf("FetchVersions = %v, %v", numbers(versions), err) + } + var distros []any + for _, artifact := range versions[0].Artifacts { + distros = append(distros, artifact.Metadata["distro"]) + } + if want := []any{"bookworm", "trixie"}; !reflect.DeepEqual(distros, want) { + t.Errorf("artifact distros = %v, want %v", distros, want) + } +} + +// An index the Release lists but the archive doesn't serve, as on a mirror +// caught mid-sync, fails the read rather than leaving its packages out. +func TestListedIndexMissing(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.index("bookworm", "main/binary-arm64", "Packages.xz", bookwormARM64) + a.index("bookworm", "main/binary-arm64", "Packages.gz", bookwormARM64) + a.release("bookworm", "bookworm", true) + xzPath, gzPath := "dists/bookworm/main/binary-arm64/Packages.xz", "dists/bookworm/main/binary-arm64/Packages.gz" + xzData, gzData := a.served[xzPath], a.served[gzPath] + delete(a.served, xzPath) + delete(a.served, gzPath) + base := a.server().URL + "/debian" + + r := newRegistry(base, "suites=bookworm&architectures=amd64,arm64") + if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrNoIndex) { + t.Fatalf("err = %v, want ErrNoIndex", err) + } + // Nothing was kept, so the index is read once the mirror has it. + a.mu.Lock() + a.served[xzPath], a.served[gzPath] = xzData, gzData + a.mu.Unlock() + versions, err := r.FetchVersions(context.Background(), "curl") + if err != nil || len(versions) != 1 || len(versions[0].Artifacts) != 2 { + t.Errorf("after the mirror caught up: %+v, %v", versions, err) + } +} + +// An architecture the Release doesn't list at all is skipped, as before. +func TestUnlistedArchitectureSkipped(t *testing.T) { + a := newArchive(t) + a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) + a.release("bookworm", "bookworm", true) + base := a.server().URL + "/debian" + if _, err := newRegistry(base, "suites=bookworm&architectures=amd64,riscv64").FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions: %v", err) + } +} diff --git a/internal/deb/parse.go b/internal/deb/parse.go index 40861fc..76d3a50 100644 --- a/internal/deb/parse.go +++ b/internal/deb/parse.go @@ -161,8 +161,12 @@ type release struct { // parseRelease reads a Release file, or an InRelease file, whose // clearsigned text is read without checking the signature. func parseRelease(data []byte) (*release, error) { + text, err := clearsignedText(data) + if err != nil { + return nil, err + } rel := &release{files: map[string]fileSum{}} - scanner := bufio.NewScanner(bytes.NewReader(clearsignedText(data))) + scanner := bufio.NewScanner(bytes.NewReader(text)) scanner.Buffer(nil, maxLineSize) field := "" for scanner.Scan() { @@ -202,24 +206,28 @@ func parseRelease(data []byte) (*release, error) { // clearsignedText returns the signed text of an OpenPGP clearsigned // message, with dash escapes removed, or data unchanged if it is not one. -func clearsignedText(data []byte) []byte { +// A clearsigned message without its header separator or its signature is +// an error rather than an empty Release, which would check nothing. +func clearsignedText(data []byte) ([]byte, error) { const begin, sig = "-----BEGIN PGP SIGNED MESSAGE-----", "-----BEGIN PGP SIGNATURE-----" if !bytes.HasPrefix(data, []byte(begin)) { - return data + return data, nil } + data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) // Armor headers such as Hash: end at the first blank line. _, body, ok := bytes.Cut(data, []byte("\n\n")) if !ok { - return nil + return nil, fmt.Errorf("%w: clearsigned Release has no header separator", ErrMalformedIndex) } - if i := bytes.Index(body, []byte("\n"+sig)); i >= 0 { - body = body[:i+1] + end := bytes.Index(body, []byte("\n"+sig)) + if end < 0 { + return nil, fmt.Errorf("%w: clearsigned Release has no signature", ErrMalformedIndex) } - lines := bytes.SplitAfter(body, []byte("\n")) + lines := bytes.SplitAfter(body[:end+1], []byte("\n")) for i, line := range lines { lines[i] = bytes.TrimPrefix(line, []byte("- ")) } - return bytes.Join(lines, nil) + return bytes.Join(lines, nil), nil } // distro is the codename a suite's packages are recorded under: @@ -244,35 +252,26 @@ func parseDependencies(fields ...string) []dependency { for _, field := range fields { for _, group := range strings.Split(field, ",") { first, _, _ := strings.Cut(group, "|") - first = cutBetween(cutBetween(first, "[", "]"), "<", ">") - name, requirement, _ := strings.Cut(first, "(") - name = strings.TrimSpace(name) - name, _, _ = strings.Cut(name, ":") - if name == "" { + // The version comes first: its << and >> are not profiles. + name, rest, versioned := strings.Cut(first, "(") + requirement := "" + if versioned { + requirement, _, _ = strings.Cut(rest, ")") + } + // What is left of the name ends at a qualifier, restriction or + // profile. + if i := strings.IndexAny(name, ":[<"); i >= 0 { + name = name[:i] + } + if name = strings.TrimSpace(name); name == "" { continue } - requirement = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(requirement), ")")) - deps = append(deps, dependency{name: name, requirement: requirement}) + deps = append(deps, dependency{name: name, requirement: strings.TrimSpace(requirement)}) } } return deps } -// cutBetween removes every open...close span from s. -func cutBetween(s, open, close string) string { - for { - i := strings.Index(s, open) - if i < 0 { - return s - } - j := strings.Index(s[i:], close) - if j < 0 { - return s[:i] - } - s = s[:i] + s[i+j+len(close):] - } -} - // safePoolPath reports whether a Filename stays under the archive root: // relative, without dot segments, escapes or URL syntax. func safePoolPath(p string) bool { diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go index 93ddef2..a12b8ae 100644 --- a/internal/deb/parse_test.go +++ b/internal/deb/parse_test.go @@ -5,9 +5,13 @@ import ( "encoding/binary" "errors" "hash/crc32" + "io" + "os" "reflect" "strings" "testing" + + "github.com/ulikunitz/xz" ) func TestParseStanzas(t *testing.T) { @@ -82,6 +86,35 @@ func TestParseRelease(t *testing.T) { } } +// A clearsigned file may use CRLF line endings. +func TestParseReleaseCRLF(t *testing.T) { + data := "-----BEGIN PGP SIGNED MESSAGE-----\r\nHash: SHA256\r\n\r\n" + + "Codename: bookworm\r\nSHA256:\r\n abcd 12 main/binary-amd64/Packages.xz\r\n" + + "-----BEGIN PGP SIGNATURE-----\r\n\r\niQIz\r\n-----END PGP SIGNATURE-----\r\n" + + "Codename: unsigned\r\n" + rel, err := parseRelease([]byte(data)) + if err != nil { + t.Fatalf("parseRelease: %v", err) + } + want := map[string]fileSum{"main/binary-amd64/Packages.xz": {sha256: "abcd", size: 12}} + if rel.codename != "bookworm" || !reflect.DeepEqual(rel.files, want) { + t.Errorf("release = %+v", rel) + } +} + +// A clearsigned file without its header separator or signature is not +// read as an empty Release, which would leave the indexes unchecked. +func TestParseReleaseBrokenEnvelope(t *testing.T) { + for name, data := range map[string]string{ + "no blank line": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\nCodename: bookworm\n", + "no signature": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\nCodename: bookworm\n", + } { + if _, err := parseRelease([]byte(data)); !errors.Is(err, ErrMalformedIndex) { + t.Errorf("%s: err = %v, want ErrMalformedIndex", name, err) + } + } +} + func TestParseReleaseMalformedEntry(t *testing.T) { for _, entry := range []string{" abcd notasize main/binary-amd64/Packages", " abcd 12 main/binary-amd64/Packages extra", " abcd 12"} { if _, err := parseRelease([]byte("SHA256:\n" + entry + "\n")); !errors.Is(err, ErrMalformedIndex) { @@ -104,10 +137,12 @@ func TestDistroFromCodename(t *testing.T) { } func TestParseDependencies(t *testing.T) { - got := parseDependencies("libc6 (>= 2.34)", "adduser, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>=1:1.1.4), libselinux1 [linux-any], libfoo , ") + got := parseDependencies("libc6 (>= 2.34)", "adduser, libssl3 (>= 3.0.13) | libssl3t64, zlib1g:any (>=1:1.1.4), libselinux1 [linux-any], libfoo , "+ + "libold (<< 2.0), libnew (>> 1.0) [amd64] , libexact (= 1:2-3), libbare[i386], libfirst | libsecond (>= 1)") want := []dependency{ {"libc6", ">= 2.34"}, {"adduser", ""}, {"libssl3", ">= 3.0.13"}, {"zlib1g", ">=1:1.1.4"}, {"libselinux1", ""}, {"libfoo", ""}, + {"libold", "<< 2.0"}, {"libnew", ">> 1.0"}, {"libexact", "= 1:2-3"}, {"libbare", ""}, {"libfirst", ""}, } if !reflect.DeepEqual(got, want) { t.Errorf("parseDependencies =\n %+v\nwant\n %+v", got, want) @@ -204,7 +239,7 @@ func xzBlockHeader(flags byte, body ...[]byte) []byte { for _, b := range body { h = append(h, b...) } - for (len(h)+4)%4 != 0 { + for len(h)%4 != 0 { h = append(h, 0) } h = append(h, 0, 0, 0, 0) @@ -214,16 +249,41 @@ func xzBlockHeader(flags byte, body ...[]byte) []byte { func lzma2Props(bits byte) []byte { return []byte{lzma2Filter, 0x01, bits} } -// xzStream builds one stream whose blocks hold only their headers. Nothing -// here is meant to decode; checkXZ reads only the container. -func xzStream(headers ...[]byte) []byte { - out := append(bytes.Clone(xzHeaderMagic), 0, 1, 0, 0, 0, 0) - index := []byte{0} - index = append(index, xzVarint(uint64(len(headers)))...) - for _, h := range headers { - out = append(out, h...) - index = append(index, xzVarint(uint64(len(h)))...) - index = append(index, xzVarint(1)...) +// xzBlock is a block header and the LZMA2 chunks after it, without the end +// marker. Nothing here is meant to decode; checkXZ reads only the layout. +type xzBlock struct { + header []byte + chunks []byte +} + +// xzFile describes a stream to build. +type xzFile struct { + blocks []xzBlock + checkType byte // 1 is CRC32, four bytes after each block + // records is how many blocks the index lists; -1 lists them all. + records int +} + +func (f xzFile) bytes() []byte { + checkSize := xzCheckSizes[f.checkType] + out := append(bytes.Clone(xzHeaderMagic), 0, f.checkType, 0, 0, 0, 0) + records := f.records + if records < 0 { + records = len(f.blocks) + } + index := append([]byte{0}, xzVarint(uint64(records))...) + for i, b := range f.blocks { + block := append(bytes.Clone(b.header), b.chunks...) + block = append(block, 0) // the LZMA2 end marker + if i < records { + index = append(index, xzVarint(uint64(len(block)+checkSize))...) + index = append(index, xzVarint(1)...) + } + for len(block)%4 != 0 { + block = append(block, 0) + } + out = append(out, block...) + out = append(out, make([]byte, checkSize)...) } for len(index)%4 != 0 { index = append(index, 0) @@ -236,12 +296,54 @@ func xzStream(headers ...[]byte) []byte { return append(out, footer...) } +// xzStream builds a stream of blocks with these headers, a CRC32 check and +// an index listing them all. +func xzStream(headers ...[]byte) []byte { + f := xzFile{checkType: 1, records: -1} + for _, h := range headers { + f.blocks = append(f.blocks, xzBlock{header: h}) + } + return f.bytes() +} + func TestCheckXZContainer(t *testing.T) { - small := lzma2Props(22) - large := lzma2Props(38) - valid := xzStream(xzBlockHeader(0x00, small)) - if err := checkXZ(valid, maxXZDict); err != nil { - t.Fatalf("valid synthetic stream: %v", err) + small := xzBlockHeader(0x00, lzma2Props(22)) + large := xzBlockHeader(0x00, lzma2Props(38)) + valid := xzStream(small) + + // Chunks as LZMA2 frames them: stored (control, size-1, data) and LZMA + // (control, uncompressed size-1, compressed size-1, properties on a + // reset, data). Their data is filled with bytes that would read as a + // block header asking for a large dictionary, or as an end marker, if + // a chunk were walked by the wrong size. + filler := func(n int) []byte { return bytes.Repeat(append([]byte{0}, large...), n)[:n] } + stored := append([]byte{0x01, 0x00, 0x1F}, filler(0x20)...) + storedNoReset := append([]byte{0x02, 0x01, 0x0F}, filler(0x110)...) + lzmaChunk := append([]byte{0x80, 0xFF, 0xFF, 0x00, 0x2F}, filler(0x30)...) + lzmaWithProps := append([]byte{0xE0, 0x00, 0x10, 0x01, 0x04, 0x5D}, filler(0x105)...) + concat := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) } + + for name, data := range map[string][]byte{ + "one block": valid, + "two blocks": xzStream(small, small), + "no blocks": xzStream(), + "no check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 0, records: -1}.bytes(), + "crc64 check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 4, records: -1}.bytes(), + "sha256 check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 10, records: -1}.bytes(), + "stored chunk": xzFile{blocks: []xzBlock{{small, stored}, {header: small}}, checkType: 1, records: -1}.bytes(), + "stored without reset": xzFile{blocks: []xzBlock{{small, storedNoReset}, {header: small}}, checkType: 1, records: -1}.bytes(), + "lzma chunk": xzFile{blocks: []xzBlock{{small, lzmaChunk}, {header: small}}, checkType: 1, records: -1}.bytes(), + "lzma chunk with properties": xzFile{blocks: []xzBlock{{small, lzmaWithProps}, {header: small}}, checkType: 1, records: -1}.bytes(), + "several chunks": xzFile{blocks: []xzBlock{{small, concat(lzmaWithProps, stored, lzmaChunk, storedNoReset)}, {header: small}}, checkType: 1, records: -1}.bytes(), + "sizes then small": xzStream(xzBlockHeader(0xC0, xzVarint(100), xzVarint(200), lzma2Props(22))), + "delta then small": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, lzma2Props(22))), + "padded streams": concat(valid, make([]byte, 8), valid, make([]byte, 4)), + } { + t.Run("valid/"+name, func(t *testing.T) { + if err := checkXZ(data, maxXZDict); err != nil { + t.Errorf("err = %v, want none", err) + } + }) } patch := func(data []byte, at int, b ...byte) []byte { @@ -252,30 +354,36 @@ func TestCheckXZContainer(t *testing.T) { copy(out[at:], b) return out } - indexStart := len(valid) - 12 - 8 // one record: indicator, count, two sizes, padding, CRC tenByteOne := []byte{0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00} - for name, data := range map[string][]byte{ - "footer magic": patch(valid, -2, 'Y', 'X'), - "header magic": patch(valid, 0, 0xFE), - "index indicator": patch(valid, indexStart, 1), - "index past the start": patch(valid, -8, 0xFF, 0xFF, 0xFF, 0x0F), - "huge record count": xzStreamCount(t, valid, 1<<40), - "zero header size": xzStream(append([]byte{0}, xzBlockHeader(0x00, small)[1:]...)), - "header past block": xzStream(append([]byte{0x40}, xzBlockHeader(0x00, small)[1:]...)), + "footer magic": patch(valid, -2, 'Y', 'X'), + "header magic": patch(valid, 0, 0xFE), + "header past the end": xzStream(append([]byte{0xFF}, small[1:]...)), + // The index lists one block; the decoder reads both headers. + "block the index leaves out": xzFile{blocks: []xzBlock{{header: small}, {header: large}}, checkType: 1, records: 1}.bytes(), + "index lists more than it holds": xzFile{blocks: []xzBlock{{header: small}}, checkType: 1, records: 1 << 40}.bytes(), + "large second block": xzStream(small, large), + "large after a stored chunk": xzFile{blocks: []xzBlock{{small, stored}, {header: large}}, checkType: 1, records: -1}.bytes(), + "large after an lzma chunk": xzFile{blocks: []xzBlock{{small, lzmaWithProps}, {header: large}}, checkType: 1, records: -1}.bytes(), + "large in a second stream": concat(valid, make([]byte, 4), xzStream(large)), + "bad chunk control": xzFile{blocks: []xzBlock{{small, []byte{0x05, 0x00, 0x00}}}, checkType: 1, records: -1}.bytes(), + "chunk past the end": xzFile{blocks: []xzBlock{{small, []byte{0x01, 0xFF, 0xFF}}}, checkType: 1, records: -1}.bytes(), // Read as filter flags, these sizes would name a one-byte filter // that isn't LZMA2 and hide the one after them. - "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), large)), - // The same with each size on its own: past the size, the filter - // flags claim 33 bytes of properties, which aren't there. - "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), - "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), - "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, large)), - "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), - "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), - "large second block": xzStream(xzBlockHeader(0x00, small), xzBlockHeader(0x00, large)), + "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), lzma2Props(38))), + "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), + "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), + "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, lzma2Props(38))), + "last filter not lzma2": xzStream(xzBlockHeader(0x01, lzma2Props(22), []byte{0x03, 0x01, 0x00})), + "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), + "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), + "padding of two bytes": concat(valid, make([]byte, 2)), + "padding with a stray byte": concat(valid, []byte{0, 1, 0, 0}, valid), + "padding of six bytes": concat(valid, make([]byte, 6), valid), + "garbage after a stream": concat(valid, []byte("garbage!")), + "padding before a stream": concat(make([]byte, 4), valid), } { - t.Run(name, func(t *testing.T) { + t.Run("refused/"+name, func(t *testing.T) { if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { t.Errorf("err = %v, want ErrMalformedIndex", err) } @@ -283,13 +391,54 @@ func TestCheckXZContainer(t *testing.T) { } } -// xzStreamCount rewrites the record count of a one-block stream's index. -func xzStreamCount(t *testing.T, stream []byte, count uint64) []byte { - t.Helper() - indexStart := len(stream) - 12 - 8 - index := append([]byte{0}, xzVarint(count)...) - if len(index) > 8 { - t.Fatal("count too long for the index") +// TestCheckXZRealFiles walks files real encoders wrote: Debian's own index, +// and a multi-block file of compressible and incompressible data, which +// holds both kinds of chunk. Every truncation of them is refused. +func TestCheckXZRealFiles(t *testing.T) { + debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") + if err != nil { + t.Fatal(err) + } + noise := make([]byte, 200_000) + state := uint32(1) + for i := range noise { + state = state*1664525 + 1013904223 + noise[i] = byte(state >> 24) + } + var buf bytes.Buffer + w, err := xz.WriterConfig{BlockSize: 64 << 10}.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + for _, part := range [][]byte{[]byte(strings.Repeat(bookwormAMD64, 200)), noise, []byte(strings.Repeat("x", 300_000))} { + if _, err := w.Write(part); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + + for name, data := range map[string][]byte{"debian": debian, "multi-block": buf.Bytes()} { + if err := checkXZ(data, maxXZDict); err != nil { + t.Errorf("%s: %v", name, err) + } + // The decoder agrees it is whole, so the walk covered every block. + r, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(io.Discard, r); err != nil { + t.Fatalf("%s does not decode: %v", name, err) + } + step := 1 + if len(data) > 20_000 { + step = 97 + } + for n := 0; n < len(data); n += step { + if err := checkXZ(data[:n], maxXZDict); !errors.Is(err, ErrMalformedIndex) { + t.Fatalf("%s cut to %d bytes: err = %v, want ErrMalformedIndex", name, n, err) + } + } } - return append(append(bytes.Clone(stream[:indexStart]), append(index, make([]byte, 8-len(index))...)...), stream[len(stream)-12:]...) } diff --git a/internal/deb/testdata/bookworm-updates-Packages.xz b/internal/deb/testdata/bookworm-updates-Packages.xz new file mode 100644 index 0000000000000000000000000000000000000000..2498e0032691e7d1ad79cd6db4a2eeb54d39342a GIT binary patch literal 6924 zcmV+n8}sD-H+ooF000E$*0e?hz|1xE{{bNZ761SM0000C1!g+nfAku}T>vN;NM_sG zFopgW;I~mvOFy8&^$kuXrjLMtTtqid|FD|Nu7$duLQcp=k*GgK!LMep#%DoNTxz ztzZAezZBscsU9{6n+bkHRjLI`Ciy)QiEfc;hl~~2yhjkLSvTx_h|$H>?>PQlHVfr- ze>cN({@8MEXY;xMi|2yilIgDxD!?w&d>l$O9Y>^R$~@kau?zvua~C}^Tz_Nxy5K}T zpe~_m$qoMTvU;F_dZl3FdTeVzA;pP-34^(P!AOy$_>Lb%Uy0_?2IuTzNq4+}uyC??nn3q1u91GkGG!BZFI zyPXgWW7RJ8w(TFt67!z~P$`cJcP7ucjhcyqjt1x?1H=4V22dFRQ_C#G*@)U}TXi3> z2-jftPdn**D5f&V#3)KMc|M!R3JIH%jd&Jb6r^eN)7H_Y@`Y|-XDOZsPlVTe~m>|PHPh5bB+twA;P?^ldYfj z=BonhXnh36qGNk#!Og9}4FXFzbI%}tRM=Hs-`%Z?PeS(Ii?me!p(mYhe|j?ULZl{A zDLr-eGqP$bnn%+>X<35avYa&1EBiV5ZVK!YoBn_zk+N!-FTnL;W#z?JmIFO^(ki#T zm>vdce9l&Na^h_Zx$@?^+DIA|q^uAKEp0CnI^K!EiUea{v(wb(d8S&iwu})k^El>WFT+&tzBZ;&ia_#0Rqot zi~lqJbp_iur!}g{I_#M>~ci)N*77#b%jWH6T8NJk#Q3}DX4*BBe+s5|Da^XtB)rm>u;Y{D?MBc%2M z5cNe*E)Jg^&T>2*QeT_1uoE(5%>Zg$Av^-ON`z=kX|--iC%Ifcp)F?5Lxkhq+!ixP zGtIc_a}%)ES+qSLP7uTlibNOei~#a%-TRUW;wVj zE}|61exG#hk0m1U(A$Re{@6fAsgS) zLC~mKkN=vzlCI}{z{h90xdF(dw%np%ZI3I++u_9hw8E-86@U1A81r#pjsED?LtKtX zp0=CM`K4ak0OdE8{{;JM>w3Y^GJohZYX4iFQlJfs)*pTMlig>XR8P74(KmN-b`P9` z)P-|$rYv1eQv$^Rnc2XTrE8qv-4`x-GQjVFsN?l@ z>V##H`q8N{{`?}Jw%0;2V1W-bjF-4a)7Q{y&2cN7P`}M3&or6XO7AKbryuiN$l%x> z!t0c+DJK!q#i!18ANJ1R|`)wgC+W5SI8ONu((pY zR1_r3=hV`$;Sk~6ubdBi?HuY!SZXRuP@VZjFWn$nNVgudc&7dR-pFkty2&wHPnHOm z(knN=Yw^m0J>0Z9k2*iglOn4zvo5q8K>(Qt0T+2k6L^dlB)c>Z; zjAl3`l0^IdMipqS;~{ru*c)r1p!<;=gc67|Ys;xEo&87|k=mkeyWtX5pg%z8i2QHi zR%o`*)Xl;U7~{U+XuM*}Ic0>x(I7$O{-v%qG%J6ndlkbj7o@~rRRzgPA0|A#2^+%Q zQWKiWO6Si4ZCRDlYt0gfjv7WX|JE`Jb0d{`W1sT`O4jxeQTK;qk>o(ZqB5pnUO_`L zvUX_7+?H(EZ3*Q0EH77M?n?_n*5lZ0tG(0%I>+Qf6t1*&&~FT5G1k$fC|nj>ITvH4 zVH};Mnau)!A<`RB)KZgu;8&^A{SBoCpA#@-e`GJw=J1+{s*a^Yx0#^3+e;As=dddf z8o}!SaqI3k6z`|5_6kf*vW~|79>u@pZ7(UD>~$%kHsOG~Wz>eUj6*G(eUI?s1s@{D zOrNMIGnf;Yh|1QUtt|$t2KXno-}Pv{J@Y%G8vC|u1VNh1QEFji-wWI36_N*)hl>4& z`&wJHw|PHhxoS1DeyVM)eRpTm`3g+0Xs~#*hD2*uG^QJG%ImE6%U3Z({*e07Y__jx z;7)3y1$l!~QI#Urcb6*pdG6;Pio{WmGMzrCb~nr4ZcG*M*AOBiQb5859s+!ENlARCgZ-&wS01AGK=Sj39Jbp?Eso=B ziVaWw2-r59b#|wR24#>>C+Cmp8&=DwDc-BdUV4MCj(=zb>PT3vHwBvV+fkoJ16UQP z*KHQHk#`BiyqBwS^lImOTU-Bo&V8KE6KiBM-a7==Oao5tdGOS<4YQ#>& z6+O!t(Iv1P`1I%}T6-bXySyPy$6{M9fW)a)xUHpwiBq$!k9 zgV$Gpu|n=#X7GxIln(F@o_;HBRXj}Ftj)%$6oN%K7d#NrfKfbss3dnx4m)x!p~2bi zf_Q~C;#mIZQ_xUBw)w2YktSlu7qj_S;rb=yi(gv^oxY^AV(FPlkC{=dHE}4{j`?_e zQLC~ipfV{s5x8mqs6yA9OI@FwwWfZJRsS4Lu7dj6Sq4XL*U$rPRG zbMU(CR0LLSauP0zGr|t$b~h?CE&im5NPTnBRLzjGpdi7Ui2-Mj@q(X^>-%;i-8W%$ zqN}u_h3-_z9|qrLw7_0px6vYz+=$lD)Moe4>Wk*YpwRc$A*%`k#a zwSh<3Ti736VElB`e8$}jDBmOOe6ok@+Mhan(8g`(tF+GJj#v_f%uGF!nzFgMs z4fU_<22)imuglr|=C_uPdN0>?8o~F0hN;Gz9m-&C3Wa>XR+iLMwvSrK)j|KkupVE2 zK-@N)928gl^XE7HTf?n2B2BV|OA{3nvot|+@5&iQ}u5Xo~qP7w)Td~-&Ao~`c$X(sj$0dPEF%d^vo9h9AG`cNa#K-T$MKq~63GOc+bp8BE* zdTHBQml_8P^$iXL(Mr^uib(zYK#bB$A)C{F`d|Qz{};FU#_{NN?EX!MPP8B}cfZ9Q zfz~%&YA6DIT91lrRuk>O9yxSF!ycuWd-Ht(o%dmBta&bFIOxdlnwy)R)@1YdQ6uaa zJ&t&W8p}8iw+)#_JqgphY<3Zs{7ZZOCWB7oE|(`BH&z_0P-usDX-7cZLJSP85{BpE zh5j5BQ3>n~cO9`ivbvVD!SMrMcoo;BTyIYAz09G%w%p3V<{*pdsQ(FD%5*y{mE47~ z#TT<ip0=&Z-l2|MaC*|&`hlvIVIE_tQmR!9|+&HBC7e;P#&$Dj|?mT+KiCTZH$ zB};MsKrw+R(yw*jp$TGP*fxSuW$|G3J?950eqlysn=T^b#Ft!5USYyR^w!NX4ay$8S0Lw90L6$46(_KFm4QuTn^kQw6(4 zO6Q9MG}C!RS4TD!8u1P=_*q&9gw9i~OilXr83h*S{V_Cne8W!SQ$0MsRuKiLNIE@N z*Y_wEZFdFyL-YL+Pys*MusKC{$kFo zE=K?>^5mOfx?(gCw>o?9`qgncN-LVU0<*A%vb+VGMvZ;_#)<|J`~XZ!*hqEMco_bt z-Zf&K71-Z@FU|216LrW+#m$^#l;Yi1MJBO!A{H)(6ND}%o(be*u@d4<=H}i4N9i3X zQSHYaAKT>%SG|cQwH^jIT;qeb$<<23N?o&s<}A7$06g;#;bdxEA*ia zEXDZ_I_Ppt5der+$HTKe$99oP%|dC88u|gW*+f)(bc+FnyPCuZQ|gCeTi%VT9EkN) znNf&?XW8g{{+w8+vbVYG%{_eMy zD{)dH(l#~(x;$m(Rbbnjv1Nytk^FNU$DgLnY|}=(4w-U>eh?t=*WVfd(j=V;#oHb^ zZp#2Dg8$mWsub9yg4-%en#J1k%ARLGp5l+scMCh6eD2=rxAXd!5hJ5{LUV3^F9g9c z`?)lL%803nJHXX~iI3fmtFrB9T%@t=$KxTo3ry~EgYZMh$sLbma>QE0Fvhg#JiL$b zk3O;$M>hlf)2U$3+hl?;6u=tZrkZvRI@1Yhf3`s%*z zPVFag7q(6327T=At zEv7=0Jhe?KRe5Fg3$*^$6JGe98U<`nq4EbChA;MQBOm``Sz zWd7Hqd*)<`AtVes!pPKja5U?Zw~az}4J1~1xu?{fwmkx%iivUaz!U`9 zcfw&kyi4`$4x9nOzIHODF7i=KMzOY^ZMv6{|IcjZ706; zfd-L>ym{jtrBZB6D~o?AlsDWjQd^r~+(vq}_T`{a_m2v>E_)8&;tdTJ%_HiF#R`WCSXs;Y@3`Ao|F##9VqnZoSV#>gx2I3IT==w5IPV18GJrr8IUOocMZXI zM2MgrPzO~Liy9cpdCIP?i%`j^yp^X!FcPZy4IEVCjbqjiQnpbpgRj%+6+27Df$7ZlM6kF>030H%(I`_vQy3 zL2@Zz07bBMh@|r$uF*q)r77CM*7&jJbdk5qKvne-2Ry!jBC)sS_hBOhyNx<8B$X`I z+i&F9LgO8zA62sEPwf1Z&+@#@`a!F~LisM@9o${(?nJWa6w-(hGI(v=_+{cLO?K+& z*nwmA7XS&MCJ*xdtOds(n)UDIo$tm;*g0w`5GuI&J$QZKnA0`7d} zq0Tl-J?o#@!`i==6R5N%NH=+qi>C&M&`fK+bL+Q~{fO?H4^ND4(SjBLNC6+=9FJbN zMe<>sTsAtPJ>>S#99|Jz^iuzl#^e8SW2Ac_UMy(FL4Si0D5Fg|G55L1&PsF;XlJXw zlB|HW%D&->(HA_5R2935`%p+9M3=m=xrWek1{yE=5Y|lkPPsQhC}dsVmL4(CN2(K& zrP#wa9yn{DDxnzbE7c$h=?}bSoznJP*-Al=zl8uyY@tSYEr93iAh zYrMp~s-$Lo%+B=9ca;QHBi+fsm5t8wv_KZ2t=?`BxaGXjNVu2HqymV+FGls)kO(lO z*8ORENXD06zcbaI=@g`oS6UP(mo9038*4-#Ok2mF61Ar_P1k@3jryk~2rPfs4RbqL z8FT3zI4VlTk5vb__>xJfW9)a(o})d-XD?)_(?P!b{{9JpM?D#YQ^Zgut+Bp$`n+jN z&7wx9cW&E_?NTr@*wHssp}^Rvq=AcseXeDMCYikY?c!I|CMCU3FX+?dzA(2aC{1DH z2@}^|*0mTS6QCt)YX#Z9~J$#szh_4QZc&r*@> z_JE<6nUxY*dK&tQTks{SF6`iI2rvMfPIfm^pQwmpdRKjlDeQ@P1|`ze-`?{47xPF#=?Ca2Pq!Y+g|TMxywn(gcAA%9pb#>hj3RX zd8Ng(Op^9&94kCFNM_U5Y0sd=p5f}z2tNA4hXSpZLBw2Fj4yGy!TW~~*oBh;TXFg> zU8SJY;GL?I^EMPZ!lb!5OCi4L9H~}*oW`j_7UZvum+bmkT7Wo8?xN88wG|Q7mQ)OJ zU7!4o#C-g169^c~A_|;HiIzS&rP2nkyr_>G5o+?KVbz!`WfodEXZHHhiXj70MTUEk7;i{x@|J*BUmQM4TJ{dW^H+M-YB1i@l&d; zJ~8-ve_OrZ_7yTl$t3t}j{J<?#8OTcz4o{UOm757cga(1}3&tMgRc#EiT)= 4 && bytes.Equal(data[end-4:end], []byte{0, 0, 0, 0}) { - end -= 4 - } - if end == 0 { - break + pos, streams := 0, 0 + for pos < len(data) { + if streams > 0 && data[pos] == 0 { + // Stream padding comes in multiples of four zero bytes. + if len(data)-pos < 4 || !bytes.Equal(data[pos:pos+4], []byte{0, 0, 0, 0}) { + return fmt.Errorf("%w: xz: bad stream padding", ErrMalformedIndex) + } + pos += 4 + continue } - start, err := checkXZStream(data[:end], maxDict) + next, err := checkXZStream(data, pos, maxDict) if err != nil { return fmt.Errorf("%w: xz: %w", ErrMalformedIndex, err) } - end = start + pos = next streams++ } if streams == 0 { @@ -51,123 +64,159 @@ func checkXZ(data []byte, maxDict uint64) error { return nil } -// checkXZStream checks the stream that ends data and returns its offset. -func checkXZStream(data []byte, maxDict uint64) (int, error) { - if len(data) < xzHeaderSize+xzFooterSize { - return 0, fmt.Errorf("stream too short") - } - footer := data[len(data)-xzFooterSize:] - if !bytes.Equal(footer[10:], xzFooterMagic) { - return 0, fmt.Errorf("no stream footer") - } - indexSize := (int(binary.LittleEndian.Uint32(footer[4:8])) + 1) * 4 - indexStart := len(data) - xzFooterSize - indexSize - if indexStart < 0 { - return 0, fmt.Errorf("index outside the stream") +// checkXZStream checks the stream that starts at pos and returns where it +// ends. +func checkXZStream(data []byte, pos int, maxDict uint64) (int, error) { + if len(data)-pos < xzHeaderSize { + return 0, errXZTruncated + } + header := data[pos : pos+xzHeaderSize] + if !bytes.Equal(header[:len(xzHeaderMagic)], xzHeaderMagic) { + return 0, errors.New("no stream header") + } + checkSize := xzCheckSizes[header[7]&0x0F] + pos += xzHeaderSize + + for { + if pos >= len(data) { + return 0, errXZTruncated + } + if data[pos] == 0 { + break // the index indicator: no more blocks + } + next, err := checkXZBlock(data, pos, checkSize, maxDict) + if err != nil { + return 0, err + } + pos = next } - blocks, err := xzIndexBlocks(data[indexStart:len(data)-xzFooterSize], indexStart) + + pos, err := skipXZIndex(data, pos) if err != nil { return 0, err } - total := 0 - for _, size := range blocks { - total += size + if len(data)-pos < xzFooterSize { + return 0, errXZTruncated } - start := indexStart - total - xzHeaderSize - if start < 0 || !bytes.Equal(data[start:start+len(xzHeaderMagic)], xzHeaderMagic) { - return 0, fmt.Errorf("no stream header") + if !bytes.Equal(data[pos+10:pos+xzFooterSize], xzFooterMagic) { + return 0, errors.New("no stream footer") } - offset := start + xzHeaderSize - for _, size := range blocks { - if err := checkXZBlockHeader(data[offset:offset+size], maxDict); err != nil { - return 0, err - } - offset += size - } - return start, nil + return pos + xzFooterSize, nil } -// xzIndexBlocks reads an xz index and returns the space each block takes, -// its unpadded size rounded up to four bytes. No block is larger than -// limit, the bytes before the index. -func xzIndexBlocks(index []byte, limit int) ([]int, error) { - if len(index) == 0 || index[0] != 0 { - return nil, fmt.Errorf("no index") +// checkXZBlock checks the header of the block that starts at pos and +// returns where the block ends: after its LZMA2 chunks, padding and check. +func checkXZBlock(data []byte, pos, checkSize int, maxDict uint64) (int, error) { + start := pos + headerSize := (int(data[pos]) + 1) * 4 + if len(data)-pos < headerSize { + return 0, errXZTruncated + } + if err := checkXZBlockHeader(data[pos:pos+headerSize], maxDict); err != nil { + return 0, err } - pos := 1 - count, n := readVarint(index[pos:]) - if n == 0 { - return nil, fmt.Errorf("bad index") + pos += headerSize + + // LZMA2 is the last filter of every block, so the block's data is a + // run of LZMA2 chunks up to an end marker. + for { + if pos >= len(data) { + return 0, errXZTruncated + } + control := data[pos] + if control == 0 { + pos++ + break + } + var chunk int + switch { + case control == 1 || control == 2: // stored: control, size + if len(data)-pos < 3 { + return 0, errXZTruncated + } + chunk = 3 + int(binary.BigEndian.Uint16(data[pos+1:])) + 1 + case control >= 0x80: // LZMA: control, sizes, properties on a reset + if len(data)-pos < 5 { + return 0, errXZTruncated + } + chunk = 5 + int(binary.BigEndian.Uint16(data[pos+3:])) + 1 + if control >= 0xC0 { + chunk++ + } + default: + return 0, errors.New("bad LZMA2 chunk") + } + // A chunk running past the end is caught by the next read. + pos += chunk } + + // The block is padded to four bytes, then comes its check. + return pos + (4-(pos-start)%4)%4 + checkSize, nil +} + +// skipXZIndex returns where the index that starts at pos ends. +func skipXZIndex(data []byte, pos int) (int, error) { + start := pos + pos++ // the indicator + // A malformed count reads as no records, which the footer then fails. + count, n := readVarint(data[pos:]) pos += n - // Each record is at least two bytes, which bounds count. - if count > uint64(len(index)) { - return nil, fmt.Errorf("bad index") - } - blocks := make([]int, 0, count) - for range count { - unpadded, n := readVarint(index[pos:]) - // Bounded so the sizes below cannot overflow int. - if n == 0 || unpadded > uint64(limit) { - return nil, fmt.Errorf("bad index record") - } - pos += n - if _, n = readVarint(index[pos:]); n == 0 { - return nil, fmt.Errorf("bad index record") + for ; count > 0; count-- { + for range 2 { // unpadded and uncompressed size + _, n := readVarint(data[pos:]) + if n == 0 { + return 0, errors.New("bad index record") + } + pos += n } - pos += n - blocks = append(blocks, int((unpadded+3)&^3)) } - return blocks, nil + // Padding to four bytes, then the CRC32. + return pos + (4-(pos-start)%4)%4 + 4, nil } -// checkXZBlockHeader reads the filters of the block that starts block. +// checkXZBlockHeader reads the filters of a block header. func checkXZBlockHeader(block []byte, maxDict uint64) error { - if len(block) == 0 || block[0] == 0 { - return fmt.Errorf("no block header") - } - size := (int(block[0]) + 1) * 4 - if size > len(block) { - return fmt.Errorf("block header past the block") - } - header := block[1 : size-4] // without the size byte and the CRC32 - if len(header) == 0 { - return fmt.Errorf("short block header") - } + // Without the size byte and the CRC32; a header is at least 8 bytes. + header := block[1 : len(block)-4] flags := header[0] pos := 1 for _, present := range []byte{0x40, 0x80} { // compressed and uncompressed sizes if flags&present != 0 { _, n := readVarint(header[pos:]) if n == 0 { - return fmt.Errorf("bad block header") + return errors.New("bad block header") } pos += n } } + last := uint64(0) for range int(flags&0x03) + 1 { id, n := readVarint(header[pos:]) if n == 0 { - return fmt.Errorf("bad filter flags") + return errors.New("bad filter flags") } pos += n propsSize, n := readVarint(header[pos:]) if n == 0 || uint64(len(header)-pos-n) < propsSize { - return fmt.Errorf("bad filter flags") + return errors.New("bad filter flags") } pos += n props := header[pos : pos+int(propsSize)] pos += int(propsSize) + last = id if id != lzma2Filter { continue } if len(props) != 1 || props[0] > 40 { - return fmt.Errorf("bad LZMA2 properties") + return errors.New("bad LZMA2 properties") } if dict := lzma2DictSize(props[0]); dict > maxDict { return fmt.Errorf("LZMA2 dictionary of %d bytes is larger than %d", dict, maxDict) } } + if last != lzma2Filter { + return errors.New("last filter is not LZMA2") + } return nil } From b4dced4e73e148010ab55db85642a9ead56dd930 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 16:53:06 +0100 Subject: [PATCH 5/7] Keep the xz test constants within a 32-bit int --- internal/deb/parse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go index a12b8ae..78b0248 100644 --- a/internal/deb/parse_test.go +++ b/internal/deb/parse_test.go @@ -361,7 +361,7 @@ func TestCheckXZContainer(t *testing.T) { "header past the end": xzStream(append([]byte{0xFF}, small[1:]...)), // The index lists one block; the decoder reads both headers. "block the index leaves out": xzFile{blocks: []xzBlock{{header: small}, {header: large}}, checkType: 1, records: 1}.bytes(), - "index lists more than it holds": xzFile{blocks: []xzBlock{{header: small}}, checkType: 1, records: 1 << 40}.bytes(), + "index lists more than it holds": xzFile{blocks: []xzBlock{{header: small}}, checkType: 1, records: 1 << 30}.bytes(), "large second block": xzStream(small, large), "large after a stored chunk": xzFile{blocks: []xzBlock{{small, stored}, {header: large}}, checkType: 1, records: -1}.bytes(), "large after an lzma chunk": xzFile{blocks: []xzBlock{{small, lzmaWithProps}, {header: large}}, checkType: 1, records: -1}.bytes(), From 720dfd6e30ae8992297b7c286810fdb68a03735a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 17:16:06 +0100 Subject: [PATCH 6/7] Bound the xz dictionary where blocks are decoded Checking block headers ahead of decoding cannot be made sound: where a block's data ends is only known by decoding it, since the LZMA2 decoder may stop short of the compressed size a chunk declares. A file could place a block header where the check did not look. internal/xzlimit reads the .xz container itself and hands each block's data to ulikunitz/xz's LZMA2 decoder with exactly the dictionary the block asks for, refusing one over the limit as it is reached. It verifies the header, block, index and footer checksums. Packages.xz is now decoded as a stream like the other variants, so its size limit and the check before decoding are gone. --- internal/deb/deb.go | 41 +- internal/deb/deb_test.go | 23 +- internal/deb/parse_test.go | 261 ----------- internal/deb/xzcheck.go | 242 ---------- .../testdata/bookworm-updates-Packages.xz | Bin internal/xzlimit/xzlimit.go | 435 ++++++++++++++++++ internal/xzlimit/xzlimit_test.go | 401 ++++++++++++++++ 7 files changed, 846 insertions(+), 557 deletions(-) delete mode 100644 internal/deb/xzcheck.go rename internal/{deb => xzlimit}/testdata/bookworm-updates-Packages.xz (100%) create mode 100644 internal/xzlimit/xzlimit.go create mode 100644 internal/xzlimit/xzlimit_test.go diff --git a/internal/deb/deb.go b/internal/deb/deb.go index 8fd63fe..7740942 100644 --- a/internal/deb/deb.go +++ b/internal/deb/deb.go @@ -3,7 +3,6 @@ package deb import ( - "bytes" "compress/gzip" "context" "crypto/sha256" @@ -20,8 +19,8 @@ import ( "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/registries/internal/xzlimit" "github.com/git-pkgs/vers" - "github.com/ulikunitz/xz" ) const ( @@ -31,11 +30,7 @@ const ( // maxIndexSize bounds one decompressed Packages index. noble universe // amd64 is about 73 MB. maxIndexSize = 512 << 20 - // maxXZSize bounds a Packages.xz, which is read whole so its blocks can - // be checked before decoding. sid main amd64 is about 11 MB. - maxXZSize = 128 << 20 - - archAll = "all" + archAll = "all" ) var ( @@ -83,7 +78,6 @@ type Registry struct { urls *URLs maxIndexSize int64 - maxXZSize int64 mu sync.Mutex packages map[string][]record @@ -93,7 +87,7 @@ func New(baseURL string, client *core.Client) *Registry { if client == nil { client = core.DefaultClient() } - r := &Registry{client: client, maxIndexSize: maxIndexSize, maxXZSize: maxXZSize} + r := &Registry{client: client, maxIndexSize: maxIndexSize} r.configErr = r.configure(baseURL) r.urls = &URLs{repositoryURL: r.repositoryURL()} return r @@ -329,25 +323,8 @@ func (r *Registry) parseIndex(body io.Reader, variant, indexURL string, sum *fil var decoded io.Reader switch variant { case "Packages.xz": - data, err := io.ReadAll(io.LimitReader(counted, r.maxXZSize+1)) - if err != nil { - return nil, err - } - if int64(len(data)) > r.maxXZSize { - return nil, fmt.Errorf("%w: %s is larger than %d bytes", ErrMalformedIndex, indexURL, r.maxXZSize) - } - // Checked before decoding, so a bad file never reaches the decoder. - if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { - return nil, err - } - if err := checkXZ(data, maxXZDict); err != nil { - return nil, err - } - xr, err := xz.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) - } - decoded = xr + // Bounded as it decodes: a block asking for more is an error. + decoded = xzlimit.NewReader(counted, xzlimit.MaxDict) case "Packages.gz": gz, err := gzip.NewReader(counted) if err != nil { @@ -367,11 +344,9 @@ func (r *Registry) parseIndex(body io.Reader, variant, indexURL string, sum *fil if limited.N == 0 { return nil, fmt.Errorf("%w: %s decompresses to more than %d bytes", ErrMalformedIndex, indexURL, r.maxIndexSize) } - // The gzip reader and the parser read to the end of the body. - if variant != "Packages.xz" { - if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { - return nil, err - } + // The decoders and the parser read to the end of the body. + if err := checkSum(hasher, counted.n, sum, indexURL); err != nil { + return nil, err } return recs, nil } diff --git a/internal/deb/deb_test.go b/internal/deb/deb_test.go index 57d933f..796257e 100644 --- a/internal/deb/deb_test.go +++ b/internal/deb/deb_test.go @@ -560,27 +560,8 @@ func TestDecompressedIndexLimit(t *testing.T) { } } -func TestXZSizeLimit(t *testing.T) { - a := newArchive(t) - a.index("bookworm", "main/binary-amd64", "Packages.xz", bookwormAMD64) - a.release("bookworm", "bookworm", true) - base := a.server().URL + "/debian" - size := int64(len(a.served["dists/bookworm/main/binary-amd64/Packages.xz"])) - - r := newRegistry(base, "suites=bookworm&architectures=amd64") - r.maxXZSize = size - 1 - if _, err := r.FetchVersions(context.Background(), "curl"); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("over the limit: err = %v, want ErrMalformedIndex", err) - } - r = newRegistry(base, "suites=bookworm&architectures=amd64") - r.maxXZSize = size - if _, err := r.FetchVersions(context.Background(), "curl"); err != nil { - t.Errorf("at the limit: %v", err) - } -} - -// An .xz whose blocks ask for too large a dictionary is refused before it -// is decoded, even when the Release lists it. +// An .xz whose blocks ask for too large a dictionary is refused, even when +// the Release lists it. func TestXZDictionaryRefused(t *testing.T) { a := newArchive(t) path := "dists/bookworm/main/binary-amd64/Packages.xz" diff --git a/internal/deb/parse_test.go b/internal/deb/parse_test.go index 78b0248..e5193d5 100644 --- a/internal/deb/parse_test.go +++ b/internal/deb/parse_test.go @@ -5,13 +5,9 @@ import ( "encoding/binary" "errors" "hash/crc32" - "io" - "os" "reflect" "strings" "testing" - - "github.com/ulikunitz/xz" ) func TestParseStanzas(t *testing.T) { @@ -185,260 +181,3 @@ func setXZDict(t *testing.T, data []byte, bits byte) []byte { binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) return out } - -func TestCheckXZDictionary(t *testing.T) { - small := xzData(t, bookwormAMD64) - if err := checkXZ(small, maxXZDict); err != nil { - t.Fatalf("checkXZ(8 MiB dictionary): %v", err) - } - // 2 GiB, past the 64 MiB limit. - if err := checkXZ(setXZDict(t, small, 38), maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("2 GiB dictionary: err = %v, want ErrMalformedIndex", err) - } - // A large dictionary in a second stream, after padding, is found too. - two := append(append(bytes.Clone(small), 0, 0, 0, 0), setXZDict(t, small, 38)...) - if err := checkXZ(two, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("second stream: err = %v, want ErrMalformedIndex", err) - } - if err := checkXZ(append(append(bytes.Clone(small), 0, 0, 0, 0), small...), maxXZDict); err != nil { - t.Errorf("two small streams: %v", err) - } - for name, data := range map[string][]byte{ - "empty": nil, - "truncated": small[:len(small)-3], - "garbage": []byte(strings.Repeat("x", 64)), - } { - if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("%s: err = %v, want ErrMalformedIndex", name, err) - } - } -} - -func TestXZDictSize(t *testing.T) { - for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { - if got := lzma2DictSize(bits); got != want { - t.Errorf("lzma2DictSize(%d) = %d, want %d", bits, got, want) - } - } -} - -// xzVarint encodes an xz multibyte integer. -func xzVarint(v uint64) []byte { - var out []byte - for v >= 0x80 { - out = append(out, byte(v)|0x80) - v >>= 7 - } - return append(out, byte(v)) -} - -// xzBlockHeader builds a block header from its flags byte and the bytes -// after it (sizes and filter flags), padded and with a CRC32 slot. -func xzBlockHeader(flags byte, body ...[]byte) []byte { - h := []byte{0, flags} - for _, b := range body { - h = append(h, b...) - } - for len(h)%4 != 0 { - h = append(h, 0) - } - h = append(h, 0, 0, 0, 0) - h[0] = byte(len(h)/4 - 1) - return h -} - -func lzma2Props(bits byte) []byte { return []byte{lzma2Filter, 0x01, bits} } - -// xzBlock is a block header and the LZMA2 chunks after it, without the end -// marker. Nothing here is meant to decode; checkXZ reads only the layout. -type xzBlock struct { - header []byte - chunks []byte -} - -// xzFile describes a stream to build. -type xzFile struct { - blocks []xzBlock - checkType byte // 1 is CRC32, four bytes after each block - // records is how many blocks the index lists; -1 lists them all. - records int -} - -func (f xzFile) bytes() []byte { - checkSize := xzCheckSizes[f.checkType] - out := append(bytes.Clone(xzHeaderMagic), 0, f.checkType, 0, 0, 0, 0) - records := f.records - if records < 0 { - records = len(f.blocks) - } - index := append([]byte{0}, xzVarint(uint64(records))...) - for i, b := range f.blocks { - block := append(bytes.Clone(b.header), b.chunks...) - block = append(block, 0) // the LZMA2 end marker - if i < records { - index = append(index, xzVarint(uint64(len(block)+checkSize))...) - index = append(index, xzVarint(1)...) - } - for len(block)%4 != 0 { - block = append(block, 0) - } - out = append(out, block...) - out = append(out, make([]byte, checkSize)...) - } - for len(index)%4 != 0 { - index = append(index, 0) - } - index = append(index, 0, 0, 0, 0) - out = append(out, index...) - footer := make([]byte, 12) - binary.LittleEndian.PutUint32(footer[4:8], uint32(len(index)/4-1)) - copy(footer[10:], xzFooterMagic) - return append(out, footer...) -} - -// xzStream builds a stream of blocks with these headers, a CRC32 check and -// an index listing them all. -func xzStream(headers ...[]byte) []byte { - f := xzFile{checkType: 1, records: -1} - for _, h := range headers { - f.blocks = append(f.blocks, xzBlock{header: h}) - } - return f.bytes() -} - -func TestCheckXZContainer(t *testing.T) { - small := xzBlockHeader(0x00, lzma2Props(22)) - large := xzBlockHeader(0x00, lzma2Props(38)) - valid := xzStream(small) - - // Chunks as LZMA2 frames them: stored (control, size-1, data) and LZMA - // (control, uncompressed size-1, compressed size-1, properties on a - // reset, data). Their data is filled with bytes that would read as a - // block header asking for a large dictionary, or as an end marker, if - // a chunk were walked by the wrong size. - filler := func(n int) []byte { return bytes.Repeat(append([]byte{0}, large...), n)[:n] } - stored := append([]byte{0x01, 0x00, 0x1F}, filler(0x20)...) - storedNoReset := append([]byte{0x02, 0x01, 0x0F}, filler(0x110)...) - lzmaChunk := append([]byte{0x80, 0xFF, 0xFF, 0x00, 0x2F}, filler(0x30)...) - lzmaWithProps := append([]byte{0xE0, 0x00, 0x10, 0x01, 0x04, 0x5D}, filler(0x105)...) - concat := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) } - - for name, data := range map[string][]byte{ - "one block": valid, - "two blocks": xzStream(small, small), - "no blocks": xzStream(), - "no check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 0, records: -1}.bytes(), - "crc64 check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 4, records: -1}.bytes(), - "sha256 check": xzFile{blocks: []xzBlock{{header: small}, {header: small}}, checkType: 10, records: -1}.bytes(), - "stored chunk": xzFile{blocks: []xzBlock{{small, stored}, {header: small}}, checkType: 1, records: -1}.bytes(), - "stored without reset": xzFile{blocks: []xzBlock{{small, storedNoReset}, {header: small}}, checkType: 1, records: -1}.bytes(), - "lzma chunk": xzFile{blocks: []xzBlock{{small, lzmaChunk}, {header: small}}, checkType: 1, records: -1}.bytes(), - "lzma chunk with properties": xzFile{blocks: []xzBlock{{small, lzmaWithProps}, {header: small}}, checkType: 1, records: -1}.bytes(), - "several chunks": xzFile{blocks: []xzBlock{{small, concat(lzmaWithProps, stored, lzmaChunk, storedNoReset)}, {header: small}}, checkType: 1, records: -1}.bytes(), - "sizes then small": xzStream(xzBlockHeader(0xC0, xzVarint(100), xzVarint(200), lzma2Props(22))), - "delta then small": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, lzma2Props(22))), - "padded streams": concat(valid, make([]byte, 8), valid, make([]byte, 4)), - } { - t.Run("valid/"+name, func(t *testing.T) { - if err := checkXZ(data, maxXZDict); err != nil { - t.Errorf("err = %v, want none", err) - } - }) - } - - patch := func(data []byte, at int, b ...byte) []byte { - out := bytes.Clone(data) - if at < 0 { - at += len(out) - } - copy(out[at:], b) - return out - } - tenByteOne := []byte{0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00} - for name, data := range map[string][]byte{ - "footer magic": patch(valid, -2, 'Y', 'X'), - "header magic": patch(valid, 0, 0xFE), - "header past the end": xzStream(append([]byte{0xFF}, small[1:]...)), - // The index lists one block; the decoder reads both headers. - "block the index leaves out": xzFile{blocks: []xzBlock{{header: small}, {header: large}}, checkType: 1, records: 1}.bytes(), - "index lists more than it holds": xzFile{blocks: []xzBlock{{header: small}}, checkType: 1, records: 1 << 30}.bytes(), - "large second block": xzStream(small, large), - "large after a stored chunk": xzFile{blocks: []xzBlock{{small, stored}, {header: large}}, checkType: 1, records: -1}.bytes(), - "large after an lzma chunk": xzFile{blocks: []xzBlock{{small, lzmaWithProps}, {header: large}}, checkType: 1, records: -1}.bytes(), - "large in a second stream": concat(valid, make([]byte, 4), xzStream(large)), - "bad chunk control": xzFile{blocks: []xzBlock{{small, []byte{0x05, 0x00, 0x00}}}, checkType: 1, records: -1}.bytes(), - "chunk past the end": xzFile{blocks: []xzBlock{{small, []byte{0x01, 0xFF, 0xFF}}}, checkType: 1, records: -1}.bytes(), - // Read as filter flags, these sizes would name a one-byte filter - // that isn't LZMA2 and hide the one after them. - "large after sizes": xzStream(xzBlockHeader(0xC0, xzVarint(3), xzVarint(1), lzma2Props(38))), - "compressed size only": xzStream(xzBlockHeader(0x40, xzVarint(3), []byte{0x01, 0x21})), - "uncompressed size only": xzStream(xzBlockHeader(0x80, xzVarint(3), []byte{0x01, 0x21})), - "large second filter": xzStream(xzBlockHeader(0x01, []byte{0x03, 0x01, 0x00}, lzma2Props(38))), - "last filter not lzma2": xzStream(xzBlockHeader(0x01, lzma2Props(22), []byte{0x03, 0x01, 0x00})), - "properties past 40": xzStream(xzBlockHeader(0x00, lzma2Props(200))), - "overlong varint": xzStream(xzBlockHeader(0x00, []byte{lzma2Filter}, tenByteOne, []byte{22})), - "padding of two bytes": concat(valid, make([]byte, 2)), - "padding with a stray byte": concat(valid, []byte{0, 1, 0, 0}, valid), - "padding of six bytes": concat(valid, make([]byte, 6), valid), - "garbage after a stream": concat(valid, []byte("garbage!")), - "padding before a stream": concat(make([]byte, 4), valid), - } { - t.Run("refused/"+name, func(t *testing.T) { - if err := checkXZ(data, maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Errorf("err = %v, want ErrMalformedIndex", err) - } - }) - } -} - -// TestCheckXZRealFiles walks files real encoders wrote: Debian's own index, -// and a multi-block file of compressible and incompressible data, which -// holds both kinds of chunk. Every truncation of them is refused. -func TestCheckXZRealFiles(t *testing.T) { - debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") - if err != nil { - t.Fatal(err) - } - noise := make([]byte, 200_000) - state := uint32(1) - for i := range noise { - state = state*1664525 + 1013904223 - noise[i] = byte(state >> 24) - } - var buf bytes.Buffer - w, err := xz.WriterConfig{BlockSize: 64 << 10}.NewWriter(&buf) - if err != nil { - t.Fatal(err) - } - for _, part := range [][]byte{[]byte(strings.Repeat(bookwormAMD64, 200)), noise, []byte(strings.Repeat("x", 300_000))} { - if _, err := w.Write(part); err != nil { - t.Fatal(err) - } - } - if err := w.Close(); err != nil { - t.Fatal(err) - } - - for name, data := range map[string][]byte{"debian": debian, "multi-block": buf.Bytes()} { - if err := checkXZ(data, maxXZDict); err != nil { - t.Errorf("%s: %v", name, err) - } - // The decoder agrees it is whole, so the walk covered every block. - r, err := xz.NewReader(bytes.NewReader(data)) - if err != nil { - t.Fatal(err) - } - if _, err := io.Copy(io.Discard, r); err != nil { - t.Fatalf("%s does not decode: %v", name, err) - } - step := 1 - if len(data) > 20_000 { - step = 97 - } - for n := 0; n < len(data); n += step { - if err := checkXZ(data[:n], maxXZDict); !errors.Is(err, ErrMalformedIndex) { - t.Fatalf("%s cut to %d bytes: err = %v, want ErrMalformedIndex", name, n, err) - } - } - } -} diff --git a/internal/deb/xzcheck.go b/internal/deb/xzcheck.go deleted file mode 100644 index 7baf486..0000000 --- a/internal/deb/xzcheck.go +++ /dev/null @@ -1,242 +0,0 @@ -package deb - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" -) - -// maxXZDict bounds the LZMA2 dictionary an index may ask for. The decoder -// allocates the dictionary a block header names, up to 4 GiB, and its own -// DictCap does not lower that (ulikunitz/xz#84). Debian uses 8 MiB. -const maxXZDict = 64 << 20 - -const ( - xzHeaderSize = 12 - xzFooterSize = 12 - lzma2Filter = 0x21 - maxVarint = 9 -) - -var ( - xzHeaderMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} - xzFooterMagic = []byte{'Y', 'Z'} - - errXZTruncated = errors.New("file ends early") - - // xzCheckSizes is the size of the check after each block, by the check - // type in the stream flags. - xzCheckSizes = [16]int{0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64} -) - -// checkXZ checks every block of every stream in an .xz file for an LZMA2 -// dictionary larger than maxDict, before the file is decoded. -// -// It reads the file front to back as the decoder does: each block header, -// then the LZMA2 chunks of the block, whose headers give their sizes, to -// where the next block header starts. The index at the end of a stream -// also lists the blocks, but the decoder reads a block header and -// allocates its dictionary before it compares the blocks with the index, -// so an index that leaves a block out would hide it from a check that -// trusted it. -func checkXZ(data []byte, maxDict uint64) error { - pos, streams := 0, 0 - for pos < len(data) { - if streams > 0 && data[pos] == 0 { - // Stream padding comes in multiples of four zero bytes. - if len(data)-pos < 4 || !bytes.Equal(data[pos:pos+4], []byte{0, 0, 0, 0}) { - return fmt.Errorf("%w: xz: bad stream padding", ErrMalformedIndex) - } - pos += 4 - continue - } - next, err := checkXZStream(data, pos, maxDict) - if err != nil { - return fmt.Errorf("%w: xz: %w", ErrMalformedIndex, err) - } - pos = next - streams++ - } - if streams == 0 { - return fmt.Errorf("%w: xz: no stream", ErrMalformedIndex) - } - return nil -} - -// checkXZStream checks the stream that starts at pos and returns where it -// ends. -func checkXZStream(data []byte, pos int, maxDict uint64) (int, error) { - if len(data)-pos < xzHeaderSize { - return 0, errXZTruncated - } - header := data[pos : pos+xzHeaderSize] - if !bytes.Equal(header[:len(xzHeaderMagic)], xzHeaderMagic) { - return 0, errors.New("no stream header") - } - checkSize := xzCheckSizes[header[7]&0x0F] - pos += xzHeaderSize - - for { - if pos >= len(data) { - return 0, errXZTruncated - } - if data[pos] == 0 { - break // the index indicator: no more blocks - } - next, err := checkXZBlock(data, pos, checkSize, maxDict) - if err != nil { - return 0, err - } - pos = next - } - - pos, err := skipXZIndex(data, pos) - if err != nil { - return 0, err - } - if len(data)-pos < xzFooterSize { - return 0, errXZTruncated - } - if !bytes.Equal(data[pos+10:pos+xzFooterSize], xzFooterMagic) { - return 0, errors.New("no stream footer") - } - return pos + xzFooterSize, nil -} - -// checkXZBlock checks the header of the block that starts at pos and -// returns where the block ends: after its LZMA2 chunks, padding and check. -func checkXZBlock(data []byte, pos, checkSize int, maxDict uint64) (int, error) { - start := pos - headerSize := (int(data[pos]) + 1) * 4 - if len(data)-pos < headerSize { - return 0, errXZTruncated - } - if err := checkXZBlockHeader(data[pos:pos+headerSize], maxDict); err != nil { - return 0, err - } - pos += headerSize - - // LZMA2 is the last filter of every block, so the block's data is a - // run of LZMA2 chunks up to an end marker. - for { - if pos >= len(data) { - return 0, errXZTruncated - } - control := data[pos] - if control == 0 { - pos++ - break - } - var chunk int - switch { - case control == 1 || control == 2: // stored: control, size - if len(data)-pos < 3 { - return 0, errXZTruncated - } - chunk = 3 + int(binary.BigEndian.Uint16(data[pos+1:])) + 1 - case control >= 0x80: // LZMA: control, sizes, properties on a reset - if len(data)-pos < 5 { - return 0, errXZTruncated - } - chunk = 5 + int(binary.BigEndian.Uint16(data[pos+3:])) + 1 - if control >= 0xC0 { - chunk++ - } - default: - return 0, errors.New("bad LZMA2 chunk") - } - // A chunk running past the end is caught by the next read. - pos += chunk - } - - // The block is padded to four bytes, then comes its check. - return pos + (4-(pos-start)%4)%4 + checkSize, nil -} - -// skipXZIndex returns where the index that starts at pos ends. -func skipXZIndex(data []byte, pos int) (int, error) { - start := pos - pos++ // the indicator - // A malformed count reads as no records, which the footer then fails. - count, n := readVarint(data[pos:]) - pos += n - for ; count > 0; count-- { - for range 2 { // unpadded and uncompressed size - _, n := readVarint(data[pos:]) - if n == 0 { - return 0, errors.New("bad index record") - } - pos += n - } - } - // Padding to four bytes, then the CRC32. - return pos + (4-(pos-start)%4)%4 + 4, nil -} - -// checkXZBlockHeader reads the filters of a block header. -func checkXZBlockHeader(block []byte, maxDict uint64) error { - // Without the size byte and the CRC32; a header is at least 8 bytes. - header := block[1 : len(block)-4] - flags := header[0] - pos := 1 - for _, present := range []byte{0x40, 0x80} { // compressed and uncompressed sizes - if flags&present != 0 { - _, n := readVarint(header[pos:]) - if n == 0 { - return errors.New("bad block header") - } - pos += n - } - } - last := uint64(0) - for range int(flags&0x03) + 1 { - id, n := readVarint(header[pos:]) - if n == 0 { - return errors.New("bad filter flags") - } - pos += n - propsSize, n := readVarint(header[pos:]) - if n == 0 || uint64(len(header)-pos-n) < propsSize { - return errors.New("bad filter flags") - } - pos += n - props := header[pos : pos+int(propsSize)] - pos += int(propsSize) - last = id - if id != lzma2Filter { - continue - } - if len(props) != 1 || props[0] > 40 { - return errors.New("bad LZMA2 properties") - } - if dict := lzma2DictSize(props[0]); dict > maxDict { - return fmt.Errorf("LZMA2 dictionary of %d bytes is larger than %d", dict, maxDict) - } - } - if last != lzma2Filter { - return errors.New("last filter is not LZMA2") - } - return nil -} - -// lzma2DictSize decodes the LZMA2 dictionary size property, 0 to 40. -func lzma2DictSize(bits byte) uint64 { - if bits == 40 { - return 1<<32 - 1 - } - return uint64(2|bits&1) << (bits/2 + 11) -} - -// readVarint reads an xz multibyte integer, returning 0 bytes read when it -// is malformed or truncated. -func readVarint(b []byte) (uint64, int) { - var v uint64 - for i := 0; i < len(b) && i < maxVarint; i++ { - v |= uint64(b[i]&0x7F) << (7 * i) - if b[i]&0x80 == 0 { - return v, i + 1 - } - } - return 0, 0 -} diff --git a/internal/deb/testdata/bookworm-updates-Packages.xz b/internal/xzlimit/testdata/bookworm-updates-Packages.xz similarity index 100% rename from internal/deb/testdata/bookworm-updates-Packages.xz rename to internal/xzlimit/testdata/bookworm-updates-Packages.xz diff --git a/internal/xzlimit/xzlimit.go b/internal/xzlimit/xzlimit.go new file mode 100644 index 0000000..65a9351 --- /dev/null +++ b/internal/xzlimit/xzlimit.go @@ -0,0 +1,435 @@ +// Package xzlimit decodes .xz files with a bound on the memory a file can +// make the decoder allocate. +// +// An .xz block header names the LZMA2 dictionary its data needs, up to +// 4 GiB, and github.com/ulikunitz/xz allocates what it names: its DictCap +// is a minimum, not a limit (ulikunitz/xz#84). The header cannot be checked +// ahead of decoding either, since where a block's data ends is only known +// by decoding it. So this package reads the .xz container itself, refuses +// a block whose dictionary is over the limit as it reaches it, and hands +// the block's data to ulikunitz/xz's LZMA2 decoder with exactly the +// dictionary the block asked for. +package xzlimit + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash" + "hash/crc32" + "hash/crc64" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// ErrMalformed is returned for a file that is not a well-formed .xz file +// this package can read, or whose blocks ask for too large a dictionary. +var ErrMalformed = errors.New("malformed xz file") + +// MaxDict is the dictionary size callers allow by default. xz -9 uses +// 64 MiB; Debian's indexes use 8 MiB. +const MaxDict = 64 << 20 + +const ( + headerSize = 12 + footerSize = 12 + lzma2Filter = 0x21 + maxVarint = 9 + + checkNone = 0x00 + checkCRC32 = 0x01 + checkCRC64 = 0x04 + checkSHA256 = 0x0A +) + +var ( + headerMagic = []byte{0xFD, '7', 'z', 'X', 'Z', 0x00} + footerMagic = []byte{'Y', 'Z'} + crc64Table = crc64.MakeTable(crc64.ECMA) +) + +// Reader decodes an .xz file: one or more streams, each of blocks whose +// only filter is LZMA2, as xz writes them by default. +type Reader struct { + src *source + maxDict uint64 + err error + + streams int + inStream bool + checkType byte + flags [2]byte + records []record // the blocks of the current stream + + // The block being decoded. + block io.Reader + check hash.Hash + blockStart int64 // offset of the block header + produced int64 +} + +// record is what the index lists for a block. +type record struct { + unpadded int64 + uncompressed int64 +} + +// NewReader returns a Reader that refuses any block asking for a +// dictionary larger than maxDict bytes. +func NewReader(r io.Reader, maxDict uint64) *Reader { + return &Reader{src: &source{r: bufio.NewReader(r)}, maxDict: maxDict} +} + +// source counts the bytes read, which gives the size of each block. It is +// an io.ByteReader so the LZMA2 decoder reads no further than it decodes. +type source struct { + r *bufio.Reader + n int64 +} + +func (s *source) Read(p []byte) (int, error) { + n, err := s.r.Read(p) + s.n += int64(n) + return n, err +} + +func (s *source) ReadByte() (byte, error) { + b, err := s.r.ReadByte() + if err == nil { + s.n++ + } + return b, err +} + +// full reads exactly n bytes; the file ending inside them is malformed. +func (s *source) full(n int) ([]byte, error) { + buf := make([]byte, n) + if _, err := io.ReadFull(s, buf); err != nil { + return nil, early(err) + } + return buf, nil +} + +// early turns the end of the file, where more was expected, into an error. +func early(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + return err +} + +func (r *Reader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + for r.err == nil { + if r.block == nil { + r.err = r.next() + continue + } + n, err := r.block.Read(p) + if n > 0 { + _, _ = r.check.Write(p[:n]) + r.produced += int64(n) + return n, nil + } + if err == io.EOF { + err = r.finishBlock() + } + r.err = err + } + if r.err == io.EOF { + return 0, io.EOF + } + // Both are kept: a caller may look for the source's error, such as a + // size limit, as well as for a malformed file. + if !errors.Is(r.err, ErrMalformed) { + r.err = fmt.Errorf("%w: %w", ErrMalformed, r.err) + } + return 0, r.err +} + +// next reads up to the next block's data: a stream header, a block header, +// or the index and footer that end a stream. It returns io.EOF at the end +// of the file. +func (r *Reader) next() error { + if !r.inStream { + return r.startStream() + } + first, err := r.src.ReadByte() + if err != nil { + return early(err) + } + if first == 0 { + return r.endStream() + } + return r.startBlock(first) +} + +// startStream skips stream padding and reads a stream header. +func (r *Reader) startStream() error { + for { + peek, err := r.src.r.Peek(1) + if err == io.EOF && r.streams > 0 { + return io.EOF + } + if err != nil { + return early(err) + } + if peek[0] != 0 || r.streams == 0 { + break + } + // Stream padding comes in multiples of four zero bytes. + padding, err := r.src.full(4) + if err != nil { + return err + } + if !bytes.Equal(padding, []byte{0, 0, 0, 0}) { + return errors.New("bad stream padding") + } + } + header, err := r.src.full(headerSize) + if err != nil { + return err + } + if !bytes.Equal(header[:6], headerMagic) { + return errors.New("no stream header") + } + if crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:]) { + return errors.New("stream header checksum") + } + if header[6] != 0 { + return errors.New("unsupported stream flags") + } + switch header[7] { + case checkNone, checkCRC32, checkCRC64, checkSHA256: + default: + return fmt.Errorf("unsupported check type %#x", header[7]) + } + r.flags = [2]byte{header[6], header[7]} + r.checkType = header[7] + r.records = r.records[:0] + r.inStream = true + r.streams++ + return nil +} + +// startBlock reads the block header whose first byte is sizeByte and +// starts decoding the block's data. +func (r *Reader) startBlock(sizeByte byte) error { + r.blockStart = r.src.n - 1 + size := (int(sizeByte) + 1) * 4 + rest, err := r.src.full(size - 1) + if err != nil { + return err + } + header := append([]byte{sizeByte}, rest...) + if crc32.ChecksumIEEE(header[:size-4]) != binary.LittleEndian.Uint32(header[size-4:]) { + return errors.New("block header checksum") + } + dict, err := blockDictSize(header[1 : size-4]) + if err != nil { + return err + } + if dict > r.maxDict { + return fmt.Errorf("block asks for a dictionary of %d bytes, more than %d", dict, r.maxDict) + } + // Exactly what the block asked for, never more. The smallest size the + // format can name, 4096, is also the decoder's minimum. + block, err := lzma.Reader2Config{DictCap: int(dict)}.NewReader2(r.src) + if err != nil { + return err + } + r.block = block + r.check = newCheck(r.checkType) + r.produced = 0 + return nil +} + +// blockDictSize reads a block header's fields, without its size byte and +// checksum, and returns the dictionary size of its LZMA2 filter. +func blockDictSize(fields []byte) (uint64, error) { + flags := fields[0] + if flags&0x3C != 0 { + return 0, errors.New("reserved block flags") + } + if flags&0x03 != 0 { + return 0, errors.New("unsupported filter chain: more than one filter") + } + pos := 1 + for _, present := range []byte{0x40, 0x80} { // compressed and uncompressed sizes + if flags&present != 0 { + _, n := uvarint(fields[pos:]) + if n == 0 { + return 0, errors.New("bad block header") + } + pos += n + } + } + // One filter: LZMA2, with one byte of properties. + if len(fields)-pos < 3 || fields[pos] != lzma2Filter || fields[pos+1] != 1 { + return 0, errors.New("unsupported filter chain: not LZMA2") + } + bits := fields[pos+2] + if bits > 40 { + return 0, errors.New("bad LZMA2 properties") + } + for _, b := range fields[pos+3:] { + if b != 0 { + return 0, errors.New("bad block header padding") + } + } + return dictSize(bits), nil +} + +// dictSize decodes the LZMA2 dictionary size property, 0 to 40. +func dictSize(bits byte) uint64 { + if bits == 40 { + return 1<<32 - 1 + } + return uint64(2|bits&1) << (bits/2 + 11) +} + +func newCheck(checkType byte) hash.Hash { + switch checkType { + case checkCRC32: + return crc32.NewIEEE() + case checkCRC64: + return crc64.New(crc64Table) + case checkSHA256: + return sha256.New() + } + return noCheck{} +} + +// noCheck is the check of a stream that has none. +type noCheck struct{ hash.Hash } + +func (noCheck) Write(p []byte) (int, error) { return len(p), nil } +func (noCheck) Sum([]byte) []byte { return nil } + +// finishBlock reads what follows a block's data: its padding and check. +func (r *Reader) finishBlock() error { + unpadded := r.src.n - r.blockStart + if padding := int((4 - unpadded%4) % 4); padding > 0 { + zeros, err := r.src.full(padding) + if err != nil { + return err + } + if !bytes.Equal(zeros, make([]byte, padding)) { + return errors.New("bad block padding") + } + } + want := r.check.Sum(nil) + // CRCs are stored little-endian, and hash.Hash gives them big-endian. + if r.checkType == checkCRC32 || r.checkType == checkCRC64 { + for i, j := 0, len(want)-1; i < j; i, j = i+1, j-1 { + want[i], want[j] = want[j], want[i] + } + } + got, err := r.src.full(len(want)) + if err != nil { + return err + } + if !bytes.Equal(got, want) { + return errors.New("block check mismatch") + } + r.records = append(r.records, record{unpadded: unpadded + int64(len(want)), uncompressed: r.produced}) + r.block = nil + return nil +} + +// endStream reads the index, whose indicator byte has been read, and the +// stream footer, and checks them against the blocks that were decoded. +func (r *Reader) endStream() error { + indexStart := r.src.n - 1 + crc := crc32.NewIEEE() + _, _ = crc.Write([]byte{0}) + in := io.TeeReader(r.src, crc) + readUvarint := func() (uint64, error) { + var buf [maxVarint]byte + for i := range buf { + if _, err := io.ReadFull(in, buf[i:i+1]); err != nil { + return 0, early(err) + } + if buf[i]&0x80 == 0 { + v, _ := uvarint(buf[:i+1]) + return v, nil + } + } + return 0, errors.New("bad index") + } + + count, err := readUvarint() + if err != nil { + return err + } + if count != uint64(len(r.records)) { + return fmt.Errorf("index lists %d blocks, the stream has %d", count, len(r.records)) + } + for _, rec := range r.records { + unpadded, err := readUvarint() + if err != nil { + return err + } + uncompressed, err := readUvarint() + if err != nil { + return err + } + if unpadded != uint64(rec.unpadded) || uncompressed != uint64(rec.uncompressed) { + return errors.New("index does not match the blocks") + } + } + if padding := int((4 - (r.src.n-indexStart)%4) % 4); padding > 0 { + zeros := make([]byte, padding) + if _, err := io.ReadFull(in, zeros); err != nil { + return early(err) + } + if !bytes.Equal(zeros, make([]byte, padding)) { + return errors.New("bad index padding") + } + } + sum := crc.Sum32() + stored, err := r.src.full(4) + if err != nil { + return err + } + if binary.LittleEndian.Uint32(stored) != sum { + return errors.New("index checksum") + } + indexSize := r.src.n - indexStart + + footer, err := r.src.full(footerSize) + if err != nil { + return err + } + switch { + case !bytes.Equal(footer[10:], footerMagic): + return errors.New("no stream footer") + case crc32.ChecksumIEEE(footer[4:10]) != binary.LittleEndian.Uint32(footer): + return errors.New("stream footer checksum") + case int64(binary.LittleEndian.Uint32(footer[4:8])+1)*4 != indexSize: + return errors.New("stream footer does not match the index") + case footer[8] != r.flags[0] || footer[9] != r.flags[1]: + return errors.New("stream footer does not match the header") + } + r.inStream = false + return nil +} + +// uvarint decodes an xz multibyte integer, returning 0 bytes read when it +// is malformed or truncated. +func uvarint(b []byte) (uint64, int) { + var v uint64 + for i := 0; i < len(b) && i < maxVarint; i++ { + v |= uint64(b[i]&0x7F) << (7 * i) + if b[i]&0x80 == 0 { + return v, i + 1 + } + } + return 0, 0 +} diff --git a/internal/xzlimit/xzlimit_test.go b/internal/xzlimit/xzlimit_test.go new file mode 100644 index 0000000..6107168 --- /dev/null +++ b/internal/xzlimit/xzlimit_test.go @@ -0,0 +1,401 @@ +package xzlimit + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "io" + "os" + "strings" + "testing" + + "github.com/ulikunitz/xz" +) + +const sample = "Package: curl\nVersion: 7.88.1-10\nArchitecture: amd64\n\n" + +func compress(t *testing.T, config xz.WriterConfig, parts ...[]byte) []byte { + t.Helper() + var buf bytes.Buffer + w, err := config.NewWriter(&buf) + if err != nil { + t.Fatal(err) + } + for _, part := range parts { + if _, err := w.Write(part); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// noise is bytes that don't compress, which LZMA2 stores in stored chunks. +func noise(n int) []byte { + out := make([]byte, n) + state := uint32(1) + for i := range out { + state = state*1664525 + 1013904223 + out[i] = byte(state >> 24) + } + return out +} + +func decode(data []byte, maxDict uint64) ([]byte, error) { + return io.ReadAll(NewReader(bytes.NewReader(data), maxDict)) +} + +// reference decodes with the library's own reader. +func reference(t *testing.T, data []byte) []byte { + t.Helper() + r, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + out, err := io.ReadAll(r) + if err != nil { + t.Fatalf("the reference reader fails: %v", err) + } + return out +} + +func TestDecodesWhatTheLibraryDecodes(t *testing.T) { + debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") + if err != nil { + t.Fatal(err) + } + text := []byte(strings.Repeat(sample, 5000)) + mixed := [][]byte{text, noise(200_000), []byte(strings.Repeat("x", 300_000))} + for name, data := range map[string][]byte{ + "debian's own index": debian, + "one small block": compress(t, xz.WriterConfig{}, []byte(sample)), + "empty": compress(t, xz.WriterConfig{}), + "several blocks": compress(t, xz.WriterConfig{BlockSize: 64 << 10}, mixed...), + "crc32 check": compress(t, xz.WriterConfig{CheckSum: xz.CRC32, BlockSize: 64 << 10}, mixed...), + "sha256 check": compress(t, xz.WriterConfig{CheckSum: xz.SHA256, BlockSize: 64 << 10}, mixed...), + "no check": compress(t, xz.WriterConfig{NoCheckSum: true, BlockSize: 64 << 10}, mixed...), + "small dictionary": compress(t, xz.WriterConfig{DictCap: 4096}, text), + "two streams": append(compress(t, xz.WriterConfig{}, text), compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample))...), + "two padded streams": bytes.Join([][]byte{compress(t, xz.WriterConfig{}, text), make([]byte, 8), compress(t, xz.WriterConfig{}, []byte(sample)), make([]byte, 4)}, nil), + } { + t.Run(name, func(t *testing.T) { + want := reference(t, data) + got, err := decode(data, MaxDict) + if err != nil { + t.Fatalf("decode: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("decoded %d bytes, the library decodes %d; they differ", len(got), len(want)) + } + }) + } +} + +// TestReadsInSmallPieces checks nothing depends on how much a Read asks for. +func TestReadsInSmallPieces(t *testing.T) { + data := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 500)), noise(9000)) + want := reference(t, data) + r := NewReader(iotestOneByte{bytes.NewReader(data)}, MaxDict) + var got []byte + buf := make([]byte, 7) + for { + n, err := r.Read(buf) + got = append(got, buf[:n]...) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Read: %v", err) + } + } + if !bytes.Equal(got, want) { + t.Errorf("decoded %d bytes, want %d", len(got), len(want)) + } + if n, err := r.Read(nil); n != 0 || err != nil { + t.Errorf("Read(nil) = %d, %v", n, err) + } +} + +// iotestOneByte hands out one byte per Read. +type iotestOneByte struct{ r io.Reader } + +func (o iotestOneByte) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + return o.r.Read(p[:1]) +} + +// setDict rewrites the dictionary size of the block header at offset in +// data, fixing the header's checksum, so a small block claims a large +// dictionary. +func setDict(t *testing.T, data []byte, offset int, bits byte) []byte { + t.Helper() + out := bytes.Clone(data) + size := (int(out[offset]) + 1) * 4 + header := out[offset : offset+size] + i := bytes.Index(header, []byte{lzma2Filter, 0x01}) + if i < 0 { + t.Fatal("no LZMA2 filter in the block header") + } + header[i+2] = bits + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} + +// blockOffsets returns where each block header of a one-stream file starts, +// from the sizes its index lists. +func blockOffsets(t *testing.T, data []byte) []int { + t.Helper() + r := NewReader(bytes.NewReader(data), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil { + t.Fatal(err) + } + var offsets []int + offset := headerSize + for _, rec := range r.records { + offsets = append(offsets, offset) + offset += int((rec.unpadded + 3) &^ 3) + } + return offsets +} + +func TestDictionaryLimit(t *testing.T) { + small := compress(t, xz.WriterConfig{}, []byte(sample)) + // 128 MiB, past the limit. + if _, err := decode(setDict(t, small, headerSize, 30), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "dictionary") { + t.Errorf("128 MiB dictionary: err = %v, want ErrMalformed naming the dictionary", err) + } + // The limit itself is allowed: 64 MiB is property 28. + if dictSize(28) != MaxDict { + t.Fatalf("dictSize(28) = %d", dictSize(28)) + } + if _, err := decode(small, 8<<20); err != nil { + t.Errorf("a block at the limit: %v", err) + } + if _, err := decode(small, 8<<20-1); !errors.Is(err, ErrMalformed) { + t.Errorf("a block one byte over the limit: err = %v, want ErrMalformed", err) + } + + // A later block is held to the limit when the reader gets to it, after + // the blocks before it were decoded. + blocks := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 500))) + offsets := blockOffsets(t, blocks) + if len(offsets) < 3 { + t.Fatalf("the fixture has %d blocks, want several", len(offsets)) + } + got, err := decode(setDict(t, blocks, offsets[2], 38), MaxDict) + if !errors.Is(err, ErrMalformed) { + t.Errorf("large third block: err = %v, want ErrMalformed", err) + } + if len(got) != 2*(4<<10) { + t.Errorf("decoded %d bytes before the refused block, want the two blocks before it", len(got)) + } + // And one in a second stream. + two := bytes.Join([][]byte{small, make([]byte, 4), setDict(t, small, headerSize, 38)}, nil) + if got, err := decode(two, MaxDict); !errors.Is(err, ErrMalformed) || string(got) != sample { + t.Errorf("large block in a second stream: %q, %v", got, err) + } +} + +func TestDictSize(t *testing.T) { + for bits, want := range map[byte]uint64{0: 4096, 1: 6144, 18: 2 << 20, 19: 3 << 20, 22: 8 << 20, 40: 1<<32 - 1} { + if got := dictSize(bits); got != want { + t.Errorf("dictSize(%d) = %d, want %d", bits, got, want) + } + } +} + +// fixCRC recomputes the CRC32 that ends a header of size bytes at offset. +func fixCRC(data []byte, offset, size int) { + binary.LittleEndian.PutUint32(data[offset+size-4:], crc32.ChecksumIEEE(data[offset:offset+size-4])) +} + +func TestMalformedFiles(t *testing.T) { + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(strings.Repeat(sample, 20))) + blockHeaderSize := (int(valid[headerSize]) + 1) * 4 + patch := func(at int, b ...byte) []byte { + out := bytes.Clone(valid) + if at < 0 { + at += len(out) + } + copy(out[at:], b) + return out + } + // The block header: size, flags, filter id, properties size, properties. + header := func(edit func(h []byte)) []byte { + out := bytes.Clone(valid) + edit(out[headerSize : headerSize+blockHeaderSize]) + fixCRC(out, headerSize, blockHeaderSize) + return out + } + streamFlags := func(flag0, checkType byte) []byte { + out := patch(6, flag0, checkType) + binary.LittleEndian.PutUint32(out[8:], crc32.ChecksumIEEE(out[6:8])) + copy(out[len(out)-4:], []byte{flag0, checkType}) + binary.LittleEndian.PutUint32(out[len(out)-12:], crc32.ChecksumIEEE(out[len(out)-8:len(out)-2])) + return out + } + + for name, data := range map[string][]byte{ + "empty": nil, + "not xz": []byte(strings.Repeat("plain text ", 10)), + "header magic": patch(0, 0xFE), + "header checksum": patch(8, valid[8]^1), + "stream flags": streamFlags(1, checkCRC32), + "unsupported check": streamFlags(0, 0x02), + "block header checksum": patch(headerSize+blockHeaderSize-1, valid[headerSize+blockHeaderSize-1]^1), + "reserved block flags": header(func(h []byte) { h[1] = 0x04 }), + "two filters": header(func(h []byte) { h[1] = 0x01 }), + "filter not lzma2": header(func(h []byte) { h[2] = 0x03 }), + "properties size": header(func(h []byte) { h[3] = 0x02 }), + // 200 would shift the size to nothing and read as a tiny dictionary. + "properties past 40": header(func(h []byte) { h[4] = 200 }), + "block header padding": header(func(h []byte) { h[5] = 1 }), + "corrupt data": patch(headerSize+blockHeaderSize+20, valid[headerSize+blockHeaderSize+20]^0xFF), + "block check": patch(-12-8-4, valid[len(valid)-12-8-4]^1), + "index checksum": patch(-12-1, valid[len(valid)-12-1]^1), + "footer magic": patch(-2, 'Y', 'X'), + "footer checksum": patch(-12, valid[len(valid)-12]^1), + "garbage after the stream": append(bytes.Clone(valid), "garbage!"...), + "padding of two bytes": append(bytes.Clone(valid), 0, 0), + "padding with a stray byte": bytes.Join([][]byte{valid, {0, 1, 0, 0}, valid}, nil), + "padding before the stream": append(make([]byte, 4), valid...), + } { + t.Run(name, func(t *testing.T) { + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("err = %v, want ErrMalformed", err) + } + }) + } +} + +// The index and footer must describe the blocks that were decoded. +func TestIndexAndFooterAreChecked(t *testing.T) { + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(strings.Repeat(sample, 20))) + // A one-block index is 8 bytes when both sizes fit a byte or two: + // find it from the footer's backward size. + indexSize := (int(binary.LittleEndian.Uint32(valid[len(valid)-8:])) + 1) * 4 + indexStart := len(valid) - footerSize - indexSize + reindex := func(edit func(index []byte)) []byte { + out := bytes.Clone(valid) + index := out[indexStart : indexStart+indexSize] + edit(index) + binary.LittleEndian.PutUint32(index[indexSize-4:], crc32.ChecksumIEEE(index[:indexSize-4])) + return out + } + refooter := func(edit func(footer []byte)) []byte { + out := bytes.Clone(valid) + footer := out[len(out)-footerSize:] + edit(footer) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return out + } + for name, data := range map[string][]byte{ + "record count": reindex(func(index []byte) { index[1] = 2 }), + "no records": reindex(func(index []byte) { index[1] = 0 }), + "unpadded size": reindex(func(index []byte) { index[2]++ }), + "uncompressed size": reindex(func(index []byte) { index[indexSize-5-1]++ }), + "backward size": refooter(func(footer []byte) { footer[4]++ }), + "footer flags": refooter(func(footer []byte) { footer[9] = checkCRC64 }), + "footer reserved flags": refooter(func(footer []byte) { footer[8] = 1 }), + } { + t.Run(name, func(t *testing.T) { + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) { + t.Errorf("err = %v, want ErrMalformed", err) + } + }) + } +} + +// Every truncation of a file is an error, never a short result. +func TestTruncatedFiles(t *testing.T) { + debian, err := os.ReadFile("testdata/bookworm-updates-Packages.xz") + if err != nil { + t.Fatal(err) + } + blocks := compress(t, xz.WriterConfig{BlockSize: 4 << 10}, []byte(strings.Repeat(sample, 300)), noise(5000)) + for name, data := range map[string][]byte{"debian": debian, "several blocks": blocks} { + step := 1 + if len(data) > 4000 { + step = 7 + } + for n := 0; n < len(data); n += step { + if _, err := decode(data[:n], MaxDict); !errors.Is(err, ErrMalformed) { + t.Fatalf("%s cut to %d of %d bytes: err = %v, want ErrMalformed", name, n, len(data), err) + } + } + } +} + +// An error from the source is still found in the error returned. +func TestSourceErrorIsKept(t *testing.T) { + valid := compress(t, xz.WriterConfig{}, noise(50_000)) + boom := errors.New("response too large") + for _, cut := range []int{0, 5, headerSize, headerSize + 3, 2000, len(valid) - 20, len(valid) - 3} { + r := NewReader(io.MultiReader(bytes.NewReader(valid[:cut]), failing{boom}), MaxDict) + if _, err := io.Copy(io.Discard, r); !errors.Is(err, boom) { + t.Errorf("source failing after %d bytes: err = %v, want it to wrap the source's error", cut, err) + } + } +} + +type failing struct{ err error } + +func (f failing) Read([]byte) (int, error) { return 0, f.err } + +func TestUnsupportedCheckIsNamed(t *testing.T) { + data := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample)) + data[7] = 0x02 + binary.LittleEndian.PutUint32(data[8:], crc32.ChecksumIEEE(data[6:8])) + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "unsupported check type") { + t.Errorf("err = %v, want ErrMalformed naming the check type", err) + } +} + +func TestUvarint(t *testing.T) { + for _, tt := range []struct { + in []byte + want uint64 + n int + }{ + {[]byte{0x00}, 0, 1}, + {[]byte{0x7F, 0xFF}, 127, 1}, + {[]byte{0x80, 0x01}, 128, 2}, + {[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}, 1<<63 - 1, 9}, + {[]byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00}, 0, 0}, // ten bytes + {[]byte{0x80}, 0, 0}, + {nil, 0, 0}, + } { + if got, n := uvarint(tt.in); got != tt.want || n != tt.n { + t.Errorf("uvarint(%x) = %d, %d; want %d, %d", tt.in, got, n, tt.want, tt.n) + } + } +} + +// An index whose record count is an overlong integer is refused, here in a +// stream with no blocks, where the count would otherwise read as zero. +func TestOverlongIndexCount(t *testing.T) { + some := compress(t, xz.WriterConfig{}, []byte(sample)) + // A stream with no blocks: the header, an index and a footer. + stream := func(index []byte) []byte { + for len(index)%4 != 0 { + index = append(index, 0) + } + index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index)) + footer := bytes.Clone(some[len(some)-footerSize:]) + binary.LittleEndian.PutUint32(footer[4:], uint32(len(index)/4-1)) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return bytes.Join([][]byte{some[:headerSize], index, footer}, nil) + } + if got, err := decode(stream([]byte{0x00, 0x00}), MaxDict); err != nil || len(got) != 0 { + t.Fatalf("a stream with no blocks: %q, %v", got, err) + } + overlong := append([]byte{0x00}, bytes.Repeat([]byte{0x80}, 9)...) + if _, err := decode(stream(overlong), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "bad index") { + t.Errorf("err = %v, want ErrMalformed for the index", err) + } +} From de645a14ebe7ffc05d3749b65ca031bf69152f6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 13 Nov 2026 17:36:21 +0100 Subject: [PATCH 7/7] Hold a block to the sizes its header declares A block header may declare the compressed and uncompressed size of its data. xzlimit read past them; a block whose data has another size is now refused. --- internal/xzlimit/xzlimit.go | 58 +++++++++++----- internal/xzlimit/xzlimit_test.go | 112 +++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 17 deletions(-) diff --git a/internal/xzlimit/xzlimit.go b/internal/xzlimit/xzlimit.go index 65a9351..33c7bed 100644 --- a/internal/xzlimit/xzlimit.go +++ b/internal/xzlimit/xzlimit.go @@ -69,9 +69,18 @@ type Reader struct { block io.Reader check hash.Hash blockStart int64 // offset of the block header + dataStart int64 // offset of the block's data + declared blockSizes produced int64 } +// blockSizes are the sizes a block header declares for its data, or -1 +// for one it leaves out. +type blockSizes struct { + compressed int64 + uncompressed int64 +} + // record is what the index lists for a block. type record struct { unpadded int64 @@ -231,7 +240,7 @@ func (r *Reader) startBlock(sizeByte byte) error { if crc32.ChecksumIEEE(header[:size-4]) != binary.LittleEndian.Uint32(header[size-4:]) { return errors.New("block header checksum") } - dict, err := blockDictSize(header[1 : size-4]) + dict, declared, err := blockFields(header[1 : size-4]) if err != nil { return err } @@ -246,44 +255,53 @@ func (r *Reader) startBlock(sizeByte byte) error { } r.block = block r.check = newCheck(r.checkType) + r.dataStart = r.blockStart + int64(size) + r.declared = declared r.produced = 0 return nil } -// blockDictSize reads a block header's fields, without its size byte and -// checksum, and returns the dictionary size of its LZMA2 filter. -func blockDictSize(fields []byte) (uint64, error) { +// blockFields reads a block header's fields, without its size byte and +// checksum, and returns the dictionary size of its LZMA2 filter and the +// sizes it declares for the block's data. +func blockFields(fields []byte) (uint64, blockSizes, error) { + sizes := blockSizes{compressed: -1, uncompressed: -1} flags := fields[0] if flags&0x3C != 0 { - return 0, errors.New("reserved block flags") + return 0, sizes, errors.New("reserved block flags") } if flags&0x03 != 0 { - return 0, errors.New("unsupported filter chain: more than one filter") + return 0, sizes, errors.New("unsupported filter chain: more than one filter") } pos := 1 - for _, present := range []byte{0x40, 0x80} { // compressed and uncompressed sizes - if flags&present != 0 { - _, n := uvarint(fields[pos:]) - if n == 0 { - return 0, errors.New("bad block header") - } - pos += n + for _, field := range []struct { + present byte + size *int64 + }{{0x40, &sizes.compressed}, {0x80, &sizes.uncompressed}} { + if flags&field.present == 0 { + continue + } + v, n := uvarint(fields[pos:]) + if n == 0 { + return 0, sizes, errors.New("bad block header") } + *field.size = int64(v) + pos += n } // One filter: LZMA2, with one byte of properties. if len(fields)-pos < 3 || fields[pos] != lzma2Filter || fields[pos+1] != 1 { - return 0, errors.New("unsupported filter chain: not LZMA2") + return 0, sizes, errors.New("unsupported filter chain: not LZMA2") } bits := fields[pos+2] if bits > 40 { - return 0, errors.New("bad LZMA2 properties") + return 0, sizes, errors.New("bad LZMA2 properties") } for _, b := range fields[pos+3:] { if b != 0 { - return 0, errors.New("bad block header padding") + return 0, sizes, errors.New("bad block header padding") } } - return dictSize(bits), nil + return dictSize(bits), sizes, nil } // dictSize decodes the LZMA2 dictionary size property, 0 to 40. @@ -314,6 +332,12 @@ func (noCheck) Sum([]byte) []byte { return nil } // finishBlock reads what follows a block's data: its padding and check. func (r *Reader) finishBlock() error { + if d := r.declared.compressed; d >= 0 && d != r.src.n-r.dataStart { + return fmt.Errorf("block header declares %d bytes of data, the block has %d", d, r.src.n-r.dataStart) + } + if d := r.declared.uncompressed; d >= 0 && d != r.produced { + return fmt.Errorf("block header declares %d bytes uncompressed, the block has %d", d, r.produced) + } unpadded := r.src.n - r.blockStart if padding := int((4 - unpadded%4) % 4); padding > 0 { zeros, err := r.src.full(padding) diff --git a/internal/xzlimit/xzlimit_test.go b/internal/xzlimit/xzlimit_test.go index 6107168..47818d3 100644 --- a/internal/xzlimit/xzlimit_test.go +++ b/internal/xzlimit/xzlimit_test.go @@ -399,3 +399,115 @@ func TestOverlongIndexCount(t *testing.T) { t.Errorf("err = %v, want ErrMalformed for the index", err) } } + +// withBlockSizes rebuilds a one-block CRC32 file so its block header +// declares these sizes (nil leaves one out), with the index and footer +// rewritten for the longer header. +func withBlockSizes(t *testing.T, valid []byte, compressed, uncompressed *uint64) []byte { + t.Helper() + r := NewReader(bytes.NewReader(valid), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil || len(r.records) != 1 { + t.Fatalf("the fixture is not a one-block file: %v", err) + } + const checkSize = 4 + oldHeaderSize := (int(valid[headerSize]) + 1) * 4 + dataSize := int(r.records[0].unpadded) - oldHeaderSize - checkSize + data := valid[headerSize+oldHeaderSize : headerSize+oldHeaderSize+dataSize] + blockEnd := headerSize + int((r.records[0].unpadded+3)&^3) + check := valid[blockEnd-checkSize : blockEnd] + + appendUvarint := func(b []byte, v uint64) []byte { + for v >= 0x80 { + b = append(b, byte(v)|0x80) + v >>= 7 + } + return append(b, byte(v)) + } + header := []byte{0, 0} + if compressed != nil { + header[1] |= 0x40 + header = appendUvarint(header, *compressed) + } + if uncompressed != nil { + header[1] |= 0x80 + header = appendUvarint(header, *uncompressed) + } + header = append(header, lzma2Filter, 0x01, valid[headerSize+4]) + for len(header)%4 != 0 { + header = append(header, 0) + } + header[0] = byte(len(header)/4 + 1 - 1) + header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header)) + + block := bytes.Join([][]byte{header, data}, nil) + unpadded := len(block) + checkSize + for len(block)%4 != 0 { + block = append(block, 0) + } + block = append(block, check...) + + index := appendUvarint([]byte{0x00, 0x01}, uint64(unpadded)) + index = appendUvarint(index, uint64(r.records[0].uncompressed)) + for len(index)%4 != 0 { + index = append(index, 0) + } + index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index)) + footer := bytes.Clone(valid[len(valid)-footerSize:]) + binary.LittleEndian.PutUint32(footer[4:], uint32(len(index)/4-1)) + binary.LittleEndian.PutUint32(footer, crc32.ChecksumIEEE(footer[4:10])) + return bytes.Join([][]byte{valid[:headerSize], block, index, footer}, nil) +} + +// A block header may declare the sizes of its data; they must be the sizes +// the block turns out to have. +func TestDeclaredBlockSizes(t *testing.T) { + text := strings.Repeat(sample, 20) + valid := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(text)) + r := NewReader(bytes.NewReader(valid), MaxDict) + if _, err := io.Copy(io.Discard, r); err != nil { + t.Fatal(err) + } + blockHeaderSize := (int(valid[headerSize]) + 1) * 4 + compressed := uint64(r.records[0].unpadded) - uint64(blockHeaderSize) - 4 + uncompressed := uint64(len(text)) + ptr := func(v uint64) *uint64 { return &v } + + for name, sizes := range map[string][2]*uint64{ + "both": {ptr(compressed), ptr(uncompressed)}, + "compressed only": {ptr(compressed), nil}, + "uncompressed only": {nil, ptr(uncompressed)}, + } { + data := withBlockSizes(t, valid, sizes[0], sizes[1]) + if got := reference(t, data); string(got) != text { + t.Fatalf("%s: the fixture does not decode with the library", name) + } + if got, err := decode(data, MaxDict); err != nil || string(got) != text { + t.Errorf("%s: decoded %d bytes, %v", name, len(got), err) + } + } + for name, sizes := range map[string][2]*uint64{ + "compressed too small": {ptr(compressed - 1), ptr(uncompressed)}, + "compressed too large": {ptr(compressed + 1), nil}, + "uncompressed zero": {nil, ptr(0)}, + "uncompressed too large": {ptr(compressed), ptr(uncompressed + 1)}, + } { + if _, err := decode(withBlockSizes(t, valid, sizes[0], sizes[1]), MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "block header") { + t.Errorf("%s: err = %v, want ErrMalformed for the block header's size", name, err) + } + } +} + +// A declared size that never ends is named as the header's fault. +func TestUnterminatedBlockSize(t *testing.T) { + data := compress(t, xz.WriterConfig{CheckSum: xz.CRC32}, []byte(sample)) + size := (int(data[headerSize]) + 1) * 4 + header := data[headerSize : headerSize+size] + header[1] = 0x40 + for i := 2; i < size-4; i++ { + header[i] = 0x80 + } + fixCRC(data, headerSize, size) + if _, err := decode(data, MaxDict); !errors.Is(err, ErrMalformed) || !strings.Contains(err.Error(), "bad block header") { + t.Errorf("err = %v, want ErrMalformed for the block header", err) + } +}