From 6e2fbe9c7d5c5a69e6a0ee4f2331f6776593081c Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Wed, 19 Aug 2026 00:26:19 +0000
Subject: [PATCH 01/10] debian copyright mirror, but in go

---
 .../debian-copyright-mirror/Dockerfile        |  18 +-
 .../mirrors/debian-copyright-mirror/main.go   | 415 ++++++++++++++++++
 .../debian-copyright-mirror/main_test.go      | 193 ++++++++
 vulnfeeds/gcs-tools/gcs.go                    |  13 +
 vulnfeeds/gcs-tools/gcs_test.go               |  47 ++
 vulnfeeds/utility/archive.go                  | 118 +++++
 vulnfeeds/utility/archive_test.go             |  69 +++
 vulnfeeds/utility/decompress.go               | 127 ++++++
 vulnfeeds/utility/decompress_test.go          | 108 +++++
 vulnfeeds/utility/download.go                 | 102 +++++
 vulnfeeds/utility/download_test.go            |  66 +++
 11 files changed, 1273 insertions(+), 3 deletions(-)
 create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
 create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
 create mode 100644 vulnfeeds/utility/archive.go
 create mode 100644 vulnfeeds/utility/archive_test.go
 create mode 100644 vulnfeeds/utility/decompress.go
 create mode 100644 vulnfeeds/utility/decompress_test.go
 create mode 100644 vulnfeeds/utility/download.go
 create mode 100644 vulnfeeds/utility/download_test.go

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
index 09afd30ba3f..6ef0187b35d 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
@@ -12,11 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM golang:1.26.5-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS GO_BUILD
+
+RUN mkdir /src
+WORKDIR /src
+
+COPY ./go.mod /src/go.mod
+COPY ./go.sum /src/go.sum
+RUN go mod download
+
+COPY ./ /src/
+RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror
+
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb
 
-RUN apk add py3-yaml
+RUN apk add --no-cache xz curl
 
-COPY ./debian-copyright-mirror.sh /
-COPY ./debian-copyright-mirror.py /
+COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/
+COPY ./cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh /
 
 ENTRYPOINT ["/debian-copyright-mirror.sh"]
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
new file mode 100644
index 00000000000..19b314650f1
--- /dev/null
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -0,0 +1,415 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main downloads all machine-readable copyright files for packages in Debian unstable.
+package main
+
+import (
+	"bufio"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"cloud.google.com/go/storage"
+	gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools"
+	"github.com/google/osv.dev/vulnfeeds/utility"
+	"github.com/google/osv.dev/vulnfeeds/utility/logger"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	DefaultFilelistURL      = "https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz"
+	DefaultURLBase          = "https://metadata.ftp-master.debian.org/changelogs"
+	DefaultPrefixFilter     = "main/"
+	DefaultMinExpectedFiles = 40000
+	DefaultNumWorkers       = 50
+	DefaultMaxFailureRate   = 0.05 // 5% maximum allowed failure rate
+)
+
+// ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader
+// and extracts the unstable_copyright path for each package matching the specified prefixFilter.
+func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error) {
+	scanner := bufio.NewScanner(r)
+	const maxLineLen = 1024 * 1024
+	buf := make([]byte, 64*1024)
+	scanner.Buffer(buf, maxLineLen)
+
+	var results []string
+	inUnstable := false
+	currentPkgFound := false
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		// Top-level package key: starts without whitespace and ends with colon
+		if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") {
+			inUnstable = false
+			currentPkgFound = false
+			continue
+		}
+
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+
+		// Check if this is a suite/version subkey under the package
+		if strings.HasPrefix(line, "  ") && !strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ":") {
+			if trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` {
+				inUnstable = true
+			} else {
+				inUnstable = false
+			}
+			continue
+		}
+
+		// If we are within the "unstable" section and haven't found a copyright file for this package yet
+		if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") {
+			entry := strings.TrimSpace(strings.TrimPrefix(trimmed, "- "))
+			entry = strings.Trim(entry, `"'`)
+			if strings.HasSuffix(entry, "unstable_copyright") {
+				if prefixFilter == "" || strings.HasPrefix(entry, prefixFilter) {
+					results = append(results, entry)
+				}
+				currentPkgFound = true
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error scanning filelist YAML: %w", err)
+	}
+
+	return results, nil
+}
+
+// DownloadFilesConcurrently downloads all files in filelist using a channel-based worker pool.
+func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelist []string, urlBase, workDir string, numWorkers int, skipExisting bool, maxFailureRate float64) error {
+	if numWorkers <= 0 {
+		numWorkers = DefaultNumWorkers
+	}
+
+	total := len(filelist)
+	logger.Info("Starting concurrent download of copyright files",
+		slog.Int("total_files", total),
+		slog.Int("workers", numWorkers),
+		slog.Bool("skip_existing", skipExisting),
+	)
+
+	jobs := make(chan string, numWorkers*2)
+	var completed atomic.Int64
+	var failed atomic.Int64
+	var skipped atomic.Int64
+	startTime := time.Now()
+
+	g, ctx := errgroup.WithContext(ctx)
+
+	for i := 0; i < numWorkers; i++ {
+		g.Go(func() error {
+			for path := range jobs {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				default:
+				}
+
+				fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/"))
+				destPath := filepath.Join(workDir, path)
+
+				if skipExisting {
+					if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 {
+						skipped.Add(1)
+						count := completed.Add(1)
+						logProgress(count, total, startTime)
+						continue
+					}
+				}
+
+				if err := utility.DownloadFile(ctx, client, fullURL, destPath, false); err != nil {
+					failed.Add(1)
+					logger.Warn("Failed to download copyright file", slog.String("url", fullURL), slog.Any("err", err))
+				} else {
+					count := completed.Add(1)
+					logProgress(count, total, startTime)
+				}
+			}
+			return nil
+		})
+	}
+
+	go func() {
+		defer close(jobs)
+		for _, path := range filelist {
+			select {
+			case <-ctx.Done():
+				return
+			case jobs <- path:
+			}
+		}
+	}()
+
+	if err := g.Wait(); err != nil {
+		return fmt.Errorf("concurrent download aborted: %w", err)
+	}
+
+	completedCount := completed.Load()
+	failedCount := failed.Load()
+	skippedCount := skipped.Load()
+
+	logger.Info("Finished downloading copyright files",
+		slog.Int64("completed", completedCount),
+		slog.Int64("failed", failedCount),
+		slog.Int64("skipped", skippedCount),
+		slog.Int("total", total),
+		slog.Duration("elapsed", time.Since(startTime)),
+	)
+
+	if total > 0 {
+		failRatio := float64(failedCount) / float64(total)
+		if failRatio > maxFailureRate || (completedCount == 0 && total > 0) {
+			return fmt.Errorf("too many downloads failed: %d/%d (%.2f%% failed, max threshold is %.2f%%)",
+				failedCount, total, failRatio*100.0, maxFailureRate*100.0)
+		}
+	}
+
+	return nil
+}
+
+func logProgress(count int64, total int, startTime time.Time) {
+	if count%5000 == 0 || count == int64(total) {
+		pct := float64(count) / float64(total) * 100.0
+		logger.Info("Download progress",
+			slog.Int64("completed", count),
+			slog.Int("total", total),
+			slog.Float64("percent", pct),
+			slog.Duration("elapsed", time.Since(startTime)),
+		)
+	}
+}
+
+// GenerateCurlConfiguration generates a curl config file for parallel downloads.
+func GenerateCurlConfiguration(filelist []string, urlBase, configPath string) error {
+	file, err := os.Create(configPath)
+	if err != nil {
+		return fmt.Errorf("failed to create curl config file: %w", err)
+	}
+	defer file.Close()
+
+	w := bufio.NewWriter(file)
+	for _, path := range filelist {
+		fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/"))
+		if _, err := fmt.Fprintf(w, "--output %s\nurl = %s\n", path, fullURL); err != nil {
+			return fmt.Errorf("failed to write curl config entry: %w", err)
+		}
+	}
+
+	return w.Flush()
+}
+
+// ExecuteCurl runs curl with the specified configuration file in the working directory.
+func ExecuteCurl(ctx context.Context, configPath, workDir string) error {
+	if err := os.MkdirAll(workDir, 0755); err != nil {
+		return fmt.Errorf("failed to create work directory: %w", err)
+	}
+
+	cmd := exec.CommandContext(ctx, "curl", "--parallel", "--create-dirs", "--config", configPath)
+	cmd.Dir = workDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	logger.Info("Executing curl in parallel", slog.String("config", configPath), slog.String("workDir", workDir))
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("curl execution failed: %w", err)
+	}
+
+	return nil
+}
+
+// UploadTarToGCS creates a tar archive of workDir and streams it directly to GCS.
+func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir, gcsURI string) error {
+	bucketName, objectName, err := gcs.ParseGCSPath(gcsURI)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Streaming tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName), slog.String("workDir", workDir))
+	bkt := storageClient.Bucket(bucketName)
+
+	pipeReader, pipeWriter := io.Pipe()
+	errChan := make(chan error, 1)
+
+	go func() {
+		err := utility.CreateTarArchive(workDir, pipeWriter)
+		_ = pipeWriter.CloseWithError(err)
+		errChan <- err
+	}()
+
+	uploadErr := gcs.UploadToGCS(ctx, bkt, objectName, pipeReader, "application/x-tar", nil)
+	archiveErr := <-errChan
+
+	if uploadErr != nil {
+		return fmt.Errorf("failed to upload tar to GCS %s: %w", gcsURI, uploadErr)
+	}
+	if archiveErr != nil {
+		return fmt.Errorf("failed to create tar archive: %w", archiveErr)
+	}
+
+	logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI))
+	return nil
+}
+
+func main() {
+	logger.InitGlobalLogger()
+	defer logger.Close()
+
+	workDirFlag := flag.String("work-dir", "", "Directory to download copyright files into.")
+	filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.")
+	urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.")
+	prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').")
+	minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.")
+	numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.")
+	skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.")
+	maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.")
+	gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.")
+	tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. /scratch/debian_copyright.tar).")
+	useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.")
+	curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.")
+	curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.")
+
+	flag.Parse()
+
+	workDir := *workDirFlag
+	if workDir == "" && flag.NArg() > 0 {
+		workDir = flag.Arg(0)
+	}
+	if workDir == "" {
+		workDir = "."
+	}
+
+	ctx := context.Background()
+
+	poolSize := *numWorkers * 2
+	if poolSize < 100 {
+		poolSize = 100
+	}
+
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			MaxIdleConns:        poolSize,
+			MaxIdleConnsPerHost: poolSize,
+			IdleConnTimeout:     90 * time.Second,
+			ForceAttemptHTTP2:   true,
+		},
+	}
+
+	logger.Info("Streaming and decompressing filelist", slog.String("url", *filelistURL))
+	decompressedReader, err := utility.FetchAndDecompressXZ(ctx, httpClient, *filelistURL)
+	if err != nil {
+		logger.Fatal("Failed to fetch and decompress filelist", slog.Any("err", err))
+	}
+	defer decompressedReader.Close()
+
+	logger.Info("Extracting unstable copyright file paths", slog.String("prefixFilter", *prefixFilter))
+	copyrightFiles, err := ExtractUnstableCopyright(decompressedReader, *prefixFilter)
+	if err != nil {
+		logger.Fatal("Failed to extract unstable copyright paths", slog.Any("err", err))
+	}
+	_ = decompressedReader.Close()
+
+	logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles)))
+
+	if len(copyrightFiles) < *minExpectedFiles {
+		logger.Fatal("Unexpectedly small number of copyright files found",
+			slog.Int("found", len(copyrightFiles)),
+			slog.Int("min_expected", *minExpectedFiles),
+		)
+	}
+
+	if *useCurl || *curlConfigOnly || *curlConfigFile != "" {
+		cfgPath := *curlConfigFile
+		if cfgPath == "" {
+			tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-curl-*")
+			if err != nil {
+				logger.Fatal("Failed to create temporary directory for curl config", slog.Any("err", err))
+			}
+			defer os.RemoveAll(tempDir)
+			cfgPath = filepath.Join(tempDir, "curl_configuration")
+		}
+
+		logger.Info("Generating curl configuration", slog.String("path", cfgPath))
+		if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil {
+			logger.Fatal("Failed to generate curl configuration", slog.Any("err", err))
+		}
+
+		if *curlConfigOnly {
+			logger.Info("Curl configuration generated successfully; exiting as requested.")
+			return
+		}
+
+		if *useCurl {
+			if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil {
+				logger.Fatal("Curl download failed", slog.Any("err", err))
+			}
+		}
+	} else {
+		if err := DownloadFilesConcurrently(ctx, httpClient, copyrightFiles, *urlBase, workDir, *numWorkers, *skipExisting, *maxFailureRate); err != nil {
+			logger.Fatal("Concurrent download failed", slog.Any("err", err))
+		}
+	}
+
+	if *tarPath != "" {
+		logger.Info("Creating local tar archive", slog.String("path", *tarPath), slog.String("workDir", workDir))
+		if err := os.MkdirAll(filepath.Dir(*tarPath), 0755); err != nil {
+			logger.Fatal("Failed to create tar destination directory", slog.Any("err", err))
+		}
+		tarFile, err := os.Create(*tarPath)
+		if err != nil {
+			logger.Fatal("Failed to create tar file", slog.Any("err", err))
+		}
+		if err := utility.CreateTarArchive(workDir, tarFile); err != nil {
+			_ = tarFile.Close()
+			logger.Fatal("Failed to write tar archive", slog.Any("err", err))
+		}
+		if err := tarFile.Close(); err != nil {
+			logger.Fatal("Failed to close tar file", slog.Any("err", err))
+		}
+		logger.Info("Successfully created local tar archive", slog.String("path", *tarPath))
+	}
+
+	gcsDest := *gcsPath
+	if gcsDest == "" {
+		gcsDest = os.Getenv("GCS_PATH")
+	}
+	if gcsDest != "" {
+		storageClient, err := storage.NewClient(ctx)
+		if err != nil {
+			logger.Fatal("Failed to create GCS client", slog.Any("err", err))
+		}
+		defer storageClient.Close()
+
+		if err := UploadTarToGCS(ctx, storageClient, workDir, gcsDest); err != nil {
+			logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err))
+		}
+	}
+
+	logger.Info("Debian copyright mirror sync completed successfully.", slog.String("workDir", workDir))
+}
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
new file mode 100644
index 00000000000..2d1b39a974a
--- /dev/null
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
@@ -0,0 +1,193 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+const sampleFilelistYAML = `
+0ad:
+  0.0.17-1:
+  - main/0/0ad/0ad_0.0.17-1_copyright
+  - main/0/0ad/0ad_0.0.17-1_changelog
+  testing:
+  - main/0/0ad/testing_copyright
+  - main/0/0ad/testing_changelog
+  unstable:
+  - main/0/0ad/unstable_changelog
+  - main/0/0ad/unstable_copyright
+  - main/0/0ad/unstable_NEWS
+0ad-data:
+  unstable:
+  - main/0/0ad-data/unstable_changelog
+  - main/0/0ad-data/unstable_copyright
+non-free-pkg:
+  unstable:
+  - non-free/n/non-free-pkg/unstable_changelog
+  - non-free/n/non-free-pkg/unstable_copyright
+no-unstable-pkg:
+  stable:
+  - main/n/no-unstable-pkg/stable_copyright
+no-copyright-pkg:
+  unstable:
+  - main/n/no-copyright-pkg/unstable_changelog
+'0xffff':
+  0.6.1-1:
+  - main/0/0xffff/0xffff_0.6.1-1_changelog
+  unstable:
+  - "main/0/0xffff/unstable_changelog"
+  - 'main/0/0xffff/unstable_copyright'
+`
+
+func TestExtractUnstableCopyright(t *testing.T) {
+	tests := []struct {
+		name         string
+		yamlContent  string
+		prefixFilter string
+		expected     []string
+	}{
+		{
+			name:         "Sample filelist with main/ filter",
+			yamlContent:  sampleFilelistYAML,
+			prefixFilter: "main/",
+			expected: []string{
+				"main/0/0ad/unstable_copyright",
+				"main/0/0ad-data/unstable_copyright",
+				"main/0/0xffff/unstable_copyright",
+			},
+		},
+		{
+			name:         "Sample filelist without filter",
+			yamlContent:  sampleFilelistYAML,
+			prefixFilter: "",
+			expected: []string{
+				"main/0/0ad/unstable_copyright",
+				"main/0/0ad-data/unstable_copyright",
+				"non-free/n/non-free-pkg/unstable_copyright",
+				"main/0/0xffff/unstable_copyright",
+			},
+		},
+		{
+			name:         "Empty YAML",
+			yamlContent:  "",
+			prefixFilter: "main/",
+			expected:     nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := strings.NewReader(tt.yamlContent)
+			got, err := ExtractUnstableCopyright(r, tt.prefixFilter)
+			if err != nil {
+				t.Fatalf("ExtractUnstableCopyright unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("ExtractUnstableCopyright() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestGenerateCurlConfiguration(t *testing.T) {
+	tempDir := t.TempDir()
+	configPath := filepath.Join(tempDir, "curl_config")
+
+	files := []string{
+		"main/0/0ad/unstable_copyright",
+		"main/0/0xffff/unstable_copyright",
+	}
+	urlBase := "https://metadata.ftp-master.debian.org/changelogs"
+
+	if err := GenerateCurlConfiguration(files, urlBase, configPath); err != nil {
+		t.Fatalf("GenerateCurlConfiguration returned error: %v", err)
+	}
+
+	content, err := os.ReadFile(configPath)
+	if err != nil {
+		t.Fatalf("Failed to read generated config: %v", err)
+	}
+
+	expected := "--output main/0/0ad/unstable_copyright\n" +
+		"url = https://metadata.ftp-master.debian.org/changelogs/main/0/0ad/unstable_copyright\n" +
+		"--output main/0/0xffff/unstable_copyright\n" +
+		"url = https://metadata.ftp-master.debian.org/changelogs/main/0/0xffff/unstable_copyright\n"
+
+	if string(content) != expected {
+		t.Errorf("GenerateCurlConfiguration() =\n%s\nwant:\n%s", string(content), expected)
+	}
+}
+
+func TestDownloadFilesConcurrently(t *testing.T) {
+	fileMap := map[string]string{
+		"main/a/pkg1/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/foo/pkg1\n",
+		"main/b/pkg2/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/bar/pkg2\n",
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path := strings.TrimPrefix(r.URL.Path, "/")
+		if content, ok := fileMap[path]; ok {
+			w.WriteHeader(http.StatusOK)
+			_, _ = fmt.Fprint(w, content)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer ts.Close()
+
+	tempDir := t.TempDir()
+	ctx := context.Background()
+	client := ts.Client()
+
+	filelist := []string{
+		"main/a/pkg1/unstable_copyright",
+		"main/b/pkg2/unstable_copyright",
+	}
+
+	if err := DownloadFilesConcurrently(ctx, client, filelist, ts.URL, tempDir, 2, false, 0.05); err != nil {
+		t.Fatalf("DownloadFilesConcurrently failed: %v", err)
+	}
+
+	for _, relPath := range filelist {
+		fullPath := filepath.Join(tempDir, relPath)
+		data, err := os.ReadFile(fullPath)
+		if err != nil {
+			t.Errorf("Failed to read %s: %v", relPath, err)
+			continue
+		}
+		if string(data) != fileMap[relPath] {
+			t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath])
+		}
+	}
+
+	// Test failure threshold breach
+	badFileList := []string{
+		"main/missing/1",
+		"main/missing/2",
+	}
+	err := DownloadFilesConcurrently(ctx, client, badFileList, ts.URL, tempDir, 2, false, 0.05)
+	if err == nil {
+		t.Errorf("Expected failure threshold error, got nil")
+	}
+}
diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go
index 8fc78d7b4ff..deb217c17e3 100644
--- a/vulnfeeds/gcs-tools/gcs.go
+++ b/vulnfeeds/gcs-tools/gcs.go
@@ -145,6 +145,19 @@ func UploadToGCS(ctx context.Context, bkt *storage.BucketHandle, objectName stri
 	return nil
 }
 
+// ParseGCSPath parses a gs:// URI into bucket and object name components.
+func ParseGCSPath(gcsURI string) (bucket, object string, err error) {
+	if !strings.HasPrefix(gcsURI, "gs://") {
+		return "", "", fmt.Errorf("invalid GCS URI: %s (must start with gs://)", gcsURI)
+	}
+	trimmed := strings.TrimPrefix(gcsURI, "gs://")
+	parts := strings.SplitN(trimmed, "/", 2)
+	if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI)
+	}
+	return parts[0], parts[1], nil
+}
+
 // UploadFile uploads a local file to a GCS bucket.
 func UploadFile(ctx context.Context, bkt *storage.BucketHandle, objectName string, filePath string) error {
 	f, err := os.Open(filePath)
diff --git a/vulnfeeds/gcs-tools/gcs_test.go b/vulnfeeds/gcs-tools/gcs_test.go
index 65d0514ad77..01cdf5deb61 100644
--- a/vulnfeeds/gcs-tools/gcs_test.go
+++ b/vulnfeeds/gcs-tools/gcs_test.go
@@ -299,3 +299,50 @@ func TestListObjectsFast(t *testing.T) {
 		t.Errorf("ListObjectsFast returned unexpected list.\ngot:  %v\nwant: %v", got, expected)
 	}
 }
+
+func TestParseGCSPath(t *testing.T) {
+	tests := []struct {
+		input       string
+		wantBucket  string
+		wantObject  string
+		expectError bool
+	}{
+		{
+			input:       "gs://my-bucket/path/to/file.tar",
+			wantBucket:  "my-bucket",
+			wantObject:  "path/to/file.tar",
+			expectError: false,
+		},
+		{
+			input:       "gs://cve-osv-conversion/debian_copyright.tar",
+			wantBucket:  "cve-osv-conversion",
+			wantObject:  "debian_copyright.tar",
+			expectError: false,
+		},
+		{
+			input:       "/local/path/file.tar",
+			expectError: true,
+		},
+		{
+			input:       "gs://bucket-only",
+			expectError: true,
+		},
+		{
+			input:       "gs://",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		bucket, obj, err := ParseGCSPath(tt.input)
+		if (err != nil) != tt.expectError {
+			t.Errorf("ParseGCSPath(%q) error = %v, expectError = %v", tt.input, err, tt.expectError)
+			continue
+		}
+		if !tt.expectError {
+			if bucket != tt.wantBucket || obj != tt.wantObject {
+				t.Errorf("ParseGCSPath(%q) = (%q, %q), want (%q, %q)", tt.input, bucket, obj, tt.wantBucket, tt.wantObject)
+			}
+		}
+	}
+}
diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go
new file mode 100644
index 00000000000..267b8e4ba4e
--- /dev/null
+++ b/vulnfeeds/utility/archive.go
@@ -0,0 +1,118 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer.
+func CreateTarArchive(sourceDir string, w io.Writer) error {
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	sourceDir = filepath.Clean(sourceDir)
+	return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if file == sourceDir {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(sourceDir, file)
+		if err != nil {
+			return fmt.Errorf("failed to get relative path for %s: %w", file, err)
+		}
+
+		tarName := filepath.ToSlash(relPath)
+		if fi.IsDir() {
+			tarName += "/"
+		}
+
+		hdr, err := tar.FileInfoHeader(fi, "")
+		if err != nil {
+			return fmt.Errorf("failed to create tar header for %s: %w", file, err)
+		}
+		hdr.Name = tarName
+
+		if err := tw.WriteHeader(hdr); err != nil {
+			return fmt.Errorf("failed to write tar header for %s: %w", file, err)
+		}
+
+		if fi.Mode().IsRegular() {
+			f, err := os.Open(file)
+			if err != nil {
+				return fmt.Errorf("failed to open file %s: %w", file, err)
+			}
+			defer f.Close()
+
+			if _, err := io.Copy(tw, f); err != nil {
+				return fmt.Errorf("failed to copy file %s to tar: %w", file, err)
+			}
+		}
+
+		return nil
+	})
+}
+
+// ExtractTarArchive extracts a tar archive stream into the destination directory.
+func ExtractTarArchive(r io.Reader, destDir string) error {
+	tr := tar.NewReader(r)
+	cleanDest := filepath.Clean(destDir)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("error reading tar entry: %w", err)
+		}
+
+		// Prevent Zip Slip / path traversal
+		targetPath := filepath.Join(cleanDest, filepath.FromSlash(hdr.Name))
+		if !strings.HasPrefix(targetPath, cleanDest+string(os.PathSeparator)) && targetPath != cleanDest {
+			return fmt.Errorf("illegal file path in tar archive: %s", hdr.Name)
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(targetPath, 0755); err != nil {
+				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
+				return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err)
+			}
+			outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, hdr.FileInfo().Mode())
+			if err != nil {
+				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
+			}
+			if _, err := io.Copy(outFile, tr); err != nil {
+				outFile.Close()
+				return fmt.Errorf("failed to write file content %s: %w", targetPath, err)
+			}
+			outFile.Close()
+		}
+	}
+
+	return nil
+}
diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go
new file mode 100644
index 00000000000..a853f771bf1
--- /dev/null
+++ b/vulnfeeds/utility/archive_test.go
@@ -0,0 +1,69 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestTarArchiveRoundtrip(t *testing.T) {
+	tempDir := t.TempDir()
+	sourceDir := filepath.Join(tempDir, "source")
+	destDir := filepath.Join(tempDir, "dest")
+
+	if err := os.MkdirAll(filepath.Join(sourceDir, "sub"), 0755); err != nil {
+		t.Fatalf("Failed to create test source dir: %v", err)
+	}
+
+	file1 := filepath.Join(sourceDir, "file1.txt")
+	file2 := filepath.Join(sourceDir, "sub", "file2.txt")
+	if err := os.WriteFile(file1, []byte("hello archive"), 0644); err != nil {
+		t.Fatalf("Failed to write file1: %v", err)
+	}
+	if err := os.WriteFile(file2, []byte("nested file content"), 0644); err != nil {
+		t.Fatalf("Failed to write file2: %v", err)
+	}
+
+	var buf bytes.Buffer
+	if err := CreateTarArchive(sourceDir, &buf); err != nil {
+		t.Fatalf("CreateTarArchive failed: %v", err)
+	}
+
+	if err := ExtractTarArchive(&buf, destDir); err != nil {
+		t.Fatalf("ExtractTarArchive failed: %v", err)
+	}
+
+	destFile1 := filepath.Join(destDir, "file1.txt")
+	destFile2 := filepath.Join(destDir, "sub", "file2.txt")
+
+	data1, err := os.ReadFile(destFile1)
+	if err != nil {
+		t.Fatalf("Failed to read extracted file1: %v", err)
+	}
+	if string(data1) != "hello archive" {
+		t.Errorf("file1 content = %q, want %q", string(data1), "hello archive")
+	}
+
+	data2, err := os.ReadFile(destFile2)
+	if err != nil {
+		t.Fatalf("Failed to read extracted file2: %v", err)
+	}
+	if string(data2) != "nested file content" {
+		t.Errorf("file2 content = %q, want %q", string(data2), "nested file content")
+	}
+}
diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go
new file mode 100644
index 00000000000..8f80ea2d0f3
--- /dev/null
+++ b/vulnfeeds/utility/decompress.go
@@ -0,0 +1,127 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+)
+
+// SubprocessReadCloser wraps a command's stdout pipe and ensures the subprocess
+// is waited for and input resources are closed when Close() is called.
+type SubprocessReadCloser struct {
+	io.ReadCloser
+	cmd         *exec.Cmd
+	bodyToClose io.Closer
+}
+
+// Close closes the underlying reader pipe, any input reader, and waits for the subprocess.
+func (s *SubprocessReadCloser) Close() error {
+	var errs []error
+	if s.ReadCloser != nil {
+		if err := s.ReadCloser.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if s.bodyToClose != nil {
+		if err := s.bodyToClose.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if s.cmd != nil {
+		if err := s.cmd.Wait(); err != nil {
+			var exitErr *exec.ExitError
+			if errors.As(err, &exitErr) && !exitErr.Success() {
+				errs = append(errs, err)
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+func getXZDecompressCommand() (string, error) {
+	if _, err := exec.LookPath("xz"); err == nil {
+		return "xz", nil
+	}
+	if _, err := exec.LookPath("unxz"); err == nil {
+		return "unxz", nil
+	}
+	return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH")
+}
+
+// DecompressXZ streams an xz-compressed reader through the system xz or unxz command.
+func DecompressXZ(ctx context.Context, r io.Reader) (io.ReadCloser, error) {
+	decompressCmd, err := getXZDecompressCommand()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := exec.CommandContext(ctx, decompressCmd, "-dc")
+	cmd.Stdin = r
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stdout pipe for %s: %w", decompressCmd, err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start %s: %w", decompressCmd, err)
+	}
+
+	var bodyToClose io.Closer
+	if closer, ok := r.(io.Closer); ok {
+		bodyToClose = closer
+	}
+
+	return &SubprocessReadCloser{
+		ReadCloser:  stdout,
+		cmd:         cmd,
+		bodyToClose: bodyToClose,
+	}, nil
+}
+
+// DecompressXZFile decompresses a local xz-compressed file.
+func DecompressXZFile(ctx context.Context, xzFilePath string) (io.ReadCloser, error) {
+	f, err := os.Open(xzFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %s: %w", xzFilePath, err)
+	}
+
+	return DecompressXZ(ctx, f)
+}
+
+// FetchAndDecompressXZ fetches an xz-compressed URL over HTTP and streams
+// its decompressed content directly through xz/unxz without saving to disk.
+func FetchAndDecompressXZ(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request for %s: %w", url, err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch %s: %w", url, err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("HTTP status %d fetching %s", resp.StatusCode, url)
+	}
+
+	return DecompressXZ(ctx, resp.Body)
+}
diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go
new file mode 100644
index 00000000000..fe404b02da1
--- /dev/null
+++ b/vulnfeeds/utility/decompress_test.go
@@ -0,0 +1,108 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var testXZPayload = []byte{
+	0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46,
+	0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3,
+	0x01, 0x00, 0x1d, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x44, 0x65, 0x62,
+	0x69, 0x61, 0x6e, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68,
+	0x74, 0x20, 0x4d, 0x69, 0x72, 0x72, 0x6f, 0x72, 0x21, 0x00, 0x00, 0x00,
+	0xc7, 0x2f, 0x01, 0x8a, 0x65, 0x5a, 0x9a, 0x97, 0x00, 0x01, 0x36, 0x1e,
+	0x3d, 0x19, 0x95, 0x53, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00,
+	0x00, 0x04, 0x59, 0x5a,
+}
+
+func TestDecompressXZ(t *testing.T) {
+	ctx := context.Background()
+	reader, err := DecompressXZ(ctx, bytes.NewReader(testXZPayload))
+	if err != nil {
+		t.Fatalf("DecompressXZ failed: %v", err)
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("Failed to read decompressed data: %v", err)
+	}
+
+	want := "Hello Debian Copyright Mirror!"
+	if string(data) != want {
+		t.Errorf("Decompressed = %q, want %q", string(data), want)
+	}
+}
+
+func TestDecompressXZFile(t *testing.T) {
+	tempDir := t.TempDir()
+	filePath := filepath.Join(tempDir, "test.xz")
+	if err := os.WriteFile(filePath, testXZPayload, 0644); err != nil {
+		t.Fatalf("Failed to write test file: %v", err)
+	}
+
+	ctx := context.Background()
+	reader, err := DecompressXZFile(ctx, filePath)
+	if err != nil {
+		t.Fatalf("DecompressXZFile failed: %v", err)
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("Failed to read decompressed data: %v", err)
+	}
+
+	want := "Hello Debian Copyright Mirror!"
+	if string(data) != want {
+		t.Errorf("Decompressed = %q, want %q", string(data), want)
+	}
+}
+
+func TestFetchAndDecompressXZ(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(testXZPayload)
+	}))
+	defer ts.Close()
+
+	ctx := context.Background()
+	client := ts.Client()
+
+	reader, err := FetchAndDecompressXZ(ctx, client, ts.URL+"/file.xz")
+	if err != nil {
+		t.Fatalf("FetchAndDecompressXZ failed: %v", err)
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("Failed to read decompressed data: %v", err)
+	}
+
+	want := "Hello Debian Copyright Mirror!"
+	if string(data) != want {
+		t.Errorf("Decompressed = %q, want %q", string(data), want)
+	}
+}
diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go
new file mode 100644
index 00000000000..1b2bed79635
--- /dev/null
+++ b/vulnfeeds/utility/download.go
@@ -0,0 +1,102 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/sethvargo/go-retry"
+)
+
+const (
+	DefaultDownloadTimeout = 30 * time.Second
+)
+
+// DownloadFile downloads a URL to a local destination file with automatic retries,
+// safe temporary file writing, and on-demand directory creation.
+// If skipExisting is true and destPath exists with non-zero size, the download is skipped.
+func DownloadFile(ctx context.Context, client *http.Client, url, destPath string, skipExisting bool) error {
+	if skipExisting {
+		if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 {
+			return nil
+		}
+	}
+
+	backoff := retry.NewExponential(1 * time.Second)
+	backoff = retry.WithMaxRetries(3, backoff)
+
+	return retry.Do(ctx, backoff, func(ctx context.Context) error {
+		reqCtx, cancel := context.WithTimeout(ctx, DefaultDownloadTimeout)
+		defer cancel()
+
+		req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil)
+		if err != nil {
+			return fmt.Errorf("failed to create request for %s: %w", url, err)
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			return retry.RetryableError(fmt.Errorf("HTTP request failed for %s: %w", url, err))
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests {
+				return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url))
+			}
+			return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)
+		}
+
+		dir := filepath.Dir(destPath)
+		tmpFile, err := os.CreateTemp(dir, "download-*")
+		if err != nil {
+			if os.IsNotExist(err) {
+				if mkdirErr := os.MkdirAll(dir, 0755); mkdirErr != nil {
+					return fmt.Errorf("failed to create directory %s: %w", dir, mkdirErr)
+				}
+				tmpFile, err = os.CreateTemp(dir, "download-*")
+			}
+			if err != nil {
+				return fmt.Errorf("failed to create temp file in %s: %w", dir, err)
+			}
+		}
+		tmpPath := tmpFile.Name()
+
+		_, copyErr := io.Copy(tmpFile, resp.Body)
+		closeErr := tmpFile.Close()
+
+		if copyErr != nil {
+			_ = os.Remove(tmpPath)
+			return retry.RetryableError(fmt.Errorf("failed to write data: %w", copyErr))
+		}
+		if closeErr != nil {
+			_ = os.Remove(tmpPath)
+			return fmt.Errorf("failed to close temp file: %w", closeErr)
+		}
+
+		if err := os.Rename(tmpPath, destPath); err != nil {
+			_ = os.Remove(tmpPath)
+			return fmt.Errorf("failed to move temp file to %s: %w", destPath, err)
+		}
+
+		return nil
+	})
+}
diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go
new file mode 100644
index 00000000000..d1eb62b48e4
--- /dev/null
+++ b/vulnfeeds/utility/download_test.go
@@ -0,0 +1,66 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utility
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestDownloadFile(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/file.txt" {
+			w.WriteHeader(http.StatusOK)
+			_, _ = fmt.Fprint(w, "downloaded content")
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer ts.Close()
+
+	tempDir := t.TempDir()
+	destPath := filepath.Join(tempDir, "nested", "dir", "file.txt")
+
+	ctx := context.Background()
+	client := ts.Client()
+
+	if err := DownloadFile(ctx, client, ts.URL+"/file.txt", destPath, false); err != nil {
+		t.Fatalf("DownloadFile failed: %v", err)
+	}
+
+	data, err := os.ReadFile(destPath)
+	if err != nil {
+		t.Fatalf("Failed to read downloaded file: %v", err)
+	}
+	if string(data) != "downloaded content" {
+		t.Errorf("content = %q, want %q", string(data), "downloaded content")
+	}
+
+	// Test skipExisting = true
+	if err := DownloadFile(ctx, client, ts.URL+"/404-should-skip", destPath, true); err != nil {
+		t.Fatalf("DownloadFile with skipExisting should have succeeded without error, got %v", err)
+	}
+
+	// Test 404
+	err = DownloadFile(ctx, client, ts.URL+"/nonexistent", filepath.Join(tempDir, "404.txt"), false)
+	if err == nil {
+		t.Errorf("Expected error for 404, got nil")
+	}
+}

From 9d4392076bc452d7ccfcf5bd87ea2daf8bcdd634 Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Wed, 19 Aug 2026 00:27:55 +0000
Subject: [PATCH 02/10] some other files + readme details

---
 .../mirrors/debian-copyright-mirror/README.md | 121 +++++++++++++
 .../mirrors/debian-copyright-mirror/build.sh  |   4 +-
 .../debian-copyright-mirror.py                | 163 ------------------
 .../debian-copyright-mirror.sh                |  17 +-
 4 files changed, 129 insertions(+), 176 deletions(-)
 create mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
 delete mode 100644 vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
new file mode 100644
index 00000000000..eb139d03051
--- /dev/null
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
@@ -0,0 +1,121 @@
+# Debian Copyright Mirror
+
+`debian-copyright-mirror` maintains a mirror of machine-readable Debian package copyright files for packages in Debian `unstable` (`main` section).
+
+---
+
+## Purpose & Usage in OSV
+
+In the OSV and vulnerability feeds pipeline, correlating Common Vulnerabilities and Exposures (CVE) entries and Common Platform Enumeration (CPE) identifiers with upstream open-source source code repositories (e.g., on GitHub, GitLab, or Git) is a crucial step for automated vulnerability management.
+
+### The Problem
+- CVE entries and NVD CPE dictionaries often lack structured upstream repository URLs, or only provide arbitrary website references.
+- Without a reliable mapping from package names and CPE products to source code repositories, automated tools cannot easily determine affected Git commits, tags, or version ranges for Git-based vulnerability matching.
+
+### How Debian Copyright Files Solve This
+Debian packages in `unstable` follow the [Machine-readable debian/copyright format (DEP-5 / Copyright Specification 1.0)](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/). These files contain structured metadata in header paragraphs, notably:
+
+```text
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: example-lib
+Source: https://github.com/example/example-lib
+```
+
+The `Source:` field provides a canonical, maintainer-verified URL to the upstream open-source repository.
+
+### Downstream Consumers
+1. **`cpe-repo-gen`** (`cmd/mirrors/cpe-repo-gen`):
+   - Downloads the `debian_copyright.tar` archive generated by this tool from Google Cloud Storage (`gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`).
+   - Scans the mirrored `unstable_copyright` files to resolve Debian package names and CPE products to upstream source code repositories.
+   - Produces the canonical `cpe_product_to_repo.json` mapping.
+2. **OSV Conversion & Enrichment Pipelines**:
+   - Use the repository mappings derived from Debian copyright data to accurately calculate affected commit ranges, detect fixed revisions, and generate OSV-format records for Debian, NVD, and other feeds.
+
+---
+
+## How It Works
+
+```mermaid
+graph TD
+    A["Debian FTP Master<br/>(filelist.yaml.xz)"] -->|HTTP Stream & xz -dc| B["debian-copyright-mirror<br/>(Go binary)"]
+    B -->|Streaming YAML Parser| C["Extract ~44k+ main/<br/>unstable_copyright paths"]
+    C -->|Channel-based Worker Pool| D["Download copyright files<br/>to local directory"]
+    D -->|Validate failure rate & count| E["Native Tar Streaming"]
+    E -->|GCS Client Writer| F["GCS Bucket<br/>(gs://cve-osv-conversion/...)"]
+    F -->|tar -xf| G["cpe-repo-gen<br/>(CPE to Repo Mapping)"]
+```
+
+1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it directly through `xz -dc` stdin without intermediate disk roundtrips.
+2. **Manifest Parsing**: Parses the YAML manifest to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section.
+3. **Validation**: Asserts that the number of discovered files meets a sanity threshold (by default, at least 40,000 files) to ensure upstream feeds were not corrupted or truncated.
+4. **Optimized Concurrent Downloads**:
+   - Spawns a channel-based worker pool (default 50 workers) with HTTP/2 keep-alive connection reuse and exponential backoff retries.
+   - Optimized file and directory creation (avoids redundant `MkdirAll` calls on existing directories).
+   - Tracks success/failure metrics and enforces a maximum failure threshold (`-max-failure-rate`) to prevent false-positive completions on network or CDN outages.
+   - Supports incremental runs with `-skip-existing` to skip re-downloading unchanged files.
+   - Optional: Delegated download via `curl --parallel` using generated curl configuration files (`-use-curl`).
+5. **Native Archival & GCS Streaming**: Packages the downloaded mirror into a `.tar` archive and streams it directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) using the Cloud Storage client.
+
+---
+
+## CLI Usage
+
+### Running Locally
+
+```bash
+# Build and run directly using Go
+go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir]
+```
+
+Examples:
+
+```bash
+# Download copyright files into /tmp/debian_copyright using 50 workers
+go run ./cmd/mirrors/debian-copyright-mirror -workers 50 /tmp/debian_copyright
+
+# Incremental run skipping already downloaded files
+go run ./cmd/mirrors/debian-copyright-mirror -skip-existing /tmp/debian_copyright
+
+# Download and create a local tar archive
+go run ./cmd/mirrors/debian-copyright-mirror -tar-path /scratch/debian_copyright.tar /tmp/debian_copyright
+
+# Download and stream directly to GCS
+go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar /tmp/debian_copyright
+```
+
+### Flags
+
+| Flag | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `-work-dir` | string | `.` | Target directory to save downloaded copyright files (can also be passed as the first positional argument). |
+| `-workers` | int | `50` | Number of concurrent download workers. |
+| `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. |
+| `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. |
+| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. |
+| `-tar-path` | string | `""` | Optional local destination path for tarball archive. |
+| `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. |
+| `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. |
+| `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). |
+| `-min-expected-files` | int | `40000` | Minimum expected number of copyright files; fails if fewer are found. |
+| `-use-curl` | bool | `false` | Delegate downloads to `curl --parallel` instead of Go worker pool. |
+| `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. |
+| `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. |
+
+---
+
+## Docker & Deployment
+
+### Building the Docker Container
+
+Build the container image from the `vulnfeeds/` directory:
+
+```bash
+docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/debian-copyright-mirror/Dockerfile .
+```
+
+### Kubernetes CronJob Deployment
+
+In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays):
+
+- **Schedule**: Runs daily (`0 5 * * *` Sydney time).
+- **Entrypoint**: Executes `debian-copyright-mirror.sh`, which runs `debian-copyright-mirror` with `-gcs-path "${GCS_PATH}"` to stream the archive directly to `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`.
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh
index 1fae76fdb88..375eb4cdaf3 100755
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh
@@ -16,7 +16,9 @@ set -ex
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+cd ../../..
+
 docker build \
   -t gcr.io/oss-vdb/debian-copyright-mirror:latest \
-  -f Dockerfile --pull . && \
+  -f cmd/mirrors/debian-copyright-mirror/Dockerfile --pull . && \
 gcloud docker -- push gcr.io/oss-vdb/debian-copyright-mirror:latest
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py
deleted file mode 100644
index c4bdfe2d202..00000000000
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py
+++ /dev/null
@@ -1,163 +0,0 @@
-"""
-Download all of the copyright files for packages in main in Debian unstable.
-
-This:
-
-Parses https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz 
-    to identify the files to retrieve
-Generates a curl configuration to download the URLs
-Executes curl to download the URLs in parallel
-"""
-
-import os
-import argparse
-import yaml
-import lzma
-import urllib.request
-import subprocess
-from typing import List, Optional, Dict
-
-
-class Error(Exception):
-  """General purpose error class."""
-
-
-def download_url_to_directory(url: str,
-                              directory: str,
-                              filename: Optional[str] = None) -> Optional[str]:
-  """
-    Downloads a URL to a specified directory.
-
-    Args:
-        url (str): The URL to download.
-        directory (str): The directory to save the file to.
-        filename (str, optional): The filename to use. If None, the filename
-                                   is extracted from the URL. Defaults to None.
-
-    Returns:
-        str: The full path to the downloaded file, or None on error.
-  """
-  try:
-    if not os.path.exists(directory):
-      os.makedirs(directory)
-
-    if filename is None:
-      filename = os.path.basename(urllib.parse.urlsplit(url).path)
-
-    filepath = os.path.join(directory, filename)
-
-    urllib.request.urlretrieve(url, filepath)
-    return filepath
-
-  except urllib.error.URLError as e:
-    print(f'Error downloading {url}: {e}')
-    return None
-  except OSError as e:  # Catch directory creation/file writing errors.
-    print(f'OS Error: {e}')
-    return None
-  except Exception as e:  # Catch any other unexpected error.
-    print(f'An unexpected error occurred: {e}')
-    return None
-
-
-def extract_unstable_copyright(filelist: str) -> Dict:
-  """
-    Extracts the 'unstable_copyright' entry for each package 
-        from an xz-compressed YAML file.
-
-    Args:
-        filelist (str): The path to the xz-compressed YAML filelist
-
-    Returns:
-        A dictionary where keys are package names and values are 
-            their 'unstable_copyright' entries, or None if no unstable 
-            copyright is found.
-  """
-  try:
-    with lzma.open(filelist, 'rt', encoding='utf-8') as f:
-      data = yaml.safe_load(f)
-
-    results = {}
-    for package, versions in data.items():
-      if 'unstable' in versions:
-        entries = versions['unstable']
-        for entry in entries:
-          if entry.endswith('unstable_copyright'):
-            results[package] = entry
-            break  # Found it, no need to continue checking this package.
-
-    return results
-
-  except FileNotFoundError:
-    print(f"Error: File not found at {filelist}")
-    return None
-  except lzma.LZMAError as e:
-    print(f"Error: LZMA decompression failed: {e}")
-    return None
-  except yaml.YAMLError as e:
-    print(f"Error: YAML parsing failed: {e}")
-    return None
-  except Exception as e:
-    print(f"An unexpected error occurred: {e}")
-    return None
-
-
-def generate_curl_configuration(filelist: List[str]):
-  """
-    Generates a curl configuration to download all of the files in filelist.
-
-    --output filename
-    url = https://url
-
-    Args:
-        filelist (List[str]): a list of files to download.
-  """
-
-  url_base = 'https://metadata.ftp-master.debian.org/changelogs'
-
-  with open('/tmp/curl_configuration', 'w') as curl_config:
-    curl_config.writelines([
-        '--output ' + path + '\n' + 'url = ' + os.path.join(url_base, path) +
-        '\n' for path in filelist
-    ])
-
-
-def execute_curl(configuration: str, directory: str):
-  """
-    Execute curl with the supplied configuration in the specified directory.
-
-    Args:
-        configuration (str): path to configuration file.
-        directory (str): path to set current working directory to.
- """
-
-  os.makedirs(directory)
-  subprocess.run(
-      ['curl', '--parallel', '--create-dirs', '--config', configuration],
-      cwd=directory,
-      check=True)
-
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('work_dir')
-  args = parser.parse_args()
-
-  download_url_to_directory(
-      'https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz',
-      '/tmp')
-  unstable_package_copyright_files = extract_unstable_copyright(
-      '/tmp/filelist.yaml.xz')
-  if unstable_package_copyright_files is None:
-    raise Error('Unexpected result determining files to download')
-  generate_curl_configuration(
-      f for f in unstable_package_copyright_files.values()
-      if f.startswith('main/'))
-  with open("/tmp/curl_configuration") as curl_configuration:
-    if len(curl_configuration.readlines()) < 80000:
-      raise Error('Unexpectly small curl configuration')
-  execute_curl('/tmp/curl_configuration', args.work_dir)
-
-
-if __name__ == '__main__':
-  main()
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh
index a51fdfa08f6..561adea27d3 100755
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh
@@ -17,20 +17,13 @@
 #
 # Maintain a GCS bucket mirror of Debian's copyright files
 #
-#
-#
 # Inputs:
-# * A local work directory
-# * GCS bucket name + path to tarball
+# * A local work directory ($WORK_DIR)
+# * GCS bucket name + path to tarball ($GCS_PATH)
 #
 
-# Setting BE_VERBOSE to an empty string or null value suppresses silencing of
-# commands
-
 mkdir -p "${WORK_DIR}" || true
 
-python3 debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/"
-
-tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" .
-
-gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}"
+debian-copyright-mirror \
+  -work-dir "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" \
+  -gcs-path "${GCS_PATH}"

From b72b692149bb8fe817b3ad06985bdcf0f3e44f3c Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Wed, 19 Aug 2026 03:46:10 +0000
Subject: [PATCH 03/10] build and stage stuff

---
 deployment/build-and-stage.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml
index 522a39ac740..80645a1d7f3 100644
--- a/deployment/build-and-stage.yaml
+++ b/deployment/build-and-stage.yaml
@@ -368,8 +368,8 @@ steps:
 
 # Build/push Debian copyright mirror image
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.']
-  dir: 'vulnfeeds/cmd/mirrors/debian-copyright-mirror'
+  args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '-f', 'cmd/mirrors/debian-copyright-mirror/Dockerfile', '.']
+  dir: 'vulnfeeds'
   id: 'build-debian-copyright-mirror'
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker

From 5074da28ac068679f8f094d37c859436a02bdabb Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Fri, 21 Aug 2026 03:09:03 +0000
Subject: [PATCH 04/10] no bash script, and work_dir nearby

---
 .../debian-copyright-mirror/Dockerfile        |  9 +++---
 .../mirrors/debian-copyright-mirror/README.md | 21 ++++++++-----
 .../debian-copyright-mirror.sh                | 29 -----------------
 .../mirrors/debian-copyright-mirror/main.go   | 31 +++++++++++++++----
 4 files changed, 42 insertions(+), 48 deletions(-)
 delete mode 100755 vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
index 6ef0187b35d..2a9dc3025a4 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
@@ -24,11 +24,10 @@ RUN go mod download
 COPY ./ /src/
 RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror
 
-FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb
+FROM alpine:3.21
 
-RUN apk add --no-cache xz curl
+RUN apk add --no-cache xz curl ca-certificates
 
-COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/
-COPY ./cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh /
+COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror
 
-ENTRYPOINT ["/debian-copyright-mirror.sh"]
+ENTRYPOINT ["/usr/local/bin/debian-copyright-mirror"]
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
index eb139d03051..89a32d77893 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
@@ -70,29 +70,33 @@ go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir]
 Examples:
 
 ```bash
-# Download copyright files into /tmp/debian_copyright using 50 workers
-go run ./cmd/mirrors/debian-copyright-mirror -workers 50 /tmp/debian_copyright
+# Download copyright files into ./debian_copyright (relative to current directory) using 50 workers
+go run ./cmd/mirrors/debian-copyright-mirror -workers 50
+
+# Specify a custom relative output directory
+go run ./cmd/mirrors/debian-copyright-mirror -out-dir debian_copyright
 
 # Incremental run skipping already downloaded files
-go run ./cmd/mirrors/debian-copyright-mirror -skip-existing /tmp/debian_copyright
+go run ./cmd/mirrors/debian-copyright-mirror -skip-existing
 
 # Download and create a local tar archive
-go run ./cmd/mirrors/debian-copyright-mirror -tar-path /scratch/debian_copyright.tar /tmp/debian_copyright
+go run ./cmd/mirrors/debian-copyright-mirror -tar-path debian_copyright.tar
 
 # Download and stream directly to GCS
-go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar /tmp/debian_copyright
+go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar
 ```
 
 ### Flags
 
 | Flag | Type | Default | Description |
 | :--- | :--- | :--- | :--- |
-| `-work-dir` | string | `.` | Target directory to save downloaded copyright files (can also be passed as the first positional argument). |
+| `-out-dir` | string | `debian_copyright` | Target directory (relative or absolute) to save downloaded copyright files. |
+| `-work-dir` | string | `""` | Alias for `-out-dir` (can also be passed as the first positional argument). |
 | `-workers` | int | `50` | Number of concurrent download workers. |
 | `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. |
 | `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. |
 | `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. |
-| `-tar-path` | string | `""` | Optional local destination path for tarball archive. |
+| `-tar-path` | string | `""` | Optional local destination path for tarball archive (e.g. `debian_copyright.tar`). |
 | `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. |
 | `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. |
 | `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). |
@@ -101,6 +105,7 @@ go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_cop
 | `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. |
 | `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. |
 
+
 ---
 
 ## Docker & Deployment
@@ -118,4 +123,4 @@ docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/deb
 In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays):
 
 - **Schedule**: Runs daily (`0 5 * * *` Sydney time).
-- **Entrypoint**: Executes `debian-copyright-mirror.sh`, which runs `debian-copyright-mirror` with `-gcs-path "${GCS_PATH}"` to stream the archive directly to `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`.
+- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment and streams the archive directly to Cloud Storage.
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh
deleted file mode 100755
index 561adea27d3..00000000000
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Maintain a GCS bucket mirror of Debian's copyright files
-#
-# Inputs:
-# * A local work directory ($WORK_DIR)
-# * GCS bucket name + path to tarball ($GCS_PATH)
-#
-
-mkdir -p "${WORK_DIR}" || true
-
-debian-copyright-mirror \
-  -work-dir "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" \
-  -gcs-path "${GCS_PATH}"
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
index 19b314650f1..c698525b69b 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -281,28 +281,47 @@ func main() {
 	logger.InitGlobalLogger()
 	defer logger.Close()
 
-	workDirFlag := flag.String("work-dir", "", "Directory to download copyright files into.")
+	// Input & source configuration flags.
 	filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.")
 	urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.")
 	prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').")
 	minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.")
+
+	// Local output directory & archive flags.
+	outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).")
+	workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.")
+	tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. debian_copyright.tar).")
+
+	// Concurrency & fault tolerance flags.
 	numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.")
 	skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.")
 	maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.")
+
+	// GCS upload flags.
 	gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.")
-	tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. /scratch/debian_copyright.tar).")
+
+	// Curl fallback flags.
 	useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.")
 	curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.")
 	curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.")
 
 	flag.Parse()
 
-	workDir := *workDirFlag
-	if workDir == "" && flag.NArg() > 0 {
+	workDir := *outDirFlag
+	if *workDirFlag != "" {
+		workDir = *workDirFlag
+	} else if flag.NArg() > 0 {
 		workDir = flag.Arg(0)
+	} else if workDir == "" {
+		if envWorkDir := os.Getenv("WORK_DIR"); envWorkDir != "" {
+			workDir = envWorkDir
+		} else {
+			workDir = "debian_copyright"
+		}
 	}
-	if workDir == "" {
-		workDir = "."
+	workDir = filepath.Clean(workDir)
+	if err := os.MkdirAll(workDir, 0755); err != nil {
+		logger.Fatal("Failed to create work directory", slog.String("workDir", workDir), slog.Any("err", err))
 	}
 
 	ctx := context.Background()

From 84522a16bb14bf01989c781ce9fd9a898626bf1b Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Fri, 21 Aug 2026 03:12:14 +0000
Subject: [PATCH 05/10] fix lint

---
 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go      | 7 ++++++-
 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go | 1 +
 vulnfeeds/gcs-tools/gcs.go                                 | 1 +
 vulnfeeds/utility/archive.go                               | 5 ++++-
 vulnfeeds/utility/archive_test.go                          | 4 ++--
 vulnfeeds/utility/decompress.go                            | 3 +++
 vulnfeeds/utility/decompress_test.go                       | 4 ++--
 vulnfeeds/utility/download.go                              | 1 +
 vulnfeeds/utility/download_test.go                         | 1 +
 9 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
index c698525b69b..7be32bc1fba 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -64,6 +64,7 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error
 		if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") {
 			inUnstable = false
 			currentPkgFound = false
+
 			continue
 		}
 
@@ -79,6 +80,7 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error
 			} else {
 				inUnstable = false
 			}
+
 			continue
 		}
 
@@ -123,7 +125,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis
 
 	g, ctx := errgroup.WithContext(ctx)
 
-	for i := 0; i < numWorkers; i++ {
+	for range numWorkers {
 		g.Go(func() error {
 			for path := range jobs {
 				select {
@@ -140,6 +142,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis
 						skipped.Add(1)
 						count := completed.Add(1)
 						logProgress(count, total, startTime)
+
 						continue
 					}
 				}
@@ -152,6 +155,7 @@ func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelis
 					logProgress(count, total, startTime)
 				}
 			}
+
 			return nil
 		})
 	}
@@ -274,6 +278,7 @@ func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir,
 	}
 
 	logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI))
+
 	return nil
 }
 
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
index 2d1b39a974a..c120c844d2d 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
@@ -150,6 +150,7 @@ func TestDownloadFilesConcurrently(t *testing.T) {
 		if content, ok := fileMap[path]; ok {
 			w.WriteHeader(http.StatusOK)
 			_, _ = fmt.Fprint(w, content)
+
 			return
 		}
 		w.WriteHeader(http.StatusNotFound)
diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go
index deb217c17e3..7ccdd37a3a7 100644
--- a/vulnfeeds/gcs-tools/gcs.go
+++ b/vulnfeeds/gcs-tools/gcs.go
@@ -155,6 +155,7 @@ func ParseGCSPath(gcsURI string) (bucket, object string, err error) {
 	if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
 		return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI)
 	}
+
 	return parts[0], parts[1], nil
 }
 
diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go
index 267b8e4ba4e..633acc24358 100644
--- a/vulnfeeds/utility/archive.go
+++ b/vulnfeeds/utility/archive.go
@@ -29,6 +29,7 @@ func CreateTarArchive(sourceDir string, w io.Writer) error {
 	defer tw.Close()
 
 	sourceDir = filepath.Clean(sourceDir)
+
 	return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -73,6 +74,8 @@ func CreateTarArchive(sourceDir string, w io.Writer) error {
 	})
 }
 
+const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit
+
 // ExtractTarArchive extracts a tar archive stream into the destination directory.
 func ExtractTarArchive(r io.Reader, destDir string) error {
 	tr := tar.NewReader(r)
@@ -106,7 +109,7 @@ func ExtractTarArchive(r io.Reader, destDir string) error {
 			if err != nil {
 				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
 			}
-			if _, err := io.Copy(outFile, tr); err != nil {
+			if _, err := io.Copy(outFile, io.LimitReader(tr, maxTarEntrySize)); err != nil {
 				outFile.Close()
 				return fmt.Errorf("failed to write file content %s: %w", targetPath, err)
 			}
diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go
index a853f771bf1..a13a075b0c3 100644
--- a/vulnfeeds/utility/archive_test.go
+++ b/vulnfeeds/utility/archive_test.go
@@ -32,10 +32,10 @@ func TestTarArchiveRoundtrip(t *testing.T) {
 
 	file1 := filepath.Join(sourceDir, "file1.txt")
 	file2 := filepath.Join(sourceDir, "sub", "file2.txt")
-	if err := os.WriteFile(file1, []byte("hello archive"), 0644); err != nil {
+	if err := os.WriteFile(file1, []byte("hello archive"), 0600); err != nil {
 		t.Fatalf("Failed to write file1: %v", err)
 	}
-	if err := os.WriteFile(file2, []byte("nested file content"), 0644); err != nil {
+	if err := os.WriteFile(file2, []byte("nested file content"), 0600); err != nil {
 		t.Fatalf("Failed to write file2: %v", err)
 	}
 
diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go
index 8f80ea2d0f3..32080ab6cb8 100644
--- a/vulnfeeds/utility/decompress.go
+++ b/vulnfeeds/utility/decompress.go
@@ -28,6 +28,7 @@ import (
 // is waited for and input resources are closed when Close() is called.
 type SubprocessReadCloser struct {
 	io.ReadCloser
+
 	cmd         *exec.Cmd
 	bodyToClose io.Closer
 }
@@ -53,6 +54,7 @@ func (s *SubprocessReadCloser) Close() error {
 			}
 		}
 	}
+
 	return errors.Join(errs...)
 }
 
@@ -63,6 +65,7 @@ func getXZDecompressCommand() (string, error) {
 	if _, err := exec.LookPath("unxz"); err == nil {
 		return "unxz", nil
 	}
+
 	return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH")
 }
 
diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go
index fe404b02da1..e66ca858e8e 100644
--- a/vulnfeeds/utility/decompress_test.go
+++ b/vulnfeeds/utility/decompress_test.go
@@ -58,7 +58,7 @@ func TestDecompressXZ(t *testing.T) {
 func TestDecompressXZFile(t *testing.T) {
 	tempDir := t.TempDir()
 	filePath := filepath.Join(tempDir, "test.xz")
-	if err := os.WriteFile(filePath, testXZPayload, 0644); err != nil {
+	if err := os.WriteFile(filePath, testXZPayload, 0600); err != nil {
 		t.Fatalf("Failed to write test file: %v", err)
 	}
 
@@ -81,7 +81,7 @@ func TestDecompressXZFile(t *testing.T) {
 }
 
 func TestFetchAndDecompressXZ(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		_, _ = w.Write(testXZPayload)
 	}))
diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go
index 1b2bed79635..df38ee6725e 100644
--- a/vulnfeeds/utility/download.go
+++ b/vulnfeeds/utility/download.go
@@ -62,6 +62,7 @@ func DownloadFile(ctx context.Context, client *http.Client, url, destPath string
 			if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests {
 				return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url))
 			}
+
 			return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)
 		}
 
diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go
index d1eb62b48e4..417e62f361b 100644
--- a/vulnfeeds/utility/download_test.go
+++ b/vulnfeeds/utility/download_test.go
@@ -29,6 +29,7 @@ func TestDownloadFile(t *testing.T) {
 		if r.URL.Path == "/file.txt" {
 			w.WriteHeader(http.StatusOK)
 			_, _ = fmt.Fprint(w, "downloaded content")
+
 			return
 		}
 		w.WriteHeader(http.StatusNotFound)

From 9b6cd2fa966fd5bc45935e621d77c02c92b278dd Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Fri, 21 Aug 2026 03:35:34 +0000
Subject: [PATCH 06/10] prevent zip slip

---
 vulnfeeds/utility/archive.go      | 67 ++++++++++++++++++++++++++-----
 vulnfeeds/utility/archive_test.go | 39 ++++++++++++++++++
 2 files changed, 96 insertions(+), 10 deletions(-)

diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go
index 633acc24358..34aad69f075 100644
--- a/vulnfeeds/utility/archive.go
+++ b/vulnfeeds/utility/archive.go
@@ -16,6 +16,7 @@ package utility
 
 import (
 	"archive/tar"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -23,6 +24,8 @@ import (
 	"strings"
 )
 
+const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit for decompression bomb protection
+
 // CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer.
 func CreateTarArchive(sourceDir string, w io.Writer) error {
 	tw := tar.NewWriter(w)
@@ -74,38 +77,77 @@ func CreateTarArchive(sourceDir string, w io.Writer) error {
 	})
 }
 
-const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit
+// sanitizeTarPath validates that an archive entry name does not escape destDir (Zip Slip protection)
+// and returns the canonical target path.
+func sanitizeTarPath(destDir, name string) (string, error) {
+	cleanDest := filepath.Clean(destDir)
+	cleanName := filepath.Clean(filepath.FromSlash(name))
+
+	// Reject absolute paths and parent directory traversals
+	if filepath.IsAbs(cleanName) || strings.HasPrefix(cleanName, string(os.PathSeparator)) {
+		return "", fmt.Errorf("illegal absolute path in tar archive: %s", name)
+	}
+	if cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(os.PathSeparator)) {
+		return "", fmt.Errorf("path traversal attempt in tar archive: %s", name)
+	}
+
+	targetPath := filepath.Join(cleanDest, cleanName)
+	cleanTarget := filepath.Clean(targetPath)
+
+	// Ensure target path is strictly inside cleanDest
+	rel, err := filepath.Rel(cleanDest, cleanTarget)
+	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
+		return "", fmt.Errorf("path traversal attempt in tar archive: %s", name)
+	}
+
+	return cleanTarget, nil
+}
 
-// ExtractTarArchive extracts a tar archive stream into the destination directory.
+// ExtractTarArchive extracts a tar archive stream into the destination directory,
+// protecting against path traversal (Zip Slip), symlink hijacking, and decompression bombs.
 func ExtractTarArchive(r io.Reader, destDir string) error {
 	tr := tar.NewReader(r)
 	cleanDest := filepath.Clean(destDir)
 
+	if err := os.MkdirAll(cleanDest, 0755); err != nil {
+		return fmt.Errorf("failed to create destination directory %s: %w", cleanDest, err)
+	}
+
 	for {
 		hdr, err := tr.Next()
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 		if err != nil {
 			return fmt.Errorf("error reading tar entry: %w", err)
 		}
 
-		// Prevent Zip Slip / path traversal
-		targetPath := filepath.Join(cleanDest, filepath.FromSlash(hdr.Name))
-		if !strings.HasPrefix(targetPath, cleanDest+string(os.PathSeparator)) && targetPath != cleanDest {
-			return fmt.Errorf("illegal file path in tar archive: %s", hdr.Name)
+		targetPath, err := sanitizeTarPath(cleanDest, hdr.Name)
+		if err != nil {
+			return err
 		}
 
 		switch hdr.Typeflag {
 		case tar.TypeDir:
+			if targetPath == cleanDest {
+				continue
+			}
 			if err := os.MkdirAll(targetPath, 0755); err != nil {
 				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
 			}
 		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
+			if targetPath == cleanDest {
+				return fmt.Errorf("tar entry attempts to overwrite destination directory: %s", hdr.Name)
+			}
+			dir := filepath.Dir(targetPath)
+			if err := os.MkdirAll(dir, 0755); err != nil {
 				return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err)
 			}
-			outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, hdr.FileInfo().Mode())
+
+			// Clean existing file to avoid writing through pre-existing symlinks
+			_ = os.Remove(targetPath)
+
+			outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
 			if err != nil {
 				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
 			}
@@ -113,7 +155,12 @@ func ExtractTarArchive(r io.Reader, destDir string) error {
 				outFile.Close()
 				return fmt.Errorf("failed to write file content %s: %w", targetPath, err)
 			}
-			outFile.Close()
+			if err := outFile.Close(); err != nil {
+				return fmt.Errorf("failed to close file %s: %w", targetPath, err)
+			}
+		default:
+			// Skip unsupported entry types (symlinks, devices, pipes) to prevent link-based traversal attacks
+			continue
 		}
 	}
 
diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go
index a13a075b0c3..f1e79c651f5 100644
--- a/vulnfeeds/utility/archive_test.go
+++ b/vulnfeeds/utility/archive_test.go
@@ -15,6 +15,7 @@
 package utility
 
 import (
+	"archive/tar"
 	"bytes"
 	"os"
 	"path/filepath"
@@ -67,3 +68,41 @@ func TestTarArchiveRoundtrip(t *testing.T) {
 		t.Errorf("file2 content = %q, want %q", string(data2), "nested file content")
 	}
 }
+
+func TestExtractTarArchive_ZipSlipProtection(t *testing.T) {
+	maliciousPaths := []string{
+		"../escaped.txt",
+		"../../../../etc/passwd",
+		"/etc/passwd",
+		"sub/../../escaped.txt",
+		"sub/../../../escaped.txt",
+		"../dest_sibling.txt",
+	}
+
+	for _, malPath := range maliciousPaths {
+		t.Run(malPath, func(t *testing.T) {
+			destDir := filepath.Join(t.TempDir(), "target")
+			var buf bytes.Buffer
+			tw := tar.NewWriter(&buf)
+
+			hdr := &tar.Header{
+				Name:     malPath,
+				Mode:     0600,
+				Size:     int64(len("malicious")),
+				Typeflag: tar.TypeReg,
+			}
+			if err := tw.WriteHeader(hdr); err != nil {
+				t.Fatalf("Failed to write tar header: %v", err)
+			}
+			if _, err := tw.Write([]byte("malicious")); err != nil {
+				t.Fatalf("Failed to write tar body: %v", err)
+			}
+			tw.Close()
+
+			err := ExtractTarArchive(&buf, destDir)
+			if err == nil {
+				t.Errorf("Expected Zip Slip error for path %q, got nil", malPath)
+			}
+		})
+	}
+}

From 507414c65925e78fd4185f18b5bfb95b5c5acb65 Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Mon, 24 Aug 2026 05:36:17 +0000
Subject: [PATCH 07/10] less code

---
 .../debian-copyright-mirror/Dockerfile        |   2 +-
 .../mirrors/debian-copyright-mirror/main.go   | 316 ++++--------------
 .../debian-copyright-mirror/main_test.go      |  53 ++-
 vulnfeeds/utility/archive.go                  | 168 ----------
 vulnfeeds/utility/archive_test.go             | 108 ------
 vulnfeeds/utility/decompress.go               | 130 -------
 vulnfeeds/utility/decompress_test.go          | 108 ------
 vulnfeeds/utility/download.go                 | 103 ------
 vulnfeeds/utility/download_test.go            |  67 ----
 9 files changed, 110 insertions(+), 945 deletions(-)
 delete mode 100644 vulnfeeds/utility/archive.go
 delete mode 100644 vulnfeeds/utility/archive_test.go
 delete mode 100644 vulnfeeds/utility/decompress.go
 delete mode 100644 vulnfeeds/utility/decompress_test.go
 delete mode 100644 vulnfeeds/utility/download.go
 delete mode 100644 vulnfeeds/utility/download_test.go

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
index 2a9dc3025a4..4cfef7f4ecc 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
@@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyr
 
 FROM alpine:3.21
 
-RUN apk add --no-cache xz curl ca-certificates
+RUN apk add --no-cache xz curl tar ca-certificates
 
 COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror
 
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
index 7be32bc1fba..09782dcb556 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -27,14 +27,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"sync/atomic"
-	"time"
 
 	"cloud.google.com/go/storage"
 	gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools"
-	"github.com/google/osv.dev/vulnfeeds/utility"
 	"github.com/google/osv.dev/vulnfeeds/utility/logger"
-	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -42,8 +38,6 @@ const (
 	DefaultURLBase          = "https://metadata.ftp-master.debian.org/changelogs"
 	DefaultPrefixFilter     = "main/"
 	DefaultMinExpectedFiles = 40000
-	DefaultNumWorkers       = 50
-	DefaultMaxFailureRate   = 0.05 // 5% maximum allowed failure rate
 )
 
 // ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader
@@ -60,7 +54,6 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		// Top-level package key: starts without whitespace and ends with colon
 		if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") {
 			inUnstable = false
 			currentPkgFound = false
@@ -73,25 +66,16 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error
 			continue
 		}
 
-		// Check if this is a suite/version subkey under the package
 		if strings.HasPrefix(line, "  ") && !strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ":") {
-			if trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` {
-				inUnstable = true
-			} else {
-				inUnstable = false
-			}
+			inUnstable = trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":`
 
 			continue
 		}
 
-		// If we are within the "unstable" section and haven't found a copyright file for this package yet
 		if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") {
-			entry := strings.TrimSpace(strings.TrimPrefix(trimmed, "- "))
-			entry = strings.Trim(entry, `"'`)
-			if strings.HasSuffix(entry, "unstable_copyright") {
-				if prefixFilter == "" || strings.HasPrefix(entry, prefixFilter) {
-					results = append(results, entry)
-				}
+			entry := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")), `"'`)
+			if strings.HasSuffix(entry, "unstable_copyright") && (prefixFilter == "" || strings.HasPrefix(entry, prefixFilter)) {
+				results = append(results, entry)
 				currentPkgFound = true
 			}
 		}
@@ -104,110 +88,35 @@ func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error
 	return results, nil
 }
 
-// DownloadFilesConcurrently downloads all files in filelist using a channel-based worker pool.
-func DownloadFilesConcurrently(ctx context.Context, client *http.Client, filelist []string, urlBase, workDir string, numWorkers int, skipExisting bool, maxFailureRate float64) error {
-	if numWorkers <= 0 {
-		numWorkers = DefaultNumWorkers
+func fetchCopyrightFiles(ctx context.Context, filelistURL, prefixFilter string) ([]string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, filelistURL, nil)
+	if err != nil {
+		return nil, err
 	}
-
-	total := len(filelist)
-	logger.Info("Starting concurrent download of copyright files",
-		slog.Int("total_files", total),
-		slog.Int("workers", numWorkers),
-		slog.Bool("skip_existing", skipExisting),
-	)
-
-	jobs := make(chan string, numWorkers*2)
-	var completed atomic.Int64
-	var failed atomic.Int64
-	var skipped atomic.Int64
-	startTime := time.Now()
-
-	g, ctx := errgroup.WithContext(ctx)
-
-	for range numWorkers {
-		g.Go(func() error {
-			for path := range jobs {
-				select {
-				case <-ctx.Done():
-					return ctx.Err()
-				default:
-				}
-
-				fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/"))
-				destPath := filepath.Join(workDir, path)
-
-				if skipExisting {
-					if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 {
-						skipped.Add(1)
-						count := completed.Add(1)
-						logProgress(count, total, startTime)
-
-						continue
-					}
-				}
-
-				if err := utility.DownloadFile(ctx, client, fullURL, destPath, false); err != nil {
-					failed.Add(1)
-					logger.Warn("Failed to download copyright file", slog.String("url", fullURL), slog.Any("err", err))
-				} else {
-					count := completed.Add(1)
-					logProgress(count, total, startTime)
-				}
-			}
-
-			return nil
-		})
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
 	}
-
-	go func() {
-		defer close(jobs)
-		for _, path := range filelist {
-			select {
-			case <-ctx.Done():
-				return
-			case jobs <- path:
-			}
-		}
-	}()
-
-	if err := g.Wait(); err != nil {
-		return fmt.Errorf("concurrent download aborted: %w", err)
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP %d fetching %s", resp.StatusCode, filelistURL)
 	}
 
-	completedCount := completed.Load()
-	failedCount := failed.Load()
-	skippedCount := skipped.Load()
-
-	logger.Info("Finished downloading copyright files",
-		slog.Int64("completed", completedCount),
-		slog.Int64("failed", failedCount),
-		slog.Int64("skipped", skippedCount),
-		slog.Int("total", total),
-		slog.Duration("elapsed", time.Since(startTime)),
-	)
-
-	if total > 0 {
-		failRatio := float64(failedCount) / float64(total)
-		if failRatio > maxFailureRate || (completedCount == 0 && total > 0) {
-			return fmt.Errorf("too many downloads failed: %d/%d (%.2f%% failed, max threshold is %.2f%%)",
-				failedCount, total, failRatio*100.0, maxFailureRate*100.0)
-		}
+	cmd := exec.CommandContext(ctx, "xz", "-dc")
+	cmd.Stdin = resp.Body
+	cmd.Stderr = os.Stderr
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, err
 	}
 
-	return nil
-}
+	files, err := ExtractUnstableCopyright(stdout, prefixFilter)
+	_ = cmd.Wait()
 
-func logProgress(count int64, total int, startTime time.Time) {
-	if count%5000 == 0 || count == int64(total) {
-		pct := float64(count) / float64(total) * 100.0
-		logger.Info("Download progress",
-			slog.Int64("completed", count),
-			slog.Int("total", total),
-			slog.Float64("percent", pct),
-			slog.Duration("elapsed", time.Since(startTime)),
-		)
-	}
+	return files, err
 }
 
 // GenerateCurlConfiguration generates a curl config file for parallel downloads.
@@ -248,67 +157,45 @@ func ExecuteCurl(ctx context.Context, configPath, workDir string) error {
 	return nil
 }
 
-// UploadTarToGCS creates a tar archive of workDir and streams it directly to GCS.
-func UploadTarToGCS(ctx context.Context, storageClient *storage.Client, workDir, gcsURI string) error {
+// CreateTarArchive creates a tar archive of workDir using the system tar command.
+func CreateTarArchive(ctx context.Context, workDir, tarPath string) error {
+	// #nosec G204 -- arguments are locally configured paths
+	cmd := exec.CommandContext(ctx, "tar", "-C", workDir, "--exclude="+filepath.Base(tarPath), "-cf", tarPath, ".")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	logger.Info("Creating tar archive", slog.String("workDir", workDir), slog.String("tarPath", tarPath))
+
+	return cmd.Run()
+}
+
+func uploadToGCS(ctx context.Context, tarPath, gcsURI string) error {
 	bucketName, objectName, err := gcs.ParseGCSPath(gcsURI)
 	if err != nil {
 		return err
 	}
-
-	logger.Info("Streaming tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName), slog.String("workDir", workDir))
-	bkt := storageClient.Bucket(bucketName)
-
-	pipeReader, pipeWriter := io.Pipe()
-	errChan := make(chan error, 1)
-
-	go func() {
-		err := utility.CreateTarArchive(workDir, pipeWriter)
-		_ = pipeWriter.CloseWithError(err)
-		errChan <- err
-	}()
-
-	uploadErr := gcs.UploadToGCS(ctx, bkt, objectName, pipeReader, "application/x-tar", nil)
-	archiveErr := <-errChan
-
-	if uploadErr != nil {
-		return fmt.Errorf("failed to upload tar to GCS %s: %w", gcsURI, uploadErr)
-	}
-	if archiveErr != nil {
-		return fmt.Errorf("failed to create tar archive: %w", archiveErr)
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create GCS client: %w", err)
 	}
+	defer client.Close()
 
-	logger.Info("Successfully uploaded tar archive to GCS", slog.String("gcsURI", gcsURI))
+	logger.Info("Uploading tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName))
 
-	return nil
+	return gcs.UploadFile(ctx, client.Bucket(bucketName), objectName, tarPath)
 }
 
 func main() {
 	logger.InitGlobalLogger()
 	defer logger.Close()
 
-	// Input & source configuration flags.
 	filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.")
 	urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.")
 	prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').")
 	minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.")
-
-	// Local output directory & archive flags.
 	outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).")
 	workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.")
-	tarPath := flag.String("tar-path", "", "Optional local destination path for tarball archive (e.g. debian_copyright.tar).")
-
-	// Concurrency & fault tolerance flags.
-	numWorkers := flag.Int("workers", DefaultNumWorkers, "Number of concurrent download workers.")
-	skipExisting := flag.Bool("skip-existing", false, "Skip downloading files that already exist on disk with non-zero size.")
-	maxFailureRate := flag.Float64("max-failure-rate", DefaultMaxFailureRate, "Maximum allowable fraction of failed downloads before failing the job.")
-
-	// GCS upload flags.
-	gcsPath := flag.String("gcs-path", "", "Destination GCS path for tarball archive (e.g. gs://bucket/path/debian_copyright.tar). Defaults to GCS_PATH env var if unset.")
-
-	// Curl fallback flags.
-	useCurl := flag.Bool("use-curl", false, "If true, delegate downloads to curl --parallel instead of Go HTTP workers.")
-	curlConfigFile := flag.String("curl-config-file", "", "Optional path to write the generated curl configuration to.")
-	curlConfigOnly := flag.Bool("curl-config-only", false, "If true, only write the curl configuration file and exit.")
+	tarPathFlag := flag.String("tar-path", "", "Optional local destination path for tarball archive.")
+	gcsPathFlag := flag.String("gcs-path", "", "Destination GCS path for tarball archive (defaults to GCS_PATH env var).")
 
 	flag.Parse()
 
@@ -325,42 +212,16 @@ func main() {
 		}
 	}
 	workDir = filepath.Clean(workDir)
-	if err := os.MkdirAll(workDir, 0755); err != nil {
-		logger.Fatal("Failed to create work directory", slog.String("workDir", workDir), slog.Any("err", err))
-	}
 
 	ctx := context.Background()
 
-	poolSize := *numWorkers * 2
-	if poolSize < 100 {
-		poolSize = 100
-	}
-
-	httpClient := &http.Client{
-		Transport: &http.Transport{
-			MaxIdleConns:        poolSize,
-			MaxIdleConnsPerHost: poolSize,
-			IdleConnTimeout:     90 * time.Second,
-			ForceAttemptHTTP2:   true,
-		},
-	}
-
-	logger.Info("Streaming and decompressing filelist", slog.String("url", *filelistURL))
-	decompressedReader, err := utility.FetchAndDecompressXZ(ctx, httpClient, *filelistURL)
+	logger.Info("Fetching and extracting copyright filelist", slog.String("url", *filelistURL))
+	copyrightFiles, err := fetchCopyrightFiles(ctx, *filelistURL, *prefixFilter)
 	if err != nil {
-		logger.Fatal("Failed to fetch and decompress filelist", slog.Any("err", err))
+		logger.Fatal("Failed to obtain copyright file list", slog.Any("err", err))
 	}
-	defer decompressedReader.Close()
-
-	logger.Info("Extracting unstable copyright file paths", slog.String("prefixFilter", *prefixFilter))
-	copyrightFiles, err := ExtractUnstableCopyright(decompressedReader, *prefixFilter)
-	if err != nil {
-		logger.Fatal("Failed to extract unstable copyright paths", slog.Any("err", err))
-	}
-	_ = decompressedReader.Close()
 
 	logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles)))
-
 	if len(copyrightFiles) < *minExpectedFiles {
 		logger.Fatal("Unexpectedly small number of copyright files found",
 			slog.Int("found", len(copyrightFiles)),
@@ -368,70 +229,39 @@ func main() {
 		)
 	}
 
-	if *useCurl || *curlConfigOnly || *curlConfigFile != "" {
-		cfgPath := *curlConfigFile
-		if cfgPath == "" {
-			tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-curl-*")
-			if err != nil {
-				logger.Fatal("Failed to create temporary directory for curl config", slog.Any("err", err))
-			}
-			defer os.RemoveAll(tempDir)
-			cfgPath = filepath.Join(tempDir, "curl_configuration")
-		}
-
-		logger.Info("Generating curl configuration", slog.String("path", cfgPath))
-		if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil {
-			logger.Fatal("Failed to generate curl configuration", slog.Any("err", err))
-		}
-
-		if *curlConfigOnly {
-			logger.Info("Curl configuration generated successfully; exiting as requested.")
-			return
-		}
+	tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-*")
+	if err != nil {
+		logger.Fatal("Failed to create temp dir", slog.Any("err", err))
+	}
+	defer os.RemoveAll(tempDir)
 
-		if *useCurl {
-			if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil {
-				logger.Fatal("Curl download failed", slog.Any("err", err))
-			}
-		}
-	} else {
-		if err := DownloadFilesConcurrently(ctx, httpClient, copyrightFiles, *urlBase, workDir, *numWorkers, *skipExisting, *maxFailureRate); err != nil {
-			logger.Fatal("Concurrent download failed", slog.Any("err", err))
-		}
+	cfgPath := filepath.Join(tempDir, "curl_configuration")
+	if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil {
+		logger.Fatal("Failed to generate curl configuration", slog.Any("err", err))
 	}
 
-	if *tarPath != "" {
-		logger.Info("Creating local tar archive", slog.String("path", *tarPath), slog.String("workDir", workDir))
-		if err := os.MkdirAll(filepath.Dir(*tarPath), 0755); err != nil {
-			logger.Fatal("Failed to create tar destination directory", slog.Any("err", err))
-		}
-		tarFile, err := os.Create(*tarPath)
-		if err != nil {
-			logger.Fatal("Failed to create tar file", slog.Any("err", err))
-		}
-		if err := utility.CreateTarArchive(workDir, tarFile); err != nil {
-			_ = tarFile.Close()
-			logger.Fatal("Failed to write tar archive", slog.Any("err", err))
-		}
-		if err := tarFile.Close(); err != nil {
-			logger.Fatal("Failed to close tar file", slog.Any("err", err))
-		}
-		logger.Info("Successfully created local tar archive", slog.String("path", *tarPath))
+	if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil {
+		logger.Fatal("Curl download failed", slog.Any("err", err))
 	}
 
-	gcsDest := *gcsPath
+	gcsDest := *gcsPathFlag
 	if gcsDest == "" {
 		gcsDest = os.Getenv("GCS_PATH")
 	}
-	if gcsDest != "" {
-		storageClient, err := storage.NewClient(ctx)
-		if err != nil {
-			logger.Fatal("Failed to create GCS client", slog.Any("err", err))
-		}
-		defer storageClient.Close()
 
-		if err := UploadTarToGCS(ctx, storageClient, workDir, gcsDest); err != nil {
-			logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err))
+	tarDest := *tarPathFlag
+	if tarDest == "" && gcsDest != "" {
+		tarDest = filepath.Join(workDir, filepath.Base(gcsDest))
+	}
+
+	if tarDest != "" {
+		if err := CreateTarArchive(ctx, workDir, tarDest); err != nil {
+			logger.Fatal("Failed to create tar archive", slog.Any("err", err))
+		}
+		if gcsDest != "" {
+			if err := uploadToGCS(ctx, tarDest, gcsDest); err != nil {
+				logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err))
+			}
 		}
 	}
 
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
index c120c844d2d..f772ffff114 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
@@ -139,10 +139,10 @@ func TestGenerateCurlConfiguration(t *testing.T) {
 	}
 }
 
-func TestDownloadFilesConcurrently(t *testing.T) {
+func TestExecuteCurl(t *testing.T) {
 	fileMap := map[string]string{
-		"main/a/pkg1/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/foo/pkg1\n",
-		"main/b/pkg2/unstable_copyright": "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nSource: https://github.com/bar/pkg2\n",
+		"main/a/pkg1/unstable_copyright": "Copyright pkg1\n",
+		"main/b/pkg2/unstable_copyright": "Copyright pkg2\n",
 	}
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -158,37 +158,56 @@ func TestDownloadFilesConcurrently(t *testing.T) {
 	defer ts.Close()
 
 	tempDir := t.TempDir()
-	ctx := context.Background()
-	client := ts.Client()
+	configPath := filepath.Join(tempDir, "curl_config")
+	workDir := filepath.Join(tempDir, "work")
 
 	filelist := []string{
 		"main/a/pkg1/unstable_copyright",
 		"main/b/pkg2/unstable_copyright",
 	}
 
-	if err := DownloadFilesConcurrently(ctx, client, filelist, ts.URL, tempDir, 2, false, 0.05); err != nil {
-		t.Fatalf("DownloadFilesConcurrently failed: %v", err)
+	if err := GenerateCurlConfiguration(filelist, ts.URL, configPath); err != nil {
+		t.Fatalf("GenerateCurlConfiguration failed: %v", err)
+	}
+
+	if err := ExecuteCurl(context.Background(), configPath, workDir); err != nil {
+		t.Fatalf("ExecuteCurl failed: %v", err)
 	}
 
 	for _, relPath := range filelist {
-		fullPath := filepath.Join(tempDir, relPath)
-		data, err := os.ReadFile(fullPath)
+		data, err := os.ReadFile(filepath.Join(workDir, relPath))
 		if err != nil {
-			t.Errorf("Failed to read %s: %v", relPath, err)
+			t.Errorf("Failed to read downloaded file %s: %v", relPath, err)
 			continue
 		}
 		if string(data) != fileMap[relPath] {
 			t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath])
 		}
 	}
+}
 
-	// Test failure threshold breach
-	badFileList := []string{
-		"main/missing/1",
-		"main/missing/2",
+func TestCreateTarArchive(t *testing.T) {
+	tempDir := t.TempDir()
+	workDir := filepath.Join(tempDir, "work")
+	subDir := filepath.Join(workDir, "main", "a", "pkg1")
+	if err := os.MkdirAll(subDir, 0755); err != nil {
+		t.Fatalf("Failed to create test directory: %v", err)
+	}
+	filePath := filepath.Join(subDir, "unstable_copyright")
+	if err := os.WriteFile(filePath, []byte("test copyright content"), 0600); err != nil {
+		t.Fatalf("Failed to write test file: %v", err)
+	}
+
+	tarPath := filepath.Join(workDir, "test_archive.tar")
+	if err := CreateTarArchive(context.Background(), workDir, tarPath); err != nil {
+		t.Fatalf("CreateTarArchive failed: %v", err)
+	}
+
+	fi, err := os.Stat(tarPath)
+	if err != nil {
+		t.Fatalf("Expected tar file to exist: %v", err)
 	}
-	err := DownloadFilesConcurrently(ctx, client, badFileList, ts.URL, tempDir, 2, false, 0.05)
-	if err == nil {
-		t.Errorf("Expected failure threshold error, got nil")
+	if fi.Size() == 0 {
+		t.Errorf("Expected non-empty tar archive")
 	}
 }
diff --git a/vulnfeeds/utility/archive.go b/vulnfeeds/utility/archive.go
deleted file mode 100644
index 34aad69f075..00000000000
--- a/vulnfeeds/utility/archive.go
+++ /dev/null
@@ -1,168 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"archive/tar"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-const maxTarEntrySize = 1 * 1024 * 1024 * 1024 // 1GB max entry size limit for decompression bomb protection
-
-// CreateTarArchive archives all files and subdirectories in sourceDir into the tar writer.
-func CreateTarArchive(sourceDir string, w io.Writer) error {
-	tw := tar.NewWriter(w)
-	defer tw.Close()
-
-	sourceDir = filepath.Clean(sourceDir)
-
-	return filepath.Walk(sourceDir, func(file string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if file == sourceDir {
-			return nil
-		}
-
-		relPath, err := filepath.Rel(sourceDir, file)
-		if err != nil {
-			return fmt.Errorf("failed to get relative path for %s: %w", file, err)
-		}
-
-		tarName := filepath.ToSlash(relPath)
-		if fi.IsDir() {
-			tarName += "/"
-		}
-
-		hdr, err := tar.FileInfoHeader(fi, "")
-		if err != nil {
-			return fmt.Errorf("failed to create tar header for %s: %w", file, err)
-		}
-		hdr.Name = tarName
-
-		if err := tw.WriteHeader(hdr); err != nil {
-			return fmt.Errorf("failed to write tar header for %s: %w", file, err)
-		}
-
-		if fi.Mode().IsRegular() {
-			f, err := os.Open(file)
-			if err != nil {
-				return fmt.Errorf("failed to open file %s: %w", file, err)
-			}
-			defer f.Close()
-
-			if _, err := io.Copy(tw, f); err != nil {
-				return fmt.Errorf("failed to copy file %s to tar: %w", file, err)
-			}
-		}
-
-		return nil
-	})
-}
-
-// sanitizeTarPath validates that an archive entry name does not escape destDir (Zip Slip protection)
-// and returns the canonical target path.
-func sanitizeTarPath(destDir, name string) (string, error) {
-	cleanDest := filepath.Clean(destDir)
-	cleanName := filepath.Clean(filepath.FromSlash(name))
-
-	// Reject absolute paths and parent directory traversals
-	if filepath.IsAbs(cleanName) || strings.HasPrefix(cleanName, string(os.PathSeparator)) {
-		return "", fmt.Errorf("illegal absolute path in tar archive: %s", name)
-	}
-	if cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(os.PathSeparator)) {
-		return "", fmt.Errorf("path traversal attempt in tar archive: %s", name)
-	}
-
-	targetPath := filepath.Join(cleanDest, cleanName)
-	cleanTarget := filepath.Clean(targetPath)
-
-	// Ensure target path is strictly inside cleanDest
-	rel, err := filepath.Rel(cleanDest, cleanTarget)
-	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
-		return "", fmt.Errorf("path traversal attempt in tar archive: %s", name)
-	}
-
-	return cleanTarget, nil
-}
-
-// ExtractTarArchive extracts a tar archive stream into the destination directory,
-// protecting against path traversal (Zip Slip), symlink hijacking, and decompression bombs.
-func ExtractTarArchive(r io.Reader, destDir string) error {
-	tr := tar.NewReader(r)
-	cleanDest := filepath.Clean(destDir)
-
-	if err := os.MkdirAll(cleanDest, 0755); err != nil {
-		return fmt.Errorf("failed to create destination directory %s: %w", cleanDest, err)
-	}
-
-	for {
-		hdr, err := tr.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
-		if err != nil {
-			return fmt.Errorf("error reading tar entry: %w", err)
-		}
-
-		targetPath, err := sanitizeTarPath(cleanDest, hdr.Name)
-		if err != nil {
-			return err
-		}
-
-		switch hdr.Typeflag {
-		case tar.TypeDir:
-			if targetPath == cleanDest {
-				continue
-			}
-			if err := os.MkdirAll(targetPath, 0755); err != nil {
-				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
-			}
-		case tar.TypeReg:
-			if targetPath == cleanDest {
-				return fmt.Errorf("tar entry attempts to overwrite destination directory: %s", hdr.Name)
-			}
-			dir := filepath.Dir(targetPath)
-			if err := os.MkdirAll(dir, 0755); err != nil {
-				return fmt.Errorf("failed to create directory for file %s: %w", targetPath, err)
-			}
-
-			// Clean existing file to avoid writing through pre-existing symlinks
-			_ = os.Remove(targetPath)
-
-			outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
-			if err != nil {
-				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
-			}
-			if _, err := io.Copy(outFile, io.LimitReader(tr, maxTarEntrySize)); err != nil {
-				outFile.Close()
-				return fmt.Errorf("failed to write file content %s: %w", targetPath, err)
-			}
-			if err := outFile.Close(); err != nil {
-				return fmt.Errorf("failed to close file %s: %w", targetPath, err)
-			}
-		default:
-			// Skip unsupported entry types (symlinks, devices, pipes) to prevent link-based traversal attacks
-			continue
-		}
-	}
-
-	return nil
-}
diff --git a/vulnfeeds/utility/archive_test.go b/vulnfeeds/utility/archive_test.go
deleted file mode 100644
index f1e79c651f5..00000000000
--- a/vulnfeeds/utility/archive_test.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"archive/tar"
-	"bytes"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestTarArchiveRoundtrip(t *testing.T) {
-	tempDir := t.TempDir()
-	sourceDir := filepath.Join(tempDir, "source")
-	destDir := filepath.Join(tempDir, "dest")
-
-	if err := os.MkdirAll(filepath.Join(sourceDir, "sub"), 0755); err != nil {
-		t.Fatalf("Failed to create test source dir: %v", err)
-	}
-
-	file1 := filepath.Join(sourceDir, "file1.txt")
-	file2 := filepath.Join(sourceDir, "sub", "file2.txt")
-	if err := os.WriteFile(file1, []byte("hello archive"), 0600); err != nil {
-		t.Fatalf("Failed to write file1: %v", err)
-	}
-	if err := os.WriteFile(file2, []byte("nested file content"), 0600); err != nil {
-		t.Fatalf("Failed to write file2: %v", err)
-	}
-
-	var buf bytes.Buffer
-	if err := CreateTarArchive(sourceDir, &buf); err != nil {
-		t.Fatalf("CreateTarArchive failed: %v", err)
-	}
-
-	if err := ExtractTarArchive(&buf, destDir); err != nil {
-		t.Fatalf("ExtractTarArchive failed: %v", err)
-	}
-
-	destFile1 := filepath.Join(destDir, "file1.txt")
-	destFile2 := filepath.Join(destDir, "sub", "file2.txt")
-
-	data1, err := os.ReadFile(destFile1)
-	if err != nil {
-		t.Fatalf("Failed to read extracted file1: %v", err)
-	}
-	if string(data1) != "hello archive" {
-		t.Errorf("file1 content = %q, want %q", string(data1), "hello archive")
-	}
-
-	data2, err := os.ReadFile(destFile2)
-	if err != nil {
-		t.Fatalf("Failed to read extracted file2: %v", err)
-	}
-	if string(data2) != "nested file content" {
-		t.Errorf("file2 content = %q, want %q", string(data2), "nested file content")
-	}
-}
-
-func TestExtractTarArchive_ZipSlipProtection(t *testing.T) {
-	maliciousPaths := []string{
-		"../escaped.txt",
-		"../../../../etc/passwd",
-		"/etc/passwd",
-		"sub/../../escaped.txt",
-		"sub/../../../escaped.txt",
-		"../dest_sibling.txt",
-	}
-
-	for _, malPath := range maliciousPaths {
-		t.Run(malPath, func(t *testing.T) {
-			destDir := filepath.Join(t.TempDir(), "target")
-			var buf bytes.Buffer
-			tw := tar.NewWriter(&buf)
-
-			hdr := &tar.Header{
-				Name:     malPath,
-				Mode:     0600,
-				Size:     int64(len("malicious")),
-				Typeflag: tar.TypeReg,
-			}
-			if err := tw.WriteHeader(hdr); err != nil {
-				t.Fatalf("Failed to write tar header: %v", err)
-			}
-			if _, err := tw.Write([]byte("malicious")); err != nil {
-				t.Fatalf("Failed to write tar body: %v", err)
-			}
-			tw.Close()
-
-			err := ExtractTarArchive(&buf, destDir)
-			if err == nil {
-				t.Errorf("Expected Zip Slip error for path %q, got nil", malPath)
-			}
-		})
-	}
-}
diff --git a/vulnfeeds/utility/decompress.go b/vulnfeeds/utility/decompress.go
deleted file mode 100644
index 32080ab6cb8..00000000000
--- a/vulnfeeds/utility/decompress.go
+++ /dev/null
@@ -1,130 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-)
-
-// SubprocessReadCloser wraps a command's stdout pipe and ensures the subprocess
-// is waited for and input resources are closed when Close() is called.
-type SubprocessReadCloser struct {
-	io.ReadCloser
-
-	cmd         *exec.Cmd
-	bodyToClose io.Closer
-}
-
-// Close closes the underlying reader pipe, any input reader, and waits for the subprocess.
-func (s *SubprocessReadCloser) Close() error {
-	var errs []error
-	if s.ReadCloser != nil {
-		if err := s.ReadCloser.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-	if s.bodyToClose != nil {
-		if err := s.bodyToClose.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-	if s.cmd != nil {
-		if err := s.cmd.Wait(); err != nil {
-			var exitErr *exec.ExitError
-			if errors.As(err, &exitErr) && !exitErr.Success() {
-				errs = append(errs, err)
-			}
-		}
-	}
-
-	return errors.Join(errs...)
-}
-
-func getXZDecompressCommand() (string, error) {
-	if _, err := exec.LookPath("xz"); err == nil {
-		return "xz", nil
-	}
-	if _, err := exec.LookPath("unxz"); err == nil {
-		return "unxz", nil
-	}
-
-	return "", errors.New("neither 'xz' nor 'unxz' command-line tool found in PATH")
-}
-
-// DecompressXZ streams an xz-compressed reader through the system xz or unxz command.
-func DecompressXZ(ctx context.Context, r io.Reader) (io.ReadCloser, error) {
-	decompressCmd, err := getXZDecompressCommand()
-	if err != nil {
-		return nil, err
-	}
-
-	cmd := exec.CommandContext(ctx, decompressCmd, "-dc")
-	cmd.Stdin = r
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create stdout pipe for %s: %w", decompressCmd, err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("failed to start %s: %w", decompressCmd, err)
-	}
-
-	var bodyToClose io.Closer
-	if closer, ok := r.(io.Closer); ok {
-		bodyToClose = closer
-	}
-
-	return &SubprocessReadCloser{
-		ReadCloser:  stdout,
-		cmd:         cmd,
-		bodyToClose: bodyToClose,
-	}, nil
-}
-
-// DecompressXZFile decompresses a local xz-compressed file.
-func DecompressXZFile(ctx context.Context, xzFilePath string) (io.ReadCloser, error) {
-	f, err := os.Open(xzFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open %s: %w", xzFilePath, err)
-	}
-
-	return DecompressXZ(ctx, f)
-}
-
-// FetchAndDecompressXZ fetches an xz-compressed URL over HTTP and streams
-// its decompressed content directly through xz/unxz without saving to disk.
-func FetchAndDecompressXZ(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request for %s: %w", url, err)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch %s: %w", url, err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		resp.Body.Close()
-		return nil, fmt.Errorf("HTTP status %d fetching %s", resp.StatusCode, url)
-	}
-
-	return DecompressXZ(ctx, resp.Body)
-}
diff --git a/vulnfeeds/utility/decompress_test.go b/vulnfeeds/utility/decompress_test.go
deleted file mode 100644
index e66ca858e8e..00000000000
--- a/vulnfeeds/utility/decompress_test.go
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-var testXZPayload = []byte{
-	0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00, 0x00, 0x04, 0xe6, 0xd6, 0xb4, 0x46,
-	0x02, 0x00, 0x21, 0x01, 0x16, 0x00, 0x00, 0x00, 0x74, 0x2f, 0xe5, 0xa3,
-	0x01, 0x00, 0x1d, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x44, 0x65, 0x62,
-	0x69, 0x61, 0x6e, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68,
-	0x74, 0x20, 0x4d, 0x69, 0x72, 0x72, 0x6f, 0x72, 0x21, 0x00, 0x00, 0x00,
-	0xc7, 0x2f, 0x01, 0x8a, 0x65, 0x5a, 0x9a, 0x97, 0x00, 0x01, 0x36, 0x1e,
-	0x3d, 0x19, 0x95, 0x53, 0x1f, 0xb6, 0xf3, 0x7d, 0x01, 0x00, 0x00, 0x00,
-	0x00, 0x04, 0x59, 0x5a,
-}
-
-func TestDecompressXZ(t *testing.T) {
-	ctx := context.Background()
-	reader, err := DecompressXZ(ctx, bytes.NewReader(testXZPayload))
-	if err != nil {
-		t.Fatalf("DecompressXZ failed: %v", err)
-	}
-	defer reader.Close()
-
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		t.Fatalf("Failed to read decompressed data: %v", err)
-	}
-
-	want := "Hello Debian Copyright Mirror!"
-	if string(data) != want {
-		t.Errorf("Decompressed = %q, want %q", string(data), want)
-	}
-}
-
-func TestDecompressXZFile(t *testing.T) {
-	tempDir := t.TempDir()
-	filePath := filepath.Join(tempDir, "test.xz")
-	if err := os.WriteFile(filePath, testXZPayload, 0600); err != nil {
-		t.Fatalf("Failed to write test file: %v", err)
-	}
-
-	ctx := context.Background()
-	reader, err := DecompressXZFile(ctx, filePath)
-	if err != nil {
-		t.Fatalf("DecompressXZFile failed: %v", err)
-	}
-	defer reader.Close()
-
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		t.Fatalf("Failed to read decompressed data: %v", err)
-	}
-
-	want := "Hello Debian Copyright Mirror!"
-	if string(data) != want {
-		t.Errorf("Decompressed = %q, want %q", string(data), want)
-	}
-}
-
-func TestFetchAndDecompressXZ(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write(testXZPayload)
-	}))
-	defer ts.Close()
-
-	ctx := context.Background()
-	client := ts.Client()
-
-	reader, err := FetchAndDecompressXZ(ctx, client, ts.URL+"/file.xz")
-	if err != nil {
-		t.Fatalf("FetchAndDecompressXZ failed: %v", err)
-	}
-	defer reader.Close()
-
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		t.Fatalf("Failed to read decompressed data: %v", err)
-	}
-
-	want := "Hello Debian Copyright Mirror!"
-	if string(data) != want {
-		t.Errorf("Decompressed = %q, want %q", string(data), want)
-	}
-}
diff --git a/vulnfeeds/utility/download.go b/vulnfeeds/utility/download.go
deleted file mode 100644
index df38ee6725e..00000000000
--- a/vulnfeeds/utility/download.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/sethvargo/go-retry"
-)
-
-const (
-	DefaultDownloadTimeout = 30 * time.Second
-)
-
-// DownloadFile downloads a URL to a local destination file with automatic retries,
-// safe temporary file writing, and on-demand directory creation.
-// If skipExisting is true and destPath exists with non-zero size, the download is skipped.
-func DownloadFile(ctx context.Context, client *http.Client, url, destPath string, skipExisting bool) error {
-	if skipExisting {
-		if fi, err := os.Stat(destPath); err == nil && fi.Size() > 0 {
-			return nil
-		}
-	}
-
-	backoff := retry.NewExponential(1 * time.Second)
-	backoff = retry.WithMaxRetries(3, backoff)
-
-	return retry.Do(ctx, backoff, func(ctx context.Context) error {
-		reqCtx, cancel := context.WithTimeout(ctx, DefaultDownloadTimeout)
-		defer cancel()
-
-		req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil)
-		if err != nil {
-			return fmt.Errorf("failed to create request for %s: %w", url, err)
-		}
-
-		resp, err := client.Do(req)
-		if err != nil {
-			return retry.RetryableError(fmt.Errorf("HTTP request failed for %s: %w", url, err))
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode != http.StatusOK {
-			if resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests {
-				return retry.RetryableError(fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url))
-			}
-
-			return fmt.Errorf("HTTP status %d for %s", resp.StatusCode, url)
-		}
-
-		dir := filepath.Dir(destPath)
-		tmpFile, err := os.CreateTemp(dir, "download-*")
-		if err != nil {
-			if os.IsNotExist(err) {
-				if mkdirErr := os.MkdirAll(dir, 0755); mkdirErr != nil {
-					return fmt.Errorf("failed to create directory %s: %w", dir, mkdirErr)
-				}
-				tmpFile, err = os.CreateTemp(dir, "download-*")
-			}
-			if err != nil {
-				return fmt.Errorf("failed to create temp file in %s: %w", dir, err)
-			}
-		}
-		tmpPath := tmpFile.Name()
-
-		_, copyErr := io.Copy(tmpFile, resp.Body)
-		closeErr := tmpFile.Close()
-
-		if copyErr != nil {
-			_ = os.Remove(tmpPath)
-			return retry.RetryableError(fmt.Errorf("failed to write data: %w", copyErr))
-		}
-		if closeErr != nil {
-			_ = os.Remove(tmpPath)
-			return fmt.Errorf("failed to close temp file: %w", closeErr)
-		}
-
-		if err := os.Rename(tmpPath, destPath); err != nil {
-			_ = os.Remove(tmpPath)
-			return fmt.Errorf("failed to move temp file to %s: %w", destPath, err)
-		}
-
-		return nil
-	})
-}
diff --git a/vulnfeeds/utility/download_test.go b/vulnfeeds/utility/download_test.go
deleted file mode 100644
index 417e62f361b..00000000000
--- a/vulnfeeds/utility/download_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utility
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestDownloadFile(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/file.txt" {
-			w.WriteHeader(http.StatusOK)
-			_, _ = fmt.Fprint(w, "downloaded content")
-
-			return
-		}
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	tempDir := t.TempDir()
-	destPath := filepath.Join(tempDir, "nested", "dir", "file.txt")
-
-	ctx := context.Background()
-	client := ts.Client()
-
-	if err := DownloadFile(ctx, client, ts.URL+"/file.txt", destPath, false); err != nil {
-		t.Fatalf("DownloadFile failed: %v", err)
-	}
-
-	data, err := os.ReadFile(destPath)
-	if err != nil {
-		t.Fatalf("Failed to read downloaded file: %v", err)
-	}
-	if string(data) != "downloaded content" {
-		t.Errorf("content = %q, want %q", string(data), "downloaded content")
-	}
-
-	// Test skipExisting = true
-	if err := DownloadFile(ctx, client, ts.URL+"/404-should-skip", destPath, true); err != nil {
-		t.Fatalf("DownloadFile with skipExisting should have succeeded without error, got %v", err)
-	}
-
-	// Test 404
-	err = DownloadFile(ctx, client, ts.URL+"/nonexistent", filepath.Join(tempDir, "404.txt"), false)
-	if err == nil {
-		t.Errorf("Expected error for 404, got nil")
-	}
-}

From 95dcbeecae885d150654ce9d3e69505459dc7489 Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Fri, 25 Sep 2026 04:25:54 +0000
Subject: [PATCH 08/10] fix some things

---
 .../debian-copyright-mirror/Dockerfile        |  2 +-
 .../mirrors/debian-copyright-mirror/README.md | 39 ++++++-------------
 .../mirrors/debian-copyright-mirror/main.go   |  3 +-
 .../debian-copyright-mirror/main_test.go      | 39 ++++++++++++++++++-
 4 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
index 4cfef7f4ecc..250cf649ce1 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile
@@ -24,7 +24,7 @@ RUN go mod download
 COPY ./ /src/
 RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror
 
-FROM alpine:3.21
+FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507
 
 RUN apk add --no-cache xz curl tar ca-certificates
 
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
index 89a32d77893..16e00721604 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md
@@ -39,22 +39,17 @@ The `Source:` field provides a canonical, maintainer-verified URL to the upstrea
 graph TD
     A["Debian FTP Master<br/>(filelist.yaml.xz)"] -->|HTTP Stream & xz -dc| B["debian-copyright-mirror<br/>(Go binary)"]
     B -->|Streaming YAML Parser| C["Extract ~44k+ main/<br/>unstable_copyright paths"]
-    C -->|Channel-based Worker Pool| D["Download copyright files<br/>to local directory"]
-    D -->|Validate failure rate & count| E["Native Tar Streaming"]
+    C -->|Generate curl config & run curl --parallel| D["Download copyright files<br/>to local directory"]
+    D -->|tar -C workDir -cf tarPath .| E["Create Tar Archive"]
     E -->|GCS Client Writer| F["GCS Bucket<br/>(gs://cve-osv-conversion/...)"]
     F -->|tar -xf| G["cpe-repo-gen<br/>(CPE to Repo Mapping)"]
 ```
 
-1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it directly through `xz -dc` stdin without intermediate disk roundtrips.
-2. **Manifest Parsing**: Parses the YAML manifest to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section.
+1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it through `xz -dc` stdin.
+2. **Manifest Parsing**: Parses the YAML manifest line-by-line to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section.
 3. **Validation**: Asserts that the number of discovered files meets a sanity threshold (by default, at least 40,000 files) to ensure upstream feeds were not corrupted or truncated.
-4. **Optimized Concurrent Downloads**:
-   - Spawns a channel-based worker pool (default 50 workers) with HTTP/2 keep-alive connection reuse and exponential backoff retries.
-   - Optimized file and directory creation (avoids redundant `MkdirAll` calls on existing directories).
-   - Tracks success/failure metrics and enforces a maximum failure threshold (`-max-failure-rate`) to prevent false-positive completions on network or CDN outages.
-   - Supports incremental runs with `-skip-existing` to skip re-downloading unchanged files.
-   - Optional: Delegated download via `curl --parallel` using generated curl configuration files (`-use-curl`).
-5. **Native Archival & GCS Streaming**: Packages the downloaded mirror into a `.tar` archive and streams it directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) using the Cloud Storage client.
+4. **Parallel Downloads via Curl**: Generates a configuration file and executes `curl --parallel --create-dirs --config <config>` to download all copyright files into `<workDir>/metadata.ftp-master.debian.org/changelogs/`.
+5. **Archiving & GCS Upload**: Uses `tar` to archive `<workDir>` (producing `./metadata.ftp-master.debian.org/changelogs/main/...` entries expected by `cpe-repo-gen`) and uploads the resulting `.tar` archive directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`).
 
 ---
 
@@ -70,19 +65,16 @@ go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir]
 Examples:
 
 ```bash
-# Download copyright files into ./debian_copyright (relative to current directory) using 50 workers
-go run ./cmd/mirrors/debian-copyright-mirror -workers 50
+# Download copyright files into ./debian_copyright (relative to current directory)
+go run ./cmd/mirrors/debian-copyright-mirror
 
 # Specify a custom relative output directory
 go run ./cmd/mirrors/debian-copyright-mirror -out-dir debian_copyright
 
-# Incremental run skipping already downloaded files
-go run ./cmd/mirrors/debian-copyright-mirror -skip-existing
-
 # Download and create a local tar archive
 go run ./cmd/mirrors/debian-copyright-mirror -tar-path debian_copyright.tar
 
-# Download and stream directly to GCS
+# Download and upload directly to GCS
 go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar
 ```
 
@@ -90,21 +82,14 @@ go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_cop
 
 | Flag | Type | Default | Description |
 | :--- | :--- | :--- | :--- |
-| `-out-dir` | string | `debian_copyright` | Target directory (relative or absolute) to save downloaded copyright files. |
+| `-out-dir` | string | `debian_copyright` | Target directory to save downloaded copyright files (defaults to `$WORK_DIR` if set). |
 | `-work-dir` | string | `""` | Alias for `-out-dir` (can also be passed as the first positional argument). |
-| `-workers` | int | `50` | Number of concurrent download workers. |
-| `-skip-existing` | bool | `false` | Skip downloading files that already exist on disk with non-zero size. |
-| `-max-failure-rate` | float | `0.05` | Maximum allowable fraction of failed downloads (e.g. `0.05` = 5%) before failing the job. |
-| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (e.g. `gs://bucket/debian_copyright.tar`). Defaults to `$GCS_PATH` if unset. |
 | `-tar-path` | string | `""` | Optional local destination path for tarball archive (e.g. `debian_copyright.tar`). |
+| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (defaults to `$GCS_PATH` if unset). |
 | `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. |
 | `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. |
 | `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). |
 | `-min-expected-files` | int | `40000` | Minimum expected number of copyright files; fails if fewer are found. |
-| `-use-curl` | bool | `false` | Delegate downloads to `curl --parallel` instead of Go worker pool. |
-| `-curl-config-file` | string | `""` | Optional path to output the generated curl configuration file. |
-| `-curl-config-only` | bool | `false` | Generate the curl configuration file and exit immediately without downloading. |
-
 
 ---
 
@@ -123,4 +108,4 @@ docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/deb
 In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays):
 
 - **Schedule**: Runs daily (`0 5 * * *` Sydney time).
-- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment and streams the archive directly to Cloud Storage.
+- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment, downloads with `curl`, packages with `tar`, and uploads the archive to Cloud Storage.
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
index 09782dcb556..74ab250d31e 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -240,7 +240,8 @@ func main() {
 		logger.Fatal("Failed to generate curl configuration", slog.Any("err", err))
 	}
 
-	if err := ExecuteCurl(ctx, cfgPath, workDir); err != nil {
+	downloadDir := filepath.Join(workDir, "metadata.ftp-master.debian.org", "changelogs")
+	if err := ExecuteCurl(ctx, cfgPath, downloadDir); err != nil {
 		logger.Fatal("Curl download failed", slog.Any("err", err))
 	}
 
diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
index f772ffff114..f41156f0b6d 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go
@@ -15,8 +15,11 @@
 package main
 
 import (
+	"archive/tar"
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -140,6 +143,7 @@ func TestGenerateCurlConfiguration(t *testing.T) {
 }
 
 func TestExecuteCurl(t *testing.T) {
+	t.Setenv("NO_PROXY", "127.0.0.1,localhost")
 	fileMap := map[string]string{
 		"main/a/pkg1/unstable_copyright": "Copyright pkg1\n",
 		"main/b/pkg2/unstable_copyright": "Copyright pkg2\n",
@@ -189,7 +193,7 @@ func TestExecuteCurl(t *testing.T) {
 func TestCreateTarArchive(t *testing.T) {
 	tempDir := t.TempDir()
 	workDir := filepath.Join(tempDir, "work")
-	subDir := filepath.Join(workDir, "main", "a", "pkg1")
+	subDir := filepath.Join(workDir, "metadata.ftp-master.debian.org", "changelogs", "main", "a", "pkg1")
 	if err := os.MkdirAll(subDir, 0755); err != nil {
 		t.Fatalf("Failed to create test directory: %v", err)
 	}
@@ -210,4 +214,37 @@ func TestCreateTarArchive(t *testing.T) {
 	if fi.Size() == 0 {
 		t.Errorf("Expected non-empty tar archive")
 	}
+
+	f, err := os.Open(tarPath)
+	if err != nil {
+		t.Fatalf("Failed to open tar archive: %v", err)
+	}
+	defer f.Close()
+
+	tr := tar.NewReader(f)
+	var entries []string
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			t.Fatalf("Error reading tar archive: %v", err)
+		}
+		entries = append(entries, hdr.Name)
+	}
+
+	wantEntry := "./metadata.ftp-master.debian.org/changelogs/main/a/pkg1/unstable_copyright"
+	found := false
+	for _, entry := range entries {
+		if entry == wantEntry {
+			found = true
+		}
+		if strings.HasSuffix(entry, "test_archive.tar") {
+			t.Errorf("Tar archive should not contain itself, found entry: %s", entry)
+		}
+	}
+	if !found {
+		t.Errorf("Expected tar entry %q not found in archive; entries = %v", wantEntry, entries)
+	}
 }

From 0077acde2cb3590233d81d1d85554b4325290e92 Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Fri, 25 Sep 2026 04:38:10 +0000
Subject: [PATCH 09/10] update dead link

---
 .../posts/supporting-debian-security-tracker-data/index.md      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/blog/content/posts/supporting-debian-security-tracker-data/index.md b/website/blog/content/posts/supporting-debian-security-tracker-data/index.md
index 1fd4cb480f9..653dd9b4459 100644
--- a/website/blog/content/posts/supporting-debian-security-tracker-data/index.md
+++ b/website/blog/content/posts/supporting-debian-security-tracker-data/index.md
@@ -15,7 +15,7 @@ A [sample CVE entry](https://osv.dev/vulnerability/CVE-2024-2466) with Debian in
 The [Debian Security Tracker](https://security-tracker.debian.org/tracker/) is a database maintained by Debian's security team to track Debian related security vulnerabilities. It aggregates information from various sources, including [Debian Security Advisories](https://www.debian.org/security/#DSAS) (DSAs), the [Common Vulnerabilities and Exposures](https://www.cve.org/) (CVE) database, the [National Vulnerability Database](https://nvd.nist.gov/) (NVD), and the [Debian bug tracking system](https://www.debian.org/Bugs/) (BTS). This makes the security tracker comprehensive, providing a complete record of reported Debian security vulnerabilities.
 
 ## Why do we need it?
-Previously, OSV.dev [relied](https://google.github.io/osv.dev/data/#converted-data) solely on Debian Security Advisories (DSAs) and Debian Long Term Support Advisories (DLAs) for Debian related vulnerability information. While these are official sources, they only cover issues that have been formally [acknowledged and patched](https://www.debian.org/doc/manuals/securing-debian-manual/dsa.en.html) by Debian. This left a blind spot in our data coverage – unfixed vulnerabilities, where the security issue has been reported but a fix has not been made available yet in Debian.
+Previously, OSV.dev [relied](https://google.github.io/osv.dev/data/#converted-data) solely on Debian Security Advisories (DSAs) and Debian Long Term Support Advisories (DLAs) for Debian related vulnerability information. While these are official sources, they only cover issues that have been formally [acknowledged and patched](https://www.debian.org/security/faq#handling) by Debian. This left a blind spot in our data coverage – unfixed vulnerabilities, where the security issue has been reported but a fix has not been made available yet in Debian.
 
 Debian Security Tracker fills this gap by providing all reported issues even if the vulnerability has not been fixed yet.
 

From 784523cbdeeb1b14ca13c3bcf69edbe66b768906 Mon Sep 17 00:00:00 2001
From: Jess Lowe <jesslowe@google.com>
Date: Mon, 28 Sep 2026 03:32:04 +0000
Subject: [PATCH 10/10] return wait error

---
 vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
index 74ab250d31e..0e193b4e4ca 100644
--- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
+++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go
@@ -18,6 +18,7 @@ package main
 import (
 	"bufio"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -114,9 +115,14 @@ func fetchCopyrightFiles(ctx context.Context, filelistURL, prefixFilter string)
 	}
 
 	files, err := ExtractUnstableCopyright(stdout, prefixFilter)
-	_ = cmd.Wait()
+	if waitErr := cmd.Wait(); waitErr != nil {
+		err = errors.Join(err, fmt.Errorf("xz decompression failed: %w", waitErr))
+	}
+	if err != nil {
+		return nil, err
+	}
 
-	return files, err
+	return files, nil
 }
 
 // GenerateCurlConfiguration generates a curl config file for parallel downloads.