diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 7adcc171682..2906141470c 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -425,8 +425,8 @@ steps: # Build/push Debian copyright mirror image - name: gcr.io/cloud-builders/docker - args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.'] - dir: 'vulnfeeds/cmd/mirrors/debian-copyright-mirror' + args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '-f', 'cmd/mirrors/debian-copyright-mirror/Dockerfile', '.'] + dir: 'vulnfeeds' id: 'build-debian-copyright-mirror' waitFor: ['setup'] - name: gcr.io/cloud-builders/docker diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile index 09afd30ba3f..250cf649ce1 100644 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile @@ -12,11 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb +FROM golang:1.26.5-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS GO_BUILD -RUN apk add py3-yaml +RUN mkdir /src +WORKDIR /src -COPY ./debian-copyright-mirror.sh / -COPY ./debian-copyright-mirror.py / +COPY ./go.mod /src/go.mod +COPY ./go.sum /src/go.sum +RUN go mod download -ENTRYPOINT ["/debian-copyright-mirror.sh"] +COPY ./ /src/ +RUN CGO_ENABLED=0 go build -o debian-copyright-mirror ./cmd/mirrors/debian-copyright-mirror + +FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 + +RUN apk add --no-cache xz curl tar ca-certificates + +COPY --from=GO_BUILD /src/debian-copyright-mirror /usr/local/bin/debian-copyright-mirror + +ENTRYPOINT ["/usr/local/bin/debian-copyright-mirror"] diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md new file mode 100644 index 00000000000..16e00721604 --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/README.md @@ -0,0 +1,111 @@ +# Debian Copyright Mirror + +`debian-copyright-mirror` maintains a mirror of machine-readable Debian package copyright files for packages in Debian `unstable` (`main` section). + +--- + +## Purpose & Usage in OSV + +In the OSV and vulnerability feeds pipeline, correlating Common Vulnerabilities and Exposures (CVE) entries and Common Platform Enumeration (CPE) identifiers with upstream open-source source code repositories (e.g., on GitHub, GitLab, or Git) is a crucial step for automated vulnerability management. + +### The Problem +- CVE entries and NVD CPE dictionaries often lack structured upstream repository URLs, or only provide arbitrary website references. +- Without a reliable mapping from package names and CPE products to source code repositories, automated tools cannot easily determine affected Git commits, tags, or version ranges for Git-based vulnerability matching. + +### How Debian Copyright Files Solve This +Debian packages in `unstable` follow the [Machine-readable debian/copyright format (DEP-5 / Copyright Specification 1.0)](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/). These files contain structured metadata in header paragraphs, notably: + +```text +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: example-lib +Source: https://github.com/example/example-lib +``` + +The `Source:` field provides a canonical, maintainer-verified URL to the upstream open-source repository. + +### Downstream Consumers +1. **`cpe-repo-gen`** (`cmd/mirrors/cpe-repo-gen`): + - Downloads the `debian_copyright.tar` archive generated by this tool from Google Cloud Storage (`gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`). + - Scans the mirrored `unstable_copyright` files to resolve Debian package names and CPE products to upstream source code repositories. + - Produces the canonical `cpe_product_to_repo.json` mapping. +2. **OSV Conversion & Enrichment Pipelines**: + - Use the repository mappings derived from Debian copyright data to accurately calculate affected commit ranges, detect fixed revisions, and generate OSV-format records for Debian, NVD, and other feeds. + +--- + +## How It Works + +```mermaid +graph TD + A["Debian FTP Master
(filelist.yaml.xz)"] -->|HTTP Stream & xz -dc| B["debian-copyright-mirror
(Go binary)"] + B -->|Streaming YAML Parser| C["Extract ~44k+ main/
unstable_copyright paths"] + C -->|Generate curl config & run curl --parallel| D["Download copyright files
to local directory"] + D -->|tar -C workDir -cf tarPath .| E["Create Tar Archive"] + E -->|GCS Client Writer| F["GCS Bucket
(gs://cve-osv-conversion/...)"] + F -->|tar -xf| G["cpe-repo-gen
(CPE to Repo Mapping)"] +``` + +1. **Manifest Streaming & Decompression**: Fetches `filelist.yaml.xz` from `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` over HTTP and streams it through `xz -dc` stdin. +2. **Manifest Parsing**: Parses the YAML manifest line-by-line to discover all packages containing an `unstable_copyright` entry under the `unstable` suite in the `main` archive section. +3. **Validation**: Asserts that the number of discovered files meets a sanity threshold (by default, at least 40,000 files) to ensure upstream feeds were not corrupted or truncated. +4. **Parallel Downloads via Curl**: Generates a configuration file and executes `curl --parallel --create-dirs --config ` to download all copyright files into `/metadata.ftp-master.debian.org/changelogs/`. +5. **Archiving & GCS Upload**: Uses `tar` to archive `` (producing `./metadata.ftp-master.debian.org/changelogs/main/...` entries expected by `cpe-repo-gen`) and uploads the resulting `.tar` archive directly to Google Cloud Storage (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`). + +--- + +## CLI Usage + +### Running Locally + +```bash +# Build and run directly using Go +go run ./cmd/mirrors/debian-copyright-mirror [flags] [work_dir] +``` + +Examples: + +```bash +# Download copyright files into ./debian_copyright (relative to current directory) +go run ./cmd/mirrors/debian-copyright-mirror + +# Specify a custom relative output directory +go run ./cmd/mirrors/debian-copyright-mirror -out-dir debian_copyright + +# Download and create a local tar archive +go run ./cmd/mirrors/debian-copyright-mirror -tar-path debian_copyright.tar + +# Download and upload directly to GCS +go run ./cmd/mirrors/debian-copyright-mirror -gcs-path gs://my-bucket/debian_copyright.tar +``` + +### Flags + +| Flag | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `-out-dir` | string | `debian_copyright` | Target directory to save downloaded copyright files (defaults to `$WORK_DIR` if set). | +| `-work-dir` | string | `""` | Alias for `-out-dir` (can also be passed as the first positional argument). | +| `-tar-path` | string | `""` | Optional local destination path for tarball archive (e.g. `debian_copyright.tar`). | +| `-gcs-path` | string | `""` | Destination GCS path for the tarball archive (defaults to `$GCS_PATH` if unset). | +| `-filelist-url` | string | `https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz` | URL of the Debian filelist YAML archive. | +| `-url-base` | string | `https://metadata.ftp-master.debian.org/changelogs` | Base URL for downloading individual copyright files. | +| `-prefix-filter` | string | `main/` | Archive section prefix to filter (e.g., `main/`). | +| `-min-expected-files` | int | `40000` | Minimum expected number of copyright files; fails if fewer are found. | + +--- + +## Docker & Deployment + +### Building the Docker Container + +Build the container image from the `vulnfeeds/` directory: + +```bash +docker build -t gcr.io/oss-vdb/debian-copyright-mirror:latest -f cmd/mirrors/debian-copyright-mirror/Dockerfile . +``` + +### Kubernetes CronJob Deployment + +In production, this mirror runs as a scheduled Kubernetes `CronJob` in GKE (defined in `deployment/clouddeploy/gke-workers/base/feeds/debian-copyright-mirror.yaml` and environment overlays): + +- **Schedule**: Runs daily (`0 5 * * *` Sydney time). +- **Entrypoint**: Runs the `debian-copyright-mirror` Go binary directly, which reads `WORK_DIR` (e.g. `/scratch`) and `GCS_PATH` (e.g. `gs://cve-osv-conversion/debian_copyright/debian_copyright.tar`) from the environment, downloads with `curl`, packages with `tar`, and uploads the archive to Cloud Storage. diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh index 1fae76fdb88..375eb4cdaf3 100755 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh @@ -16,7 +16,9 @@ set -ex # See the License for the specific language governing permissions and # limitations under the License. +cd ../../.. + docker build \ -t gcr.io/oss-vdb/debian-copyright-mirror:latest \ - -f Dockerfile --pull . && \ + -f cmd/mirrors/debian-copyright-mirror/Dockerfile --pull . && \ gcloud docker -- push gcr.io/oss-vdb/debian-copyright-mirror:latest diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py deleted file mode 100644 index c4bdfe2d202..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Download all of the copyright files for packages in main in Debian unstable. - -This: - -Parses https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz - to identify the files to retrieve -Generates a curl configuration to download the URLs -Executes curl to download the URLs in parallel -""" - -import os -import argparse -import yaml -import lzma -import urllib.request -import subprocess -from typing import List, Optional, Dict - - -class Error(Exception): - """General purpose error class.""" - - -def download_url_to_directory(url: str, - directory: str, - filename: Optional[str] = None) -> Optional[str]: - """ - Downloads a URL to a specified directory. - - Args: - url (str): The URL to download. - directory (str): The directory to save the file to. - filename (str, optional): The filename to use. If None, the filename - is extracted from the URL. Defaults to None. - - Returns: - str: The full path to the downloaded file, or None on error. - """ - try: - if not os.path.exists(directory): - os.makedirs(directory) - - if filename is None: - filename = os.path.basename(urllib.parse.urlsplit(url).path) - - filepath = os.path.join(directory, filename) - - urllib.request.urlretrieve(url, filepath) - return filepath - - except urllib.error.URLError as e: - print(f'Error downloading {url}: {e}') - return None - except OSError as e: # Catch directory creation/file writing errors. - print(f'OS Error: {e}') - return None - except Exception as e: # Catch any other unexpected error. - print(f'An unexpected error occurred: {e}') - return None - - -def extract_unstable_copyright(filelist: str) -> Dict: - """ - Extracts the 'unstable_copyright' entry for each package - from an xz-compressed YAML file. - - Args: - filelist (str): The path to the xz-compressed YAML filelist - - Returns: - A dictionary where keys are package names and values are - their 'unstable_copyright' entries, or None if no unstable - copyright is found. - """ - try: - with lzma.open(filelist, 'rt', encoding='utf-8') as f: - data = yaml.safe_load(f) - - results = {} - for package, versions in data.items(): - if 'unstable' in versions: - entries = versions['unstable'] - for entry in entries: - if entry.endswith('unstable_copyright'): - results[package] = entry - break # Found it, no need to continue checking this package. - - return results - - except FileNotFoundError: - print(f"Error: File not found at {filelist}") - return None - except lzma.LZMAError as e: - print(f"Error: LZMA decompression failed: {e}") - return None - except yaml.YAMLError as e: - print(f"Error: YAML parsing failed: {e}") - return None - except Exception as e: - print(f"An unexpected error occurred: {e}") - return None - - -def generate_curl_configuration(filelist: List[str]): - """ - Generates a curl configuration to download all of the files in filelist. - - --output filename - url = https://url - - Args: - filelist (List[str]): a list of files to download. - """ - - url_base = 'https://metadata.ftp-master.debian.org/changelogs' - - with open('/tmp/curl_configuration', 'w') as curl_config: - curl_config.writelines([ - '--output ' + path + '\n' + 'url = ' + os.path.join(url_base, path) + - '\n' for path in filelist - ]) - - -def execute_curl(configuration: str, directory: str): - """ - Execute curl with the supplied configuration in the specified directory. - - Args: - configuration (str): path to configuration file. - directory (str): path to set current working directory to. - """ - - os.makedirs(directory) - subprocess.run( - ['curl', '--parallel', '--create-dirs', '--config', configuration], - cwd=directory, - check=True) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('work_dir') - args = parser.parse_args() - - download_url_to_directory( - 'https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz', - '/tmp') - unstable_package_copyright_files = extract_unstable_copyright( - '/tmp/filelist.yaml.xz') - if unstable_package_copyright_files is None: - raise Error('Unexpected result determining files to download') - generate_curl_configuration( - f for f in unstable_package_copyright_files.values() - if f.startswith('main/')) - with open("/tmp/curl_configuration") as curl_configuration: - if len(curl_configuration.readlines()) < 80000: - raise Error('Unexpectly small curl configuration') - execute_curl('/tmp/curl_configuration', args.work_dir) - - -if __name__ == '__main__': - main() diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh deleted file mode 100755 index a51fdfa08f6..00000000000 --- a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Maintain a GCS bucket mirror of Debian's copyright files -# -# -# -# Inputs: -# * A local work directory -# * GCS bucket name + path to tarball -# - -# Setting BE_VERBOSE to an empty string or null value suppresses silencing of -# commands - -mkdir -p "${WORK_DIR}" || true - -python3 debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/changelogs/" - -tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . - -gcloud storage ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go new file mode 100644 index 00000000000..0e193b4e4ca --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main.go @@ -0,0 +1,276 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main downloads all machine-readable copyright files for packages in Debian unstable. +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + + "cloud.google.com/go/storage" + gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools" + "github.com/google/osv.dev/vulnfeeds/utility/logger" +) + +const ( + DefaultFilelistURL = "https://metadata.ftp-master.debian.org/changelogs/filelist.yaml.xz" + DefaultURLBase = "https://metadata.ftp-master.debian.org/changelogs" + DefaultPrefixFilter = "main/" + DefaultMinExpectedFiles = 40000 +) + +// ExtractUnstableCopyright parses Debian changelogs filelist YAML content from a reader +// and extracts the unstable_copyright path for each package matching the specified prefixFilter. +func ExtractUnstableCopyright(r io.Reader, prefixFilter string) ([]string, error) { + scanner := bufio.NewScanner(r) + const maxLineLen = 1024 * 1024 + buf := make([]byte, 64*1024) + scanner.Buffer(buf, maxLineLen) + + var results []string + inUnstable := false + currentPkgFound := false + + for scanner.Scan() { + line := scanner.Text() + if len(line) > 0 && line[0] != ' ' && line[0] != '\t' && strings.HasSuffix(line, ":") { + inUnstable = false + currentPkgFound = false + + continue + } + + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " -") && strings.HasSuffix(line, ":") { + inUnstable = trimmed == "unstable:" || trimmed == "'unstable':" || trimmed == `"unstable":` + + continue + } + + if inUnstable && !currentPkgFound && strings.HasPrefix(trimmed, "- ") { + entry := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "- ")), `"'`) + if strings.HasSuffix(entry, "unstable_copyright") && (prefixFilter == "" || strings.HasPrefix(entry, prefixFilter)) { + results = append(results, entry) + currentPkgFound = true + } + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning filelist YAML: %w", err) + } + + return results, nil +} + +func fetchCopyrightFiles(ctx context.Context, filelistURL, prefixFilter string) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, filelistURL, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d fetching %s", resp.StatusCode, filelistURL) + } + + cmd := exec.CommandContext(ctx, "xz", "-dc") + cmd.Stdin = resp.Body + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + files, err := ExtractUnstableCopyright(stdout, prefixFilter) + if waitErr := cmd.Wait(); waitErr != nil { + err = errors.Join(err, fmt.Errorf("xz decompression failed: %w", waitErr)) + } + if err != nil { + return nil, err + } + + return files, nil +} + +// GenerateCurlConfiguration generates a curl config file for parallel downloads. +func GenerateCurlConfiguration(filelist []string, urlBase, configPath string) error { + file, err := os.Create(configPath) + if err != nil { + return fmt.Errorf("failed to create curl config file: %w", err) + } + defer file.Close() + + w := bufio.NewWriter(file) + for _, path := range filelist { + fullURL := fmt.Sprintf("%s/%s", strings.TrimRight(urlBase, "/"), strings.TrimLeft(path, "/")) + if _, err := fmt.Fprintf(w, "--output %s\nurl = %s\n", path, fullURL); err != nil { + return fmt.Errorf("failed to write curl config entry: %w", err) + } + } + + return w.Flush() +} + +// ExecuteCurl runs curl with the specified configuration file in the working directory. +func ExecuteCurl(ctx context.Context, configPath, workDir string) error { + if err := os.MkdirAll(workDir, 0755); err != nil { + return fmt.Errorf("failed to create work directory: %w", err) + } + + cmd := exec.CommandContext(ctx, "curl", "--parallel", "--create-dirs", "--config", configPath) + cmd.Dir = workDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("Executing curl in parallel", slog.String("config", configPath), slog.String("workDir", workDir)) + if err := cmd.Run(); err != nil { + return fmt.Errorf("curl execution failed: %w", err) + } + + return nil +} + +// CreateTarArchive creates a tar archive of workDir using the system tar command. +func CreateTarArchive(ctx context.Context, workDir, tarPath string) error { + // #nosec G204 -- arguments are locally configured paths + cmd := exec.CommandContext(ctx, "tar", "-C", workDir, "--exclude="+filepath.Base(tarPath), "-cf", tarPath, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + logger.Info("Creating tar archive", slog.String("workDir", workDir), slog.String("tarPath", tarPath)) + + return cmd.Run() +} + +func uploadToGCS(ctx context.Context, tarPath, gcsURI string) error { + bucketName, objectName, err := gcs.ParseGCSPath(gcsURI) + if err != nil { + return err + } + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create GCS client: %w", err) + } + defer client.Close() + + logger.Info("Uploading tar archive to GCS", slog.String("bucket", bucketName), slog.String("object", objectName)) + + return gcs.UploadFile(ctx, client.Bucket(bucketName), objectName, tarPath) +} + +func main() { + logger.InitGlobalLogger() + defer logger.Close() + + filelistURL := flag.String("filelist-url", DefaultFilelistURL, "URL of the Debian filelist.yaml.xz file.") + urlBase := flag.String("url-base", DefaultURLBase, "Base URL for downloading Debian changelog/copyright files.") + prefixFilter := flag.String("prefix-filter", DefaultPrefixFilter, "Prefix filter for package paths to download (e.g. 'main/').") + minExpectedFiles := flag.Int("min-expected-files", DefaultMinExpectedFiles, "Minimum expected number of copyright files.") + outDirFlag := flag.String("out-dir", "", "Directory to download copyright files into (defaults to WORK_DIR env var or ./debian_copyright).") + workDirFlag := flag.String("work-dir", "", "Alias for -out-dir.") + tarPathFlag := flag.String("tar-path", "", "Optional local destination path for tarball archive.") + gcsPathFlag := flag.String("gcs-path", "", "Destination GCS path for tarball archive (defaults to GCS_PATH env var).") + + flag.Parse() + + workDir := *outDirFlag + if *workDirFlag != "" { + workDir = *workDirFlag + } else if flag.NArg() > 0 { + workDir = flag.Arg(0) + } else if workDir == "" { + if envWorkDir := os.Getenv("WORK_DIR"); envWorkDir != "" { + workDir = envWorkDir + } else { + workDir = "debian_copyright" + } + } + workDir = filepath.Clean(workDir) + + ctx := context.Background() + + logger.Info("Fetching and extracting copyright filelist", slog.String("url", *filelistURL)) + copyrightFiles, err := fetchCopyrightFiles(ctx, *filelistURL, *prefixFilter) + if err != nil { + logger.Fatal("Failed to obtain copyright file list", slog.Any("err", err)) + } + + logger.Info("Discovered copyright files", slog.Int("count", len(copyrightFiles))) + if len(copyrightFiles) < *minExpectedFiles { + logger.Fatal("Unexpectedly small number of copyright files found", + slog.Int("found", len(copyrightFiles)), + slog.Int("min_expected", *minExpectedFiles), + ) + } + + tempDir, err := os.MkdirTemp("", "debian-copyright-mirror-*") + if err != nil { + logger.Fatal("Failed to create temp dir", slog.Any("err", err)) + } + defer os.RemoveAll(tempDir) + + cfgPath := filepath.Join(tempDir, "curl_configuration") + if err := GenerateCurlConfiguration(copyrightFiles, *urlBase, cfgPath); err != nil { + logger.Fatal("Failed to generate curl configuration", slog.Any("err", err)) + } + + downloadDir := filepath.Join(workDir, "metadata.ftp-master.debian.org", "changelogs") + if err := ExecuteCurl(ctx, cfgPath, downloadDir); err != nil { + logger.Fatal("Curl download failed", slog.Any("err", err)) + } + + gcsDest := *gcsPathFlag + if gcsDest == "" { + gcsDest = os.Getenv("GCS_PATH") + } + + tarDest := *tarPathFlag + if tarDest == "" && gcsDest != "" { + tarDest = filepath.Join(workDir, filepath.Base(gcsDest)) + } + + if tarDest != "" { + if err := CreateTarArchive(ctx, workDir, tarDest); err != nil { + logger.Fatal("Failed to create tar archive", slog.Any("err", err)) + } + if gcsDest != "" { + if err := uploadToGCS(ctx, tarDest, gcsDest); err != nil { + logger.Fatal("Failed to upload tar archive to GCS", slog.Any("err", err)) + } + } + } + + logger.Info("Debian copyright mirror sync completed successfully.", slog.String("workDir", workDir)) +} diff --git a/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go new file mode 100644 index 00000000000..f41156f0b6d --- /dev/null +++ b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/main_test.go @@ -0,0 +1,250 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const sampleFilelistYAML = ` +0ad: + 0.0.17-1: + - main/0/0ad/0ad_0.0.17-1_copyright + - main/0/0ad/0ad_0.0.17-1_changelog + testing: + - main/0/0ad/testing_copyright + - main/0/0ad/testing_changelog + unstable: + - main/0/0ad/unstable_changelog + - main/0/0ad/unstable_copyright + - main/0/0ad/unstable_NEWS +0ad-data: + unstable: + - main/0/0ad-data/unstable_changelog + - main/0/0ad-data/unstable_copyright +non-free-pkg: + unstable: + - non-free/n/non-free-pkg/unstable_changelog + - non-free/n/non-free-pkg/unstable_copyright +no-unstable-pkg: + stable: + - main/n/no-unstable-pkg/stable_copyright +no-copyright-pkg: + unstable: + - main/n/no-copyright-pkg/unstable_changelog +'0xffff': + 0.6.1-1: + - main/0/0xffff/0xffff_0.6.1-1_changelog + unstable: + - "main/0/0xffff/unstable_changelog" + - 'main/0/0xffff/unstable_copyright' +` + +func TestExtractUnstableCopyright(t *testing.T) { + tests := []struct { + name string + yamlContent string + prefixFilter string + expected []string + }{ + { + name: "Sample filelist with main/ filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "main/", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Sample filelist without filter", + yamlContent: sampleFilelistYAML, + prefixFilter: "", + expected: []string{ + "main/0/0ad/unstable_copyright", + "main/0/0ad-data/unstable_copyright", + "non-free/n/non-free-pkg/unstable_copyright", + "main/0/0xffff/unstable_copyright", + }, + }, + { + name: "Empty YAML", + yamlContent: "", + prefixFilter: "main/", + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := strings.NewReader(tt.yamlContent) + got, err := ExtractUnstableCopyright(r, tt.prefixFilter) + if err != nil { + t.Fatalf("ExtractUnstableCopyright unexpected error: %v", err) + } + if !reflect.DeepEqual(got, tt.expected) { + t.Errorf("ExtractUnstableCopyright() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGenerateCurlConfiguration(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "curl_config") + + files := []string{ + "main/0/0ad/unstable_copyright", + "main/0/0xffff/unstable_copyright", + } + urlBase := "https://metadata.ftp-master.debian.org/changelogs" + + if err := GenerateCurlConfiguration(files, urlBase, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration returned error: %v", err) + } + + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read generated config: %v", err) + } + + expected := "--output main/0/0ad/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0ad/unstable_copyright\n" + + "--output main/0/0xffff/unstable_copyright\n" + + "url = https://metadata.ftp-master.debian.org/changelogs/main/0/0xffff/unstable_copyright\n" + + if string(content) != expected { + t.Errorf("GenerateCurlConfiguration() =\n%s\nwant:\n%s", string(content), expected) + } +} + +func TestExecuteCurl(t *testing.T) { + t.Setenv("NO_PROXY", "127.0.0.1,localhost") + fileMap := map[string]string{ + "main/a/pkg1/unstable_copyright": "Copyright pkg1\n", + "main/b/pkg2/unstable_copyright": "Copyright pkg2\n", + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + if content, ok := fileMap[path]; ok { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, content) + + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "curl_config") + workDir := filepath.Join(tempDir, "work") + + filelist := []string{ + "main/a/pkg1/unstable_copyright", + "main/b/pkg2/unstable_copyright", + } + + if err := GenerateCurlConfiguration(filelist, ts.URL, configPath); err != nil { + t.Fatalf("GenerateCurlConfiguration failed: %v", err) + } + + if err := ExecuteCurl(context.Background(), configPath, workDir); err != nil { + t.Fatalf("ExecuteCurl failed: %v", err) + } + + for _, relPath := range filelist { + data, err := os.ReadFile(filepath.Join(workDir, relPath)) + if err != nil { + t.Errorf("Failed to read downloaded file %s: %v", relPath, err) + continue + } + if string(data) != fileMap[relPath] { + t.Errorf("Content mismatch for %s: got %q, want %q", relPath, string(data), fileMap[relPath]) + } + } +} + +func TestCreateTarArchive(t *testing.T) { + tempDir := t.TempDir() + workDir := filepath.Join(tempDir, "work") + subDir := filepath.Join(workDir, "metadata.ftp-master.debian.org", "changelogs", "main", "a", "pkg1") + if err := os.MkdirAll(subDir, 0755); err != nil { + t.Fatalf("Failed to create test directory: %v", err) + } + filePath := filepath.Join(subDir, "unstable_copyright") + if err := os.WriteFile(filePath, []byte("test copyright content"), 0600); err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + + tarPath := filepath.Join(workDir, "test_archive.tar") + if err := CreateTarArchive(context.Background(), workDir, tarPath); err != nil { + t.Fatalf("CreateTarArchive failed: %v", err) + } + + fi, err := os.Stat(tarPath) + if err != nil { + t.Fatalf("Expected tar file to exist: %v", err) + } + if fi.Size() == 0 { + t.Errorf("Expected non-empty tar archive") + } + + f, err := os.Open(tarPath) + if err != nil { + t.Fatalf("Failed to open tar archive: %v", err) + } + defer f.Close() + + tr := tar.NewReader(f) + var entries []string + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("Error reading tar archive: %v", err) + } + entries = append(entries, hdr.Name) + } + + wantEntry := "./metadata.ftp-master.debian.org/changelogs/main/a/pkg1/unstable_copyright" + found := false + for _, entry := range entries { + if entry == wantEntry { + found = true + } + if strings.HasSuffix(entry, "test_archive.tar") { + t.Errorf("Tar archive should not contain itself, found entry: %s", entry) + } + } + if !found { + t.Errorf("Expected tar entry %q not found in archive; entries = %v", wantEntry, entries) + } +} diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go index 8fc78d7b4ff..7ccdd37a3a7 100644 --- a/vulnfeeds/gcs-tools/gcs.go +++ b/vulnfeeds/gcs-tools/gcs.go @@ -145,6 +145,20 @@ func UploadToGCS(ctx context.Context, bkt *storage.BucketHandle, objectName stri return nil } +// ParseGCSPath parses a gs:// URI into bucket and object name components. +func ParseGCSPath(gcsURI string) (bucket, object string, err error) { + if !strings.HasPrefix(gcsURI, "gs://") { + return "", "", fmt.Errorf("invalid GCS URI: %s (must start with gs://)", gcsURI) + } + trimmed := strings.TrimPrefix(gcsURI, "gs://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid GCS URI format %q (expected gs://bucket/object)", gcsURI) + } + + return parts[0], parts[1], nil +} + // UploadFile uploads a local file to a GCS bucket. func UploadFile(ctx context.Context, bkt *storage.BucketHandle, objectName string, filePath string) error { f, err := os.Open(filePath) diff --git a/vulnfeeds/gcs-tools/gcs_test.go b/vulnfeeds/gcs-tools/gcs_test.go index 524f12bb857..26b2b621780 100644 --- a/vulnfeeds/gcs-tools/gcs_test.go +++ b/vulnfeeds/gcs-tools/gcs_test.go @@ -275,3 +275,50 @@ func TestListObjectsFast(t *testing.T) { t.Errorf("ListObjectsFast returned unexpected list.\ngot: %v\nwant: %v", got, expected) } } + +func TestParseGCSPath(t *testing.T) { + tests := []struct { + input string + wantBucket string + wantObject string + expectError bool + }{ + { + input: "gs://my-bucket/path/to/file.tar", + wantBucket: "my-bucket", + wantObject: "path/to/file.tar", + expectError: false, + }, + { + input: "gs://cve-osv-conversion/debian_copyright.tar", + wantBucket: "cve-osv-conversion", + wantObject: "debian_copyright.tar", + expectError: false, + }, + { + input: "/local/path/file.tar", + expectError: true, + }, + { + input: "gs://bucket-only", + expectError: true, + }, + { + input: "gs://", + expectError: true, + }, + } + + for _, tt := range tests { + bucket, obj, err := ParseGCSPath(tt.input) + if (err != nil) != tt.expectError { + t.Errorf("ParseGCSPath(%q) error = %v, expectError = %v", tt.input, err, tt.expectError) + continue + } + if !tt.expectError { + if bucket != tt.wantBucket || obj != tt.wantObject { + t.Errorf("ParseGCSPath(%q) = (%q, %q), want (%q, %q)", tt.input, bucket, obj, tt.wantBucket, tt.wantObject) + } + } + } +} diff --git a/website/blog/content/posts/supporting-debian-security-tracker-data/index.md b/website/blog/content/posts/supporting-debian-security-tracker-data/index.md index 1fd4cb480f9..653dd9b4459 100644 --- a/website/blog/content/posts/supporting-debian-security-tracker-data/index.md +++ b/website/blog/content/posts/supporting-debian-security-tracker-data/index.md @@ -15,7 +15,7 @@ A [sample CVE entry](https://osv.dev/vulnerability/CVE-2024-2466) with Debian in The [Debian Security Tracker](https://security-tracker.debian.org/tracker/) is a database maintained by Debian's security team to track Debian related security vulnerabilities. It aggregates information from various sources, including [Debian Security Advisories](https://www.debian.org/security/#DSAS) (DSAs), the [Common Vulnerabilities and Exposures](https://www.cve.org/) (CVE) database, the [National Vulnerability Database](https://nvd.nist.gov/) (NVD), and the [Debian bug tracking system](https://www.debian.org/Bugs/) (BTS). This makes the security tracker comprehensive, providing a complete record of reported Debian security vulnerabilities. ## Why do we need it? -Previously, OSV.dev [relied](https://google.github.io/osv.dev/data/#converted-data) solely on Debian Security Advisories (DSAs) and Debian Long Term Support Advisories (DLAs) for Debian related vulnerability information. While these are official sources, they only cover issues that have been formally [acknowledged and patched](https://www.debian.org/doc/manuals/securing-debian-manual/dsa.en.html) by Debian. This left a blind spot in our data coverage – unfixed vulnerabilities, where the security issue has been reported but a fix has not been made available yet in Debian. +Previously, OSV.dev [relied](https://google.github.io/osv.dev/data/#converted-data) solely on Debian Security Advisories (DSAs) and Debian Long Term Support Advisories (DLAs) for Debian related vulnerability information. While these are official sources, they only cover issues that have been formally [acknowledged and patched](https://www.debian.org/security/faq#handling) by Debian. This left a blind spot in our data coverage – unfixed vulnerabilities, where the security issue has been reported but a fix has not been made available yet in Debian. Debian Security Tracker fills this gap by providing all reported issues even if the vulnerability has not been fixed yet.