diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 7adcc171682..47f1c67ac3a 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -453,6 +453,21 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/nvd-cve-osv'] waitFor: ['build-nvd-cve-osv', 'cloud-build-queue'] +# Build/push repository-specific GHSA conversion image +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/oss-vdb/repository-ghsa-convert:latest || exit 0'] + id: 'pull-repository-ghsa-convert' + waitFor: ['setup'] +- name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'gcr.io/oss-vdb/repository-ghsa-convert:latest', '-t', 'gcr.io/oss-vdb/repository-ghsa-convert:$COMMIT_SHA', '-f', 'cmd/converters/repository-ghsa/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/repository-ghsa-convert:latest', '--pull', '.'] + dir: 'vulnfeeds' + id: 'build-repository-ghsa-convert' + waitFor: ['pull-repository-ghsa-convert'] +- name: gcr.io/cloud-builders/docker + args: ['push', '--all-tags', 'gcr.io/oss-vdb/repository-ghsa-convert'] + waitFor: ['build-repository-ghsa-convert', 'cloud-build-queue'] + # Build website frontend assets - name: 'node:24.18' entrypoint: 'bash' @@ -543,6 +558,7 @@ steps: generatesitemap=gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA,\ gitter=gcr.io/oss-vdb/gitter:$COMMIT_SHA,\ vanir-signatures=gcr.io/oss-vdb/vanir-signatures:$COMMIT_SHA,\ + repository-ghsa-convert=gcr.io/oss-vdb/repository-ghsa-convert:$COMMIT_SHA,\ cron=gcr.io/oss-vdb/cron:$COMMIT_SHA" ] dir: deployment/clouddeploy/gke-workers @@ -612,3 +628,4 @@ images: - 'gcr.io/oss-vdb/generatesitemap:$COMMIT_SHA' - 'gcr.io/oss-vdb/gitter:$COMMIT_SHA' - 'gcr.io/oss-vdb/vanir-signatures:$COMMIT_SHA' +- 'gcr.io/oss-vdb/repository-ghsa-convert:$COMMIT_SHA' diff --git a/deployment/clouddeploy/gke-workers/base/feeds/kustomization.yaml b/deployment/clouddeploy/gke-workers/base/feeds/kustomization.yaml index 0b555254a67..a4c8b31cd97 100644 --- a/deployment/clouddeploy/gke-workers/base/feeds/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/base/feeds/kustomization.yaml @@ -9,3 +9,4 @@ resources: - debian-first-version.yaml - nvd-cve-osv.yaml - nvd-mirror.yaml +- repository-ghsa-convert.yaml diff --git a/deployment/clouddeploy/gke-workers/base/feeds/repository-ghsa-convert.yaml b/deployment/clouddeploy/gke-workers/base/feeds/repository-ghsa-convert.yaml new file mode 100644 index 00000000000..d8a1b52566a --- /dev/null +++ b/deployment/clouddeploy/gke-workers/base/feeds/repository-ghsa-convert.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: repository-ghsa-convert + labels: + cronLastSuccessfulTimeMins: "420" +spec: + timeZone: Australia/Sydney + schedule: "0 */6 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + activeDeadlineSeconds: 7200 + template: + spec: + containers: + - name: repository-ghsa-convert + image: repository-ghsa-convert + imagePullPolicy: Always + resources: + requests: + cpu: "1" + memory: "2G" + limits: + cpu: "2" + memory: "4G" + env: + - name: GITTER_HOST + value: http://gitter-service:8888 + restartPolicy: Never diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml index 7ec61ca37bb..e4f2fbc17ae 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml @@ -29,3 +29,4 @@ patches: - path: custommetrics.yaml - path: gitter.yaml - path: vanir-signatures.yaml +- path: repository-ghsa-convert.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/repository-ghsa-convert.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/repository-ghsa-convert.yaml new file mode 100644 index 00000000000..f09d706d8c0 --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/repository-ghsa-convert.yaml @@ -0,0 +1,20 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: repository-ghsa-convert +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: repository-ghsa-convert + env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test + - name: OUTPUT_BUCKET + value: osv-test-ghsa-repo-conversion + - name: REPOS_GCS_PATH + value: gs://oss-vdb-test-repos/monitored_repositories.json + - name: NUM_WORKERS + value: "8" diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml index d4a87f2fd2b..f201fd0ecf9 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml @@ -26,4 +26,5 @@ patches: - path: custommetrics.yaml - path: gitter.yaml - path: vanir-signatures.yaml +- path: repository-ghsa-convert.yaml diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/repository-ghsa-convert.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/repository-ghsa-convert.yaml new file mode 100644 index 00000000000..4d5e7021c7f --- /dev/null +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb/repository-ghsa-convert.yaml @@ -0,0 +1,20 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: repository-ghsa-convert +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: repository-ghsa-convert + env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb + - name: OUTPUT_BUCKET + value: osv-ghsa-repo-conversion + - name: REPOS_GCS_PATH + value: gs://oss-vdb-repos/monitored_repositories.json + - name: NUM_WORKERS + value: "16" diff --git a/vulnfeeds/cmd/converters/repository-ghsa/Dockerfile b/vulnfeeds/cmd/converters/repository-ghsa/Dockerfile new file mode 100644 index 00000000000..a131baf4fa8 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/Dockerfile @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS go_build + +RUN mkdir /src +WORKDIR /src + +COPY ./go.mod /src/go.mod +COPY ./go.sum /src/go.sum +RUN go mod download && go mod verify + +COPY ./ /src/ +RUN CGO_ENABLED=0 go build -o repository-ghsa ./cmd/converters/repository-ghsa/ + +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:be40864452bd6d7be21632a1dc18adf03d423de0bf2595f4a287c22400c484bb +RUN apk --no-cache add ca-certificates git + +WORKDIR /root/ +COPY --from=go_build /src/repository-ghsa ./ +COPY ./cmd/converters/repository-ghsa/run_repository_ghsa.sh ./ +RUN chmod +x /root/run_repository_ghsa.sh + +ENTRYPOINT ["/root/run_repository_ghsa.sh"] diff --git a/vulnfeeds/cmd/converters/repository-ghsa/convert.go b/vulnfeeds/cmd/converters/repository-ghsa/convert.go new file mode 100644 index 00000000000..28076e6278e --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/convert.go @@ -0,0 +1,476 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "log/slog" + "net/url" + "regexp" + "slices" + "strings" + "time" + + "github.com/google/osv.dev/vulnfeeds/conversion" + "github.com/google/osv.dev/vulnfeeds/git" + "github.com/google/osv.dev/vulnfeeds/models" + "github.com/google/osv.dev/vulnfeeds/utility" + "github.com/google/osv.dev/vulnfeeds/utility/logger" + "github.com/google/osv.dev/vulnfeeds/vulns" + "github.com/ossf/osv-schema/bindings/go/osvconstants" + "github.com/ossf/osv-schema/bindings/go/osvschema" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// ConvertAdvisoryToOSV converts a repository-level GHSA advisory into an OSV Vulnerability record +// with GIT range types resolved against the repository's git tags. +func ConvertAdvisoryToOSV(advisory GHSAAdvisory, repoTarget RepoTarget, normalizedTags map[string]git.NormalizedTag) (*vulns.Vulnerability, error) { + if advisory.GHSAID == "" { + return nil, errors.New("advisory GHSA ID is empty") + } + if repoTarget.CanonicalURL == "" && repoTarget.Owner != "" && repoTarget.Repo != "" { + repoTarget.CanonicalURL = fmt.Sprintf("https://github.com/%s/%s", repoTarget.Owner, repoTarget.Repo) + } + + pubTime := time.Unix(0, 0).UTC() + if advisory.PublishedAt != nil { + pubTime = *advisory.PublishedAt + } else if advisory.CreatedAt != nil { + pubTime = *advisory.CreatedAt + } + + modTime := pubTime + if advisory.UpdatedAt != nil { + modTime = *advisory.UpdatedAt + } + + var withdrawnTime *timestamppb.Timestamp + if advisory.WithdrawnAt != nil { + withdrawnTime = timestamppb.New(*advisory.WithdrawnAt) + } else if advisory.State == "withdrawn" { + withdrawnTime = timestamppb.New(modTime) + } + + // Build aliases and related lists + var aliases []string + if advisory.CVEID != nil && *advisory.CVEID != "" { + aliases = append(aliases, *advisory.CVEID) + } + for _, ident := range advisory.Identifiers { + if ident.Type == "CVE" && ident.Value != "" { + aliases = append(aliases, ident.Value) + } + } + aliases = vulns.Unique(aliases) + + // Build severity list + var severities []*osvschema.Severity + if advisory.CVSSSeverities != nil { + if advisory.CVSSSeverities.CVSSV3 != nil && advisory.CVSSSeverities.CVSSV3.VectorString != nil && *advisory.CVSSSeverities.CVSSV3.VectorString != "" { + severities = append(severities, &osvschema.Severity{ + Type: osvschema.Severity_CVSS_V3, + Score: *advisory.CVSSSeverities.CVSSV3.VectorString, + }) + } + if advisory.CVSSSeverities.CVSSV4 != nil && advisory.CVSSSeverities.CVSSV4.VectorString != nil && *advisory.CVSSSeverities.CVSSV4.VectorString != "" { + severities = append(severities, &osvschema.Severity{ + Type: osvschema.Severity_CVSS_V4, + Score: *advisory.CVSSSeverities.CVSSV4.VectorString, + }) + } + } + + // Collect and classify references + rawRefs := []models.Reference{ + {URL: advisory.HTMLURL, Tags: []string{"advisory"}}, + {URL: repoTarget.CanonicalURL, Tags: []string{"package"}}, + } + if advisory.CVEID != nil && *advisory.CVEID != "" { + rawRefs = append(rawRefs, models.Reference{ + URL: "https://nvd.nist.gov/vuln/detail/" + *advisory.CVEID, + Tags: []string{"advisory"}, + }) + } + if advisory.Description != nil { + for _, u := range extractURLs(*advisory.Description) { + rawRefs = append(rawRefs, models.Reference{URL: u}) + } + } + references := vulns.ClassifyReferences(rawRefs) + + // Process affected items and resolve version ranges to Git commits + affectedList, unresolvedRanges := buildAffectedList(advisory, repoTarget, normalizedTags) + + // Build database_specific map + dbSpecificMap := map[string]any{ + "github_reviewed": false, + } + cweIDs := collectCWEs(advisory) + if len(cweIDs) > 0 { + dbSpecificMap["cwe_ids"] = cweIDs + } + if advisory.Severity != nil && *advisory.Severity != "" { + dbSpecificMap["severity"] = strings.ToUpper(*advisory.Severity) + } + if advisory.URL != "" { + dbSpecificMap["url"] = advisory.URL + } + if len(unresolvedRanges) > 0 { + dbSpecificMap["unresolved_ranges"] = unresolvedRanges + } + dbSpecificStruct, err := utility.NewStructpbFromMap(dbSpecificMap) + if err != nil { + logger.Warn("Failed to construct database_specific structpb", slog.String("id", advisory.GHSAID), slog.Any("error", err)) + } + + v := &vulns.Vulnerability{ + Vulnerability: &osvschema.Vulnerability{ + SchemaVersion: osvconstants.SchemaVersion, + Id: advisory.GHSAID, + Summary: advisory.Summary, + Details: derefString(advisory.Description), + Aliases: aliases, + Published: timestamppb.New(pubTime), + Modified: timestamppb.New(modTime), + Withdrawn: withdrawnTime, + Severity: severities, + Affected: affectedList, + References: references, + DatabaseSpecific: dbSpecificStruct, + }, + } + + return v, nil +} + +// toExtractedEvents converts an AffectedVersion into a slice of raw version osvschema.Event objects, +// mirroring the behavior of vulnfeeds CVE conversion. +func toExtractedEvents(av models.AffectedVersion) []*osvschema.Event { + var events []*osvschema.Event + intro := av.Introduced + if intro == "" { + intro = "0" + } + events = append(events, &osvschema.Event{Introduced: intro}) + if av.Fixed != "" { + events = append(events, &osvschema.Event{Fixed: av.Fixed}) + } else if av.LastAffected != "" { + events = append(events, &osvschema.Event{LastAffected: av.LastAffected}) + } + + return events +} + +// buildAffectedList processes each vulnerability item in the advisory, resolving version ranges to Git commits. +// Package and ecosystem fields are omitted as repository-specific advisories represent Git repository records. +// Returns the affected list and any unresolved range records for top-level database_specific. +func buildAffectedList(advisory GHSAAdvisory, repoTarget RepoTarget, normalizedTags map[string]git.NormalizedTag) ([]*osvschema.Affected, []map[string]any) { + var gitRanges []*osvschema.Range + var unresolvedRanges []map[string]any + var allParsedRanges []models.AffectedVersion + + for _, vuln := range advisory.Vulnerabilities { + vRangeStr := derefString(vuln.VulnerableVersionRange) + patchedVersionsStr := derefString(vuln.PatchedVersions) + + parsedRanges := ParseAdvisoryVersionRanges(vRangeStr, patchedVersionsStr) + allParsedRanges = append(allParsedRanges, parsedRanges...) + + for _, pr := range parsedRanges { + gitRange := resolveRangeToGit(pr, repoTarget.CanonicalURL, normalizedTags, advisory.GHSAID) + if gitRange != nil { + gitRanges = append(gitRanges, gitRange) + } else { + unresolvedRanges = append(unresolvedRanges, map[string]any{ + "extracted_events": toExtractedEvents(pr), + "source": string(models.VersionSourceAffected), + }) + } + } + } + + // If no git ranges could be resolved, fall back to default Range_GIT introduced at dawn of time ("0") + if len(gitRanges) == 0 { + fallbackRange := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoTarget.CanonicalURL, + Events: []*osvschema.Event{ + {Introduced: "0"}, + }, + } + fallbackEvents := make([]*osvschema.Event, 0, len(allParsedRanges)*2) + for _, pr := range allParsedRanges { + fallbackEvents = append(fallbackEvents, toExtractedEvents(pr)...) + } + if len(fallbackEvents) > 0 { + dbSpecificMap := map[string]any{ + "extracted_events": fallbackEvents, + "source": string(models.VersionSourceAffected), + } + if dbSpecific, err := utility.NewStructpbFromMap(dbSpecificMap); err == nil { + fallbackRange.DatabaseSpecific = dbSpecific + } + } + gitRanges = append(gitRanges, fallbackRange) + } + + affectedList := []*osvschema.Affected{ + { + Ranges: gitRanges, + }, + } + + // Group and deduplicate ranges (merging database_specific.extracted_events) + conversion.GroupAffectedRanges(affectedList) + + return affectedList, unresolvedRanges +} + +// resolveRangeToGit resolves introduced and fixed/last_affected version strings to commit hashes using normalizedTags. +func resolveRangeToGit(av models.AffectedVersion, repoURL string, normalizedTags map[string]git.NormalizedTag, ghsaID string) *osvschema.Range { + var ( + introCommit string + fixedCommit string + lastAffCommit string + err error + ) + + // Resolve introduced + if av.Introduced == "" || av.Introduced == "0" { + introCommit = "0" + } else { + introCommit, err = git.VersionToCommit(av.Introduced, normalizedTags) + if err != nil { + logger.Debug("Could not resolve introduced version to commit", + slog.String("id", ghsaID), slog.String("version", av.Introduced), slog.Any("error", err)) + // Fall back to dawn of time so the vulnerability range still functions if fixed commit is found + introCommit = "0" + } + } + + // Resolve fixed + if av.Fixed != "" { + fixedCommit, err = git.VersionToCommit(av.Fixed, normalizedTags) + if err != nil { + logger.Debug("Could not resolve fixed version to commit", + slog.String("id", ghsaID), slog.String("version", av.Fixed), slog.Any("error", err)) + } + } + + // Resolve last_affected + if av.LastAffected != "" && fixedCommit == "" { + lastAffCommit, err = git.VersionToCommit(av.LastAffected, normalizedTags) + if err != nil { + logger.Debug("Could not resolve last_affected version to commit", + slog.String("id", ghsaID), slog.String("version", av.LastAffected), slog.Any("error", err)) + } + } + + // Only return a GIT range if at least one actual commit boundary was resolved. + if introCommit == "0" && fixedCommit == "" && lastAffCommit == "" { + return nil + } + + gitRange := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoURL, + Events: []*osvschema.Event{ + {Introduced: introCommit}, + }, + } + + if fixedCommit != "" { + gitRange.Events = append(gitRange.Events, &osvschema.Event{Fixed: fixedCommit}) + } else if lastAffCommit != "" { + gitRange.Events = append(gitRange.Events, &osvschema.Event{LastAffected: lastAffCommit}) + } + + extractedEvents := toExtractedEvents(av) + if len(extractedEvents) > 0 { + dbSpecificMap := map[string]any{ + "extracted_events": extractedEvents, + "source": string(models.VersionSourceAffected), + } + if dbSpecific, err := utility.NewStructpbFromMap(dbSpecificMap); err == nil { + gitRange.DatabaseSpecific = dbSpecific + } else { + logger.Warn("Failed to create database_specific for git range", slog.String("id", ghsaID), slog.Any("error", err)) + } + } + + return gitRange +} + +// ParseAdvisoryVersionRanges parses version ranges from GHSA fields into models.AffectedVersion slices. +// Handles standard ranges (>= 1.0, < 2.0), single bounds (< 2.0, <= 2.0), exact versions (= 1.0, 1.0), +// compound ranges (< 1.2, >= 2.0, < 2.5), and integrates patched_versions. +func ParseAdvisoryVersionRanges(vRange string, patchedVersions string) []models.AffectedVersion { + vRange = strings.TrimSpace(vRange) + patchedVersions = strings.TrimSpace(patchedVersions) + + var results []models.AffectedVersion + + if vRange != "" { + // Split by compound ranges if present (e.g. "< 1.2.0, >= 2.0.0, < 2.1.0") + subRanges := splitCompoundRanges(vRange) + for _, sr := range subRanges { + av, err := parseSingleRange(sr) + if err == nil { + results = append(results, av) + } + } + } + + // If no ranges could be parsed from vRange but patchedVersions is available + if len(results) == 0 && patchedVersions != "" { + for pv := range strings.SplitSeq(patchedVersions, ",") { + pv = strings.TrimSpace(pv) + if pv != "" { + results = append(results, models.AffectedVersion{ + Introduced: "0", + Fixed: pv, + }) + } + } + } else if len(results) > 0 && patchedVersions != "" { + // Supplement fixed versions if missing in results + firstPatched := strings.TrimSpace(strings.Split(patchedVersions, ",")[0]) + if firstPatched != "" { + for i := range results { + if results[i].Fixed == "" && results[i].LastAffected != "" { + results[i].Fixed = firstPatched + } + } + } + } + + return results +} + +func parseSingleRange(r string) (models.AffectedVersion, error) { + r = strings.TrimSpace(r) + + // Try standard parser from git package + av, err := git.ParseVersionRange(r) + if err == nil { + return av, nil + } + + // Exact version: "= 1.2.3" or "=1.2.3" + if rest, ok := strings.CutPrefix(r, "="); ok { + v := strings.TrimSpace(rest) + if v != "" { + return models.AffectedVersion{ + Introduced: v, + LastAffected: v, + }, nil + } + } + + // Bare version without operator: e.g. "1.2.3" or "v1.2.3" + if !strings.ContainsAny(r, "<>=~^") { + return models.AffectedVersion{ + Introduced: r, + LastAffected: r, + }, nil + } + + return models.AffectedVersion{}, fmt.Errorf("unable to parse version range: %s", r) +} + +// splitCompoundRanges splits strings like "< 1.2.0, >= 2.0.0, < 2.1.0" into separate ranges. +func splitCompoundRanges(vRange string) []string { + // If it doesn't contain a comma, it's a single range + if !strings.Contains(vRange, ",") { + return []string{vRange} + } + + // If it's a standard two-part range (e.g. ">= 1.0.0, < 2.0.0"), keep together + parts := strings.Split(vRange, ",") + if len(parts) == 2 { + p1 := strings.TrimSpace(parts[0]) + p2 := strings.TrimSpace(parts[1]) + if (strings.HasPrefix(p1, ">=") || strings.HasPrefix(p1, ">")) && + (strings.HasPrefix(p2, "<=") || strings.HasPrefix(p2, "<")) { + return []string{vRange} + } + } + + // Otherwise, group parts by boundary operators + var ranges []string + var currentRange []string + + for _, p := range parts { + trimmed := strings.TrimSpace(p) + if len(currentRange) == 0 { + currentRange = append(currentRange, trimmed) + continue + } + + // If currentRange already has an introduced bound and trimmed has an upper bound, combine + if (strings.HasPrefix(currentRange[0], ">=") || strings.HasPrefix(currentRange[0], ">")) && + (strings.HasPrefix(trimmed, "<=") || strings.HasPrefix(trimmed, "<")) && len(currentRange) == 1 { + currentRange = append(currentRange, trimmed) + ranges = append(ranges, strings.Join(currentRange, ", ")) + currentRange = nil + } else { + ranges = append(ranges, strings.Join(currentRange, ", ")) + currentRange = []string{trimmed} + } + } + + if len(currentRange) > 0 { + ranges = append(ranges, strings.Join(currentRange, ", ")) + } + + return ranges +} + +func collectCWEs(advisory GHSAAdvisory) []string { + var cwes []string + cwes = append(cwes, advisory.CWEIDs...) + for _, cwe := range advisory.CWEs { + if cwe.CWEID != "" { + cwes = append(cwes, cwe.CWEID) + } + } + slices.Sort(cwes) + + return slices.Compact(cwes) +} + +var urlRegex = regexp.MustCompile(`https?://[^\s)\]>"']+`) + +func extractURLs(text string) []string { + matches := urlRegex.FindAllString(text, -1) + var validURLs []string + for _, m := range matches { + m = strings.TrimRight(m, ".,;:") + if u, err := url.Parse(m); err == nil && u.Scheme != "" && u.Host != "" { + validURLs = append(validURLs, m) + } + } + + return vulns.Unique(validURLs) +} + +func derefString(s *string) string { + if s == nil { + return "" + } + + return *s +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/convert_test.go b/vulnfeeds/cmd/converters/repository-ghsa/convert_test.go new file mode 100644 index 00000000000..89ef96bbfe7 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/convert_test.go @@ -0,0 +1,947 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/osv.dev/vulnfeeds/git" + "github.com/google/osv.dev/vulnfeeds/models" + "github.com/google/osv.dev/vulnfeeds/utility" + "github.com/google/osv.dev/vulnfeeds/vulns" + "github.com/ossf/osv-schema/bindings/go/osvschema" + "google.golang.org/protobuf/testing/protocmp" +) + +func TestSplitCompoundRanges(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want []string + }{ + { + name: "standard single range", + input: ">= 1.0.0, < 2.0.0", + want: []string{">= 1.0.0, < 2.0.0"}, + }, + { + name: "single bound", + input: "< 2.0.0", + want: []string{"< 2.0.0"}, + }, + { + name: "compound multiple ranges", + input: "< 1.2.0, >= 2.0.0, < 2.1.0", + want: []string{"< 1.2.0", ">= 2.0.0, < 2.1.0"}, + }, + { + name: "compound disjoint lower bounds", + input: "< 1.0.0, >= 2.0.0", + want: []string{"< 1.0.0", ">= 2.0.0"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := splitCompoundRanges(tc.input) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("splitCompoundRanges(%q) mismatch (-want +got):\n%s", tc.input, diff) + } + }) + } +} + +func TestParseAdvisoryVersionRanges(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + vRange string + patchedVersions string + want []models.AffectedVersion + }{ + { + name: "standard range", + vRange: ">= 1.0.0, < 2.0.0", + want: []models.AffectedVersion{ + {Introduced: "1.0.0", Fixed: "2.0.0"}, + }, + }, + { + name: "less than", + vRange: "< 1.5.0", + want: []models.AffectedVersion{ + {Introduced: "0", Fixed: "1.5.0"}, + }, + }, + { + name: "less than or equal", + vRange: "<= 1.5.0", + want: []models.AffectedVersion{ + {Introduced: "0", LastAffected: "1.5.0"}, + }, + }, + { + name: "exact version with equal sign", + vRange: "= 2.1.0", + want: []models.AffectedVersion{ + {Introduced: "2.1.0", LastAffected: "2.1.0"}, + }, + }, + { + name: "bare version string", + vRange: "2.1.0", + want: []models.AffectedVersion{ + {Introduced: "2.1.0", LastAffected: "2.1.0"}, + }, + }, + { + name: "fallback to patched_versions when vRange is empty", + vRange: "", + patchedVersions: "1.2.3, 2.0.0", + want: []models.AffectedVersion{ + {Introduced: "0", Fixed: "1.2.3"}, + {Introduced: "0", Fixed: "2.0.0"}, + }, + }, + { + name: "supplement fixed with patched_versions when last_affected present", + vRange: "<= 1.2.2", + patchedVersions: "1.2.3", + want: []models.AffectedVersion{ + {Introduced: "0", Fixed: "1.2.3", LastAffected: "1.2.2"}, + }, + }, + { + name: "compound range", + vRange: "< 1.2.0, >= 2.0.0, < 2.1.0", + want: []models.AffectedVersion{ + {Introduced: "0", Fixed: "1.2.0"}, + {Introduced: "2.0.0", Fixed: "2.1.0"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := ParseAdvisoryVersionRanges(tc.vRange, tc.patchedVersions) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("ParseAdvisoryVersionRanges(%q, %q) mismatch (-want +got):\n%s", tc.vRange, tc.patchedVersions, diff) + } + }) + } +} + +func TestResolveRangeToGit(t *testing.T) { + t.Parallel() + + normalizedTags := map[string]git.NormalizedTag{ + "1-0-0": {Commit: "1111111111111111111111111111111111111111", OriginalTag: "v1.0.0"}, + "2-0-0": {Commit: "2222222222222222222222222222222222222222", OriginalTag: "v2.0.0"}, + } + + repoURL := "https://github.com/owner/repo" + + t.Run("resolved introduced and fixed", func(t *testing.T) { + av := models.AffectedVersion{ + Introduced: "1.0.0", + Fixed: "2.0.0", + } + got := resolveRangeToGit(av, repoURL, normalizedTags, "GHSA-test") + if got == nil { + t.Fatal("expected git range, got nil") + } + + dbSpec, err := utility.NewStructpbFromMap(map[string]any{ + "extracted_events": []*osvschema.Event{ + {Introduced: "1.0.0"}, + {Fixed: "2.0.0"}, + }, + "source": string(models.VersionSourceAffected), + }) + if err != nil { + t.Fatalf("failed constructing expected database_specific: %v", err) + } + + want := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoURL, + Events: []*osvschema.Event{ + {Introduced: "1111111111111111111111111111111111111111"}, + {Fixed: "2222222222222222222222222222222222222222"}, + }, + DatabaseSpecific: dbSpec, + } + + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("resolveRangeToGit mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("resolved dawn of time introduced", func(t *testing.T) { + av := models.AffectedVersion{ + Introduced: "0", + Fixed: "2.0.0", + } + got := resolveRangeToGit(av, repoURL, normalizedTags, "GHSA-test") + if got == nil { + t.Fatal("expected git range, got nil") + } + + dbSpec, err := utility.NewStructpbFromMap(map[string]any{ + "extracted_events": []*osvschema.Event{ + {Introduced: "0"}, + {Fixed: "2.0.0"}, + }, + "source": string(models.VersionSourceAffected), + }) + if err != nil { + t.Fatalf("failed constructing expected database_specific: %v", err) + } + + want := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoURL, + Events: []*osvschema.Event{ + {Introduced: "0"}, + {Fixed: "2222222222222222222222222222222222222222"}, + }, + DatabaseSpecific: dbSpec, + } + + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("resolveRangeToGit mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("resolved last_affected", func(t *testing.T) { + av := models.AffectedVersion{ + Introduced: "0", + LastAffected: "1.0.0", + } + got := resolveRangeToGit(av, repoURL, normalizedTags, "GHSA-test") + if got == nil { + t.Fatal("expected git range, got nil") + } + + dbSpec, err := utility.NewStructpbFromMap(map[string]any{ + "extracted_events": []*osvschema.Event{ + {Introduced: "0"}, + {LastAffected: "1.0.0"}, + }, + "source": string(models.VersionSourceAffected), + }) + if err != nil { + t.Fatalf("failed constructing expected database_specific: %v", err) + } + + want := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoURL, + Events: []*osvschema.Event{ + {Introduced: "0"}, + {LastAffected: "1111111111111111111111111111111111111111"}, + }, + DatabaseSpecific: dbSpec, + } + + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("resolveRangeToGit mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("open vulnerability (no fixed version)", func(t *testing.T) { + av := models.AffectedVersion{ + Introduced: "1.0.0", + } + got := resolveRangeToGit(av, repoURL, normalizedTags, "GHSA-test") + if got == nil { + t.Fatal("expected git range, got nil") + } + + dbSpec, err := utility.NewStructpbFromMap(map[string]any{ + "extracted_events": []*osvschema.Event{ + {Introduced: "1.0.0"}, + }, + "source": string(models.VersionSourceAffected), + }) + if err != nil { + t.Fatalf("failed constructing expected database_specific: %v", err) + } + + want := &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: repoURL, + Events: []*osvschema.Event{ + {Introduced: "1111111111111111111111111111111111111111"}, + }, + DatabaseSpecific: dbSpec, + } + + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("resolveRangeToGit mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("unresolvable version returns nil", func(t *testing.T) { + av := models.AffectedVersion{ + Introduced: "0", + Fixed: "9.9.9", + } + got := resolveRangeToGit(av, repoURL, normalizedTags, "GHSA-test") + if got != nil { + t.Errorf("expected nil for unresolvable version, got %+v", got) + } + }) +} + +func TestConvertAdvisoryToOSV_Full(t *testing.T) { + t.Parallel() + + cveID := "CVE-2026-99999" + desc := "Vulnerability details with link to https://example.com/exploit" + severityLevel := "high" + v3Vector := "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + v3Score := 9.8 + v4Vector := "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N" + v4Score := 9.3 + pubTime := time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC) + modTime := time.Date(2026, 3, 2, 12, 0, 0, 0, time.UTC) + + pkgName := "my-awesome-lib" + vRange := ">= 1.0.0, < 2.0.0" + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-1234-5678-9012", + CVEID: &cveID, + HTMLURL: "https://github.com/owner/repo/security/advisories/GHSA-1234-5678-9012", + URL: "https://api.github.com/repos/owner/repo/security-advisories/GHSA-1234-5678-9012", + Summary: "Test vulnerability summary", + Description: &desc, + Severity: &severityLevel, + State: "published", + PublishedAt: &pubTime, + UpdatedAt: &modTime, + Identifiers: []GHSAIdentifier{ + {Type: "GHSA", Value: "GHSA-1234-5678-9012"}, + {Type: "CVE", Value: "CVE-2026-99999"}, + }, + CVSSSeverities: &GHSACVSSSeverities{ + CVSSV3: &GHSACVSS{ + VectorString: &v3Vector, + Score: &v3Score, + }, + CVSSV4: &GHSACVSS{ + VectorString: &v4Vector, + Score: &v4Score, + }, + }, + CWEs: []GHSACWE{ + {CWEID: "CWE-79", Name: "Cross-site Scripting"}, + }, + CWEIDs: []string{"CWE-89"}, + Vulnerabilities: []GHSAVulnerability{ + { + Package: &GHSAPackage{ + Ecosystem: "npm", + Name: &pkgName, + }, + VulnerableVersionRange: &vRange, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + normalizedTags := map[string]git.NormalizedTag{ + "1-0-0": {Commit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", OriginalTag: "v1.0.0"}, + "2-0-0": {Commit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", OriginalTag: "v2.0.0"}, + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, normalizedTags) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + if vuln.GetId() != "GHSA-1234-5678-9012" { + t.Errorf("expected ID GHSA-1234-5678-9012, got %s", vuln.GetId()) + } + if vuln.GetSummary() != "Test vulnerability summary" { + t.Errorf("expected summary, got %s", vuln.GetSummary()) + } + if vuln.GetDetails() != desc { + t.Errorf("expected details %q, got %q", desc, vuln.GetDetails()) + } + + // Verify aliases + wantAliases := []string{"CVE-2026-99999"} + if diff := cmp.Diff(wantAliases, vuln.GetAliases()); diff != "" { + t.Errorf("Aliases mismatch (-want +got):\n%s", diff) + } + + // Verify severity + if len(vuln.GetSeverity()) != 2 { + t.Fatalf("expected 2 severities, got %d", len(vuln.GetSeverity())) + } + if vuln.GetSeverity()[0].GetType() != osvschema.Severity_CVSS_V3 || vuln.GetSeverity()[0].GetScore() != v3Vector { + t.Errorf("unexpected CVSS v3 severity: %+v", vuln.GetSeverity()[0]) + } + if vuln.GetSeverity()[1].GetType() != osvschema.Severity_CVSS_V4 || vuln.GetSeverity()[1].GetScore() != v4Vector { + t.Errorf("unexpected CVSS v4 severity: %+v", vuln.GetSeverity()[1]) + } + + // Verify affected + if len(vuln.GetAffected()) != 1 { + t.Fatalf("expected 1 affected item, got %d", len(vuln.GetAffected())) + } + aff := vuln.GetAffected()[0] + if aff.GetPackage() != nil { + t.Errorf("expected package to be nil, got: %+v", aff.GetPackage()) + } + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if r.GetType() != osvschema.Range_GIT || r.GetRepo() != "https://github.com/owner/repo" { + t.Errorf("unexpected range type or repo: %+v", r) + } + if len(r.GetEvents()) != 2 { + t.Fatalf("expected 2 events, got %d", len(r.GetEvents())) + } + if r.GetEvents()[0].GetIntroduced() != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { + t.Errorf("unexpected introduced: %s", r.GetEvents()[0].GetIntroduced()) + } + // Verify range database_specific fields + if r.GetDatabaseSpecific() == nil { + t.Fatal("expected range database_specific to be non-nil") + } + rDbSpec := r.GetDatabaseSpecific().AsMap() + if rDbSpec["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected range source %q, got %v", models.VersionSourceAffected, rDbSpec["source"]) + } + eventsList, ok := rDbSpec["extracted_events"].([]any) + if !ok || len(eventsList) != 2 { + t.Fatalf("expected 2 extracted_events in range database_specific, got %v", rDbSpec["extracted_events"]) + } + e0, _ := eventsList[0].(map[string]any) + e1, _ := eventsList[1].(map[string]any) + if e0["introduced"] != "1.0.0" { + t.Errorf("expected extracted introduced 1.0.0, got %v", e0["introduced"]) + } + if e1["fixed"] != "2.0.0" { + t.Errorf("expected extracted fixed 2.0.0, got %v", e1["fixed"]) + } + + // Verify database_specific fields + dbSpec := vuln.GetDatabaseSpecific().AsMap() + if dbSpec["github_reviewed"] != false { + t.Errorf("expected github_reviewed to be false, got %v", dbSpec["github_reviewed"]) + } + if dbSpec["severity"] != "HIGH" { + t.Errorf("expected severity HIGH, got %v", dbSpec["severity"]) + } + cwesList, ok := dbSpec["cwe_ids"].([]any) + if !ok || len(cwesList) != 2 { + t.Errorf("expected 2 cwe_ids, got %v", dbSpec["cwe_ids"]) + } + + // Verify references + refURLs := make(map[string]bool) + for _, ref := range vuln.GetReferences() { + refURLs[ref.GetUrl()] = true + } + if !refURLs["https://github.com/owner/repo/security/advisories/GHSA-1234-5678-9012"] { + t.Errorf("missing advisory reference URL") + } + if !refURLs["https://github.com/owner/repo"] { + t.Errorf("missing repo package reference URL") + } + if !refURLs["https://nvd.nist.gov/vuln/detail/CVE-2026-99999"] { + t.Errorf("missing NVD reference URL") + } + if !refURLs["https://example.com/exploit"] { + t.Errorf("missing extracted description reference URL") + } +} + +func TestConvertAdvisoryToOSV_NoVulnerabilities(t *testing.T) { + t.Parallel() + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-novuln-test", + Summary: "No vuln declared", + State: "published", + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "empty-vuln-repo", + CanonicalURL: "https://github.com/owner/empty-vuln-repo", + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, nil) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + if len(vuln.GetAffected()) != 1 { + t.Fatalf("expected 1 default affected item, got %d", len(vuln.GetAffected())) + } + + aff := vuln.GetAffected()[0] + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if r.GetType() != osvschema.Range_GIT || r.GetRepo() != "https://github.com/owner/empty-vuln-repo" { + t.Errorf("unexpected range type or repo: %+v", r) + } + if len(r.GetEvents()) != 1 || r.GetEvents()[0].GetIntroduced() != "0" { + t.Errorf("unexpected events: %+v", r.GetEvents()) + } +} + +func TestConvertAdvisoryToOSV_Withdrawn(t *testing.T) { + t.Parallel() + + withdrawnTime := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + advisory := GHSAAdvisory{ + GHSAID: "GHSA-withdrawn-test", + Summary: "Withdrawn test", + State: "withdrawn", + WithdrawnAt: &withdrawnTime, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, nil) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + if vuln.GetWithdrawn() == nil { + t.Fatalf("expected withdrawn timestamp, got nil") + } + if !vuln.GetWithdrawn().AsTime().Equal(withdrawnTime) { + t.Errorf("withdrawn timestamp mismatch: got %v, want %v", vuln.GetWithdrawn().AsTime(), withdrawnTime) + } +} + +func TestConvertAdvisoryToOSV_UnresolvedTagsFallback(t *testing.T) { + t.Parallel() + + vRange := "< 2.0.0" + pkgName := "lib-unresolved" + advisory := GHSAAdvisory{ + GHSAID: "GHSA-unresolved-tags", + Summary: "Unresolved tags fallback", + State: "published", + Vulnerabilities: []GHSAVulnerability{ + { + Package: &GHSAPackage{ + Ecosystem: "Go", + Name: &pkgName, + }, + VulnerableVersionRange: &vRange, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + // Empty normalized tags (e.g. tag not matched) + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, map[string]git.NormalizedTag{}) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + if len(vuln.GetAffected()) != 1 { + t.Fatalf("expected 1 affected item, got %d", len(vuln.GetAffected())) + } + aff := vuln.GetAffected()[0] + if aff.GetPackage() != nil { + t.Errorf("expected package to be nil, got: %+v", aff.GetPackage()) + } + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 fallback range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if r.GetType() != osvschema.Range_GIT || r.GetRepo() != "https://github.com/owner/repo" { + t.Errorf("expected Range_GIT fallback with repo, got type %v repo %q", r.GetType(), r.GetRepo()) + } + if len(r.GetEvents()) != 1 || r.GetEvents()[0].GetIntroduced() != "0" { + t.Errorf("unexpected fallback events: %+v", r.GetEvents()) + } + if r.GetDatabaseSpecific() == nil { + t.Fatal("expected fallback range database_specific to be non-nil") + } + fallbackDbSpec := r.GetDatabaseSpecific().AsMap() + if fallbackDbSpec["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected source %q, got %v", models.VersionSourceAffected, fallbackDbSpec["source"]) + } + fallbackEvents, ok := fallbackDbSpec["extracted_events"].([]any) + if !ok || len(fallbackEvents) != 2 { + t.Fatalf("expected 2 fallback extracted_events, got %v", fallbackDbSpec["extracted_events"]) + } + + rootDbSpec := vuln.GetDatabaseSpecific().AsMap() + unresolved, ok := rootDbSpec["unresolved_ranges"].([]any) + if !ok || len(unresolved) != 1 { + t.Fatalf("expected 1 unresolved_ranges entry in root database_specific, got %v", rootDbSpec["unresolved_ranges"]) + } + ur0, _ := unresolved[0].(map[string]any) + if ur0["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected unresolved_ranges source %q, got %v", models.VersionSourceAffected, ur0["source"]) + } +} + +func TestWriteOSVRecord(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + vuln := &vulns.Vulnerability{ + Vulnerability: &osvschema.Vulnerability{ + Id: "GHSA-write-test", + Summary: "Write test", + }, + } + + err := writeOSVRecord(vuln, tmpDir, nil, "") + if err != nil { + t.Fatalf("writeOSVRecord failed: %v", err) + } + + outPath := filepath.Join(tmpDir, "GHSA-write-test.json") + data, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("expected output file to exist: %v", err) + } + if !strings.Contains(string(data), "GHSA-write-test") { + t.Errorf("output file content does not contain vulnerability ID: %s", string(data)) + } + + // Test empty vuln ID returns error + emptyVuln := &vulns.Vulnerability{ + Vulnerability: &osvschema.Vulnerability{}, + } + err = writeOSVRecord(emptyVuln, tmpDir, nil, "") + if err == nil { + t.Errorf("expected error for empty vuln ID, got nil") + } +} + +func TestConvertAdvisoryToOSV_EmptyID(t *testing.T) { + t.Parallel() + + advisory := GHSAAdvisory{ + Summary: "Missing ID", + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + } + + _, err := ConvertAdvisoryToOSV(advisory, repoTarget, nil) + if err == nil { + t.Errorf("expected error for empty GHSA ID, got nil") + } +} + +func TestConvertAdvisoryToOSV_MultipleVulnerabilities(t *testing.T) { + t.Parallel() + + vRange1 := "< 1.0.0" + vRange2 := ">= 2.0.0, < 2.5.0" + pkg1 := "pkg-one" + pkg2 := "pkg-two" + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-multi-vuln", + Summary: "Multiple vulnerabilities in advisory", + State: "published", + Vulnerabilities: []GHSAVulnerability{ + { + Package: &GHSAPackage{ + Ecosystem: "npm", + Name: &pkg1, + }, + VulnerableVersionRange: &vRange1, + }, + { + Package: &GHSAPackage{ + Ecosystem: "Go", + Name: &pkg2, + }, + VulnerableVersionRange: &vRange2, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + normalizedTags := map[string]git.NormalizedTag{ + "1-0-0": {Commit: "1111111111111111111111111111111111111111", OriginalTag: "v1.0.0"}, + "2-0-0": {Commit: "2222222222222222222222222222222222222222", OriginalTag: "v2.0.0"}, + "2-5-0": {Commit: "2525252525252525252525252525252525252525", OriginalTag: "v2.5.0"}, + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, normalizedTags) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + // Must have exactly 1 Affected entry with nil Package + if len(vuln.GetAffected()) != 1 { + t.Fatalf("expected 1 aggregated affected entry, got %d", len(vuln.GetAffected())) + } + aff := vuln.GetAffected()[0] + if aff.GetPackage() != nil { + t.Errorf("expected package to be nil, got: %+v", aff.GetPackage()) + } + + // Must have 2 GIT ranges + if len(aff.GetRanges()) != 2 { + t.Fatalf("expected 2 GIT ranges, got %d", len(aff.GetRanges())) + } + for i, r := range aff.GetRanges() { + if r.GetType() != osvschema.Range_GIT || r.GetRepo() != "https://github.com/owner/repo" { + t.Errorf("range[%d] unexpected type %v or repo %q", i, r.GetType(), r.GetRepo()) + } + if r.GetDatabaseSpecific() == nil { + t.Errorf("range[%d] expected database_specific to be non-nil", i) + } else { + dbSpec := r.GetDatabaseSpecific().AsMap() + if dbSpec["source"] != string(models.VersionSourceAffected) { + t.Errorf("range[%d] expected source %q, got %v", i, models.VersionSourceAffected, dbSpec["source"]) + } + if _, ok := dbSpec["extracted_events"].([]any); !ok { + t.Errorf("range[%d] missing extracted_events in database_specific", i) + } + } + } +} + +func TestConvertAdvisoryToOSV_GroupingExtractedEvents(t *testing.T) { + t.Parallel() + + vRange1 := "< 1.5.0" + vRange2 := "< 2.0.0" + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-grouping-test", + Summary: "Grouping test with common introduced commit", + State: "published", + Vulnerabilities: []GHSAVulnerability{ + { + VulnerableVersionRange: &vRange1, + }, + { + VulnerableVersionRange: &vRange2, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + normalizedTags := map[string]git.NormalizedTag{ + "1-5-0": {Commit: "1515151515151515151515151515151515151515", OriginalTag: "v1.5.0"}, + "2-0-0": {Commit: "2020202020202020202020202020202020202020", OriginalTag: "v2.0.0"}, + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, normalizedTags) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + if len(vuln.GetAffected()) != 1 { + t.Fatalf("expected 1 affected item, got %d", len(vuln.GetAffected())) + } + aff := vuln.GetAffected()[0] + + // Since both ranges have introduced "0", GroupAffectedRanges groups them into 1 range with multiple fixed events + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 grouped range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if len(r.GetEvents()) != 3 { + t.Fatalf("expected 3 events (1 introduced + 2 fixed), got %d", len(r.GetEvents())) + } + + // Verify merged database_specific + if r.GetDatabaseSpecific() == nil { + t.Fatal("expected database_specific to be present on grouped range") + } + dbSpec := r.GetDatabaseSpecific().AsMap() + if dbSpec["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected source %q, got %v", models.VersionSourceAffected, dbSpec["source"]) + } + extracted, ok := dbSpec["extracted_events"].([]any) + if !ok { + t.Fatalf("expected extracted_events in grouped range database_specific, got %v", dbSpec["extracted_events"]) + } + // Merged extracted_events should contain introduced "0", fixed "1.5.0", and fixed "2.0.0" (deduplicated) + if len(extracted) != 3 { + t.Errorf("expected 3 merged extracted_events, got %d: %v", len(extracted), extracted) + } +} + +func TestConvertAdvisoryToOSV_PartialResolution(t *testing.T) { + t.Parallel() + + vRange1 := ">= 1.0.0, < 2.0.0" + vRange2 := "< 99.0.0" // non-existent tag + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-partial-resolution", + Summary: "Partial resolution test", + State: "published", + Vulnerabilities: []GHSAVulnerability{ + { + VulnerableVersionRange: &vRange1, + }, + { + VulnerableVersionRange: &vRange2, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + normalizedTags := map[string]git.NormalizedTag{ + "1-0-0": {Commit: "1111111111111111111111111111111111111111", OriginalTag: "v1.0.0"}, + "2-0-0": {Commit: "2222222222222222222222222222222222222222", OriginalTag: "v2.0.0"}, + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, normalizedTags) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + aff := vuln.GetAffected()[0] + // Range 1 resolved successfully + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 resolved range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if len(r.GetEvents()) != 2 { + t.Errorf("expected 2 events in resolved range, got %d", len(r.GetEvents())) + } + + // Range 2 could not be resolved, so it must be present in top-level unresolved_ranges + rootDbSpec := vuln.GetDatabaseSpecific().AsMap() + unresolved, ok := rootDbSpec["unresolved_ranges"].([]any) + if !ok || len(unresolved) != 1 { + t.Fatalf("expected 1 unresolved range in root database_specific, got %v", rootDbSpec["unresolved_ranges"]) + } + ur0, _ := unresolved[0].(map[string]any) + if ur0["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected source %q, got %v", models.VersionSourceAffected, ur0["source"]) + } + urEvents, ok := ur0["extracted_events"].([]any) + if !ok || len(urEvents) != 2 { + t.Fatalf("expected 2 extracted_events in unresolved range, got %v", ur0["extracted_events"]) + } +} + +func TestConvertAdvisoryToOSV_ExactVersion(t *testing.T) { + t.Parallel() + + vRange := "= 1.0.0" + + advisory := GHSAAdvisory{ + GHSAID: "GHSA-exact-version-test", + Summary: "Exact version test", + State: "published", + Vulnerabilities: []GHSAVulnerability{ + { + VulnerableVersionRange: &vRange, + }, + }, + } + + repoTarget := RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + } + + normalizedTags := map[string]git.NormalizedTag{ + "1-0-0": {Commit: "1111111111111111111111111111111111111111", OriginalTag: "v1.0.0"}, + } + + vuln, err := ConvertAdvisoryToOSV(advisory, repoTarget, normalizedTags) + if err != nil { + t.Fatalf("ConvertAdvisoryToOSV failed: %v", err) + } + + aff := vuln.GetAffected()[0] + if len(aff.GetRanges()) != 1 { + t.Fatalf("expected 1 range, got %d", len(aff.GetRanges())) + } + r := aff.GetRanges()[0] + if len(r.GetEvents()) != 2 { + t.Fatalf("expected 2 events (introduced + last_affected), got %d", len(r.GetEvents())) + } + if r.GetEvents()[0].GetIntroduced() != "1111111111111111111111111111111111111111" { + t.Errorf("unexpected introduced commit: %s", r.GetEvents()[0].GetIntroduced()) + } + if r.GetEvents()[1].GetLastAffected() != "1111111111111111111111111111111111111111" { + t.Errorf("unexpected last_affected commit: %s", r.GetEvents()[1].GetLastAffected()) + } + + dbSpec := r.GetDatabaseSpecific().AsMap() + if dbSpec["source"] != string(models.VersionSourceAffected) { + t.Errorf("expected source %q, got %v", models.VersionSourceAffected, dbSpec["source"]) + } + extracted, ok := dbSpec["extracted_events"].([]any) + if !ok || len(extracted) != 2 { + t.Fatalf("expected 2 extracted events, got %v", dbSpec["extracted_events"]) + } +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/github.go b/vulnfeeds/cmd/converters/repository-ghsa/github.go new file mode 100644 index 00000000000..c61775a3e9e --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/github.go @@ -0,0 +1,296 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/google/osv.dev/vulnfeeds/utility/logger" + "github.com/sethvargo/go-retry" +) + +// GHSAAdvisory represents a repository-level security advisory payload from GitHub REST API. +type GHSAAdvisory struct { + GHSAID string `json:"ghsa_id"` + CVEID *string `json:"cve_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + Summary string `json:"summary"` + Description *string `json:"description"` + Severity *string `json:"severity"` + Author *GHSAUser `json:"author"` + Publisher *GHSAUser `json:"publisher"` + Identifiers []GHSAIdentifier `json:"identifiers"` + State string `json:"state"` + CreatedAt *time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at"` + PublishedAt *time.Time `json:"published_at"` + ClosedAt *time.Time `json:"closed_at"` + WithdrawnAt *time.Time `json:"withdrawn_at"` + Vulnerabilities []GHSAVulnerability `json:"vulnerabilities"` + CVSSSeverities *GHSACVSSSeverities `json:"cvss_severities"` + CWEs []GHSACWE `json:"cwes"` + CWEIDs []string `json:"cwe_ids"` + Credits []GHSACredit `json:"credits"` +} + +type GHSAUser struct { + Login string `json:"login"` +} + +type GHSAIdentifier struct { + Type string `json:"type"` // "CVE" or "GHSA" + Value string `json:"value"` +} + +type GHSAVulnerability struct { + Package *GHSAPackage `json:"package"` + VulnerableVersionRange *string `json:"vulnerable_version_range"` + PatchedVersions *string `json:"patched_versions"` + VulnerableFunctions []string `json:"vulnerable_functions"` +} + +type GHSAPackage struct { + Ecosystem string `json:"ecosystem"` + Name *string `json:"name"` +} + +type GHSACVSSSeverities struct { + CVSSV3 *GHSACVSS `json:"cvss_v3"` + CVSSV4 *GHSACVSS `json:"cvss_v4"` +} + +type GHSACVSS struct { + VectorString *string `json:"vector_string"` + Score *float64 `json:"score"` +} + +type GHSACWE struct { + CWEID string `json:"cwe_id"` + Name string `json:"name"` +} + +type GHSACredit struct { + Login string `json:"login"` + Type string `json:"type"` +} + +// RepoTarget contains the parsed owner, repository name, and canonical URL. +type RepoTarget struct { + Owner string + Repo string + CanonicalURL string +} + +// ParseRepoTarget parses a repository identifier into a RepoTarget. +// Handles formats such as "owner/repo", "https://github.com/owner/repo", +// "github.com/owner/repo", and "git@github.com:owner/repo.git". +func ParseRepoTarget(raw string) (RepoTarget, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return RepoTarget{}, errors.New("empty repository identifier") + } + + // Remove common SSH and URL prefixes + if after, ok := strings.CutPrefix(trimmed, "git@github.com:"); ok { + trimmed = after + } else if after, ok := strings.CutPrefix(trimmed, "https://github.com/"); ok { + trimmed = after + } else if after, ok := strings.CutPrefix(trimmed, "http://github.com/"); ok { + trimmed = after + } else if after, ok := strings.CutPrefix(trimmed, "github.com/"); ok { + trimmed = after + } + + trimmed = strings.Trim(trimmed, "/") + trimmed = strings.TrimSuffix(trimmed, ".git") + + parts := strings.Split(trimmed, "/") + if len(parts) < 2 { + return RepoTarget{}, fmt.Errorf("invalid repository format %q: expected owner/repo", raw) + } + + owner, repo := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + repo = strings.TrimSuffix(repo, ".git") + if owner == "" || repo == "" { + return RepoTarget{}, fmt.Errorf("invalid owner or repo in %q", raw) + } + + return RepoTarget{ + Owner: owner, + Repo: repo, + CanonicalURL: fmt.Sprintf("https://github.com/%s/%s", owner, repo), + }, nil +} + +// GitHubClient handles HTTP communication with the GitHub REST API. +type GitHubClient struct { + client *http.Client + baseURL string + token string +} + +// NewGitHubClient creates a new GitHubClient. +func NewGitHubClient(token string, client *http.Client) *GitHubClient { + if client == nil { + client = http.DefaultClient + } + + return &GitHubClient{ + client: client, + baseURL: "https://api.github.com", + token: token, + } +} + +// SetBaseURL overrides the base API URL (primarily for testing). +func (c *GitHubClient) SetBaseURL(baseURL string) { + c.baseURL = strings.TrimRight(baseURL, "/") +} + +// FetchAdvisories retrieves all security advisories for a given repository, handling pagination and retries. +func (c *GitHubClient) FetchAdvisories(ctx context.Context, owner, repo, state string) ([]GHSAAdvisory, error) { + reqURL := fmt.Sprintf("%s/repos/%s/%s/security-advisories?per_page=100", c.baseURL, url.PathEscape(owner), url.PathEscape(repo)) + if state != "" && state != "all" { + reqURL += "&state=" + url.QueryEscape(state) + } + + var allAdvisories []GHSAAdvisory + + for reqURL != "" { + advisories, nextURL, err := c.fetchAdvisoriesPage(ctx, reqURL) + if err != nil { + return nil, fmt.Errorf("fetching advisories for %s/%s from %s: %w", owner, repo, reqURL, err) + } + + allAdvisories = append(allAdvisories, advisories...) + reqURL = nextURL + } + + return allAdvisories, nil +} + +func (c *GitHubClient) fetchAdvisoriesPage(ctx context.Context, requestURL string) ([]GHSAAdvisory, string, error) { + var ( + advisories []GHSAAdvisory + nextURL string + ) + + b := retry.NewExponential(1 * time.Second) + b = retry.WithMaxRetries(3, b) + + err := retry.Do(ctx, b, func(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return err + } + + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-Github-Api-Version", "2022-11-28") + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := c.client.Do(req) + if err != nil { + return retry.RetryableError(err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests || (resp.StatusCode >= 500 && resp.StatusCode < 600) { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return retry.RetryableError(fmt.Errorf("temporary HTTP error status %d: %s", resp.StatusCode, string(body))) + } + + if resp.StatusCode == http.StatusNotFound { + logger.Warn("Repository security advisories returned 404", slog.String("url", requestURL)) + return nil + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return fmt.Errorf("GitHub API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed reading GitHub API response: %w", err) + } + + if err := json.Unmarshal(bodyBytes, &advisories); err != nil { + return fmt.Errorf("failed decoding advisories JSON: %w", err) + } + + nextURL = parseNextLink(resp.Header.Get("Link")) + + return nil + }) + + if err != nil { + return nil, "", err + } + + return advisories, nextURL, nil +} + +var linkNextRegex = regexp.MustCompile(`<([^>]+)>;\s*rel="next"`) + +// parseNextLink extracts the URL from a Link header where rel="next". +func parseNextLink(linkHeader string) string { + if linkHeader == "" { + return "" + } + + for part := range strings.SplitSeq(linkHeader, ",") { + subparts := strings.Split(part, ";") + if len(subparts) < 2 { + continue + } + + isNext := false + for _, param := range subparts[1:] { + param = strings.TrimSpace(param) + if param == `rel="next"` || param == `rel='next'` || param == `rel=next` { + isNext = true + break + } + } + + if isNext { + urlPart := strings.TrimSpace(subparts[0]) + urlPart = strings.TrimPrefix(urlPart, "<") + urlPart = strings.TrimSuffix(urlPart, ">") + + return urlPart + } + } + + matches := linkNextRegex.FindStringSubmatch(linkHeader) + if len(matches) > 1 { + return matches[1] + } + + return "" +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/github_test.go b/vulnfeeds/cmd/converters/repository-ghsa/github_test.go new file mode 100644 index 00000000000..def607758f9 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/github_test.go @@ -0,0 +1,372 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseRepoTarget(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantTarget RepoTarget + expectError bool + }{ + { + name: "standard owner/repo", + input: "google/osv.dev", + wantTarget: RepoTarget{ + Owner: "google", + Repo: "osv.dev", + CanonicalURL: "https://github.com/google/osv.dev", + }, + }, + { + name: "https URL", + input: "https://github.com/gin-gonic/gin", + wantTarget: RepoTarget{ + Owner: "gin-gonic", + Repo: "gin", + CanonicalURL: "https://github.com/gin-gonic/gin", + }, + }, + { + name: "https URL with trailing slash", + input: "https://github.com/gin-gonic/gin/", + wantTarget: RepoTarget{ + Owner: "gin-gonic", + Repo: "gin", + CanonicalURL: "https://github.com/gin-gonic/gin", + }, + }, + { + name: "https URL with .git", + input: "https://github.com/gin-gonic/gin.git", + wantTarget: RepoTarget{ + Owner: "gin-gonic", + Repo: "gin", + CanonicalURL: "https://github.com/gin-gonic/gin", + }, + }, + { + name: "http URL", + input: "http://github.com/foo/bar", + wantTarget: RepoTarget{ + Owner: "foo", + Repo: "bar", + CanonicalURL: "https://github.com/foo/bar", + }, + }, + { + name: "github.com prefix without scheme", + input: "github.com/foo/bar", + wantTarget: RepoTarget{ + Owner: "foo", + Repo: "bar", + CanonicalURL: "https://github.com/foo/bar", + }, + }, + { + name: "SSH git@github.com format", + input: "git@github.com:torvalds/linux.git", + wantTarget: RepoTarget{ + Owner: "torvalds", + Repo: "linux", + CanonicalURL: "https://github.com/torvalds/linux", + }, + }, + { + name: "with leading and trailing whitespace", + input: " owner/repo ", + wantTarget: RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + }, + }, + { + name: "owner and repo with spaces", + input: " owner / repo ", + wantTarget: RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + }, + }, + { + name: "repo with trailing slash and .git", + input: "https://github.com/owner/repo.git/", + wantTarget: RepoTarget{ + Owner: "owner", + Repo: "repo", + CanonicalURL: "https://github.com/owner/repo", + }, + }, + { + name: "empty string", + input: "", + expectError: true, + }, + { + name: "single name without slash", + input: "justrepo", + expectError: true, + }, + { + name: "only owner with trailing slash", + input: "owner/", + expectError: true, + }, + { + name: "empty owner", + input: "/repo", + expectError: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := ParseRepoTarget(tc.input) + if tc.expectError { + if err == nil { + t.Fatalf("ParseRepoTarget(%q) expected error, got nil", tc.input) + } + + return + } + + if err != nil { + t.Fatalf("ParseRepoTarget(%q) unexpected error: %v", tc.input, err) + } + + if diff := cmp.Diff(tc.wantTarget, got); diff != "" { + t.Errorf("ParseRepoTarget(%q) mismatch (-want +got):\n%s", tc.input, diff) + } + }) + } +} + +func TestParseNextLink(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + header string + want string + }{ + { + name: "standard next and last links", + header: `; rel="next", ; rel="last"`, + want: "https://api.github.com/repositories/123/security-advisories?after=Y3Vyc29yOjE%3D", + }, + { + name: "only next link", + header: `; rel="next"`, + want: "https://api.github.com/repositories/123/security-advisories?after=next", + }, + { + name: "no next link (prev and first only)", + header: `; rel="prev", ; rel="first"`, + want: "", + }, + { + name: "unquoted rel=next", + header: `; rel=next`, + want: "https://api.github.com/repositories/123/security-advisories?after=unquoted", + }, + { + name: "single quoted rel='next'", + header: `; rel='next'`, + want: "https://api.github.com/repositories/123/security-advisories?after=single", + }, + { + name: "empty header", + header: "", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := parseNextLink(tc.header) + if got != tc.want { + t.Errorf("parseNextLink(%q) = %q; want %q", tc.header, got, tc.want) + } + }) + } +} + +func TestGitHubClient_FetchAdvisories_Pagination(t *testing.T) { + t.Parallel() + + page1Advisories := []GHSAAdvisory{ + {GHSAID: "GHSA-1111-1111-1111", Summary: "Advisory 1"}, + } + page2Advisories := []GHSAAdvisory{ + {GHSAID: "GHSA-2222-2222-2222", Summary: "Advisory 2"}, + } + + var serverURL string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify expected headers + if auth := r.Header.Get("Authorization"); auth != "Bearer test-token-123" { + t.Errorf("expected Bearer token, got %q", auth) + } + if r.Header.Get("Accept") != "application/vnd.github+json" { + t.Errorf("expected Accept header for github+json, got %q", r.Header.Get("Accept")) + } + if r.Header.Get("X-Github-Api-Version") != "2022-11-28" { + t.Errorf("expected API version header, got %q", r.Header.Get("X-Github-Api-Version")) + } + + cursor := r.URL.Query().Get("after") + w.Header().Set("Content-Type", "application/json") + + switch cursor { + case "": + // Page 1: return page1 and next link + w.Header().Set("Link", `<`+serverURL+`/repos/test-owner/test-repo/security-advisories?per_page=100&after=page2cursor>; rel="next"`) + _ = json.NewEncoder(w).Encode(page1Advisories) + case "page2cursor": + // Page 2: return page2 without next link + _ = json.NewEncoder(w).Encode(page2Advisories) + default: + http.NotFound(w, r) + } + })) + defer ts.Close() + serverURL = ts.URL + + client := NewGitHubClient("test-token-123", ts.Client()) + client.SetBaseURL(ts.URL) + + advisories, err := client.FetchAdvisories(context.Background(), "test-owner", "test-repo", "published") + if err != nil { + t.Fatalf("FetchAdvisories failed: %v", err) + } + + if len(advisories) != 2 { + t.Fatalf("FetchAdvisories returned %d advisories; want 2", len(advisories)) + } + if advisories[0].GHSAID != "GHSA-1111-1111-1111" || advisories[1].GHSAID != "GHSA-2222-2222-2222" { + t.Errorf("Unexpected advisories fetched: %+v", advisories) + } +} + +func TestGitHubClient_FetchAdvisories_NotFound(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer ts.Close() + + client := NewGitHubClient("", ts.Client()) + client.SetBaseURL(ts.URL) + + advisories, err := client.FetchAdvisories(context.Background(), "owner", "notfound", "all") + if err != nil { + t.Fatalf("expected nil error on 404, got: %v", err) + } + if len(advisories) != 0 { + t.Errorf("expected 0 advisories on 404, got %d", len(advisories)) + } +} + +func TestGitHubClient_FetchAdvisories_RetryOn429(t *testing.T) { + t.Parallel() + + var attempts atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"message": "rate limit exceeded"}`)) + + return + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]GHSAAdvisory{ + {GHSAID: "GHSA-recovered", Summary: "Recovered"}, + }) + })) + defer ts.Close() + + client := NewGitHubClient("", ts.Client()) + client.SetBaseURL(ts.URL) + + advisories, err := client.FetchAdvisories(context.Background(), "owner", "retry-repo", "published") + if err != nil { + t.Fatalf("FetchAdvisories failed after retry: %v", err) + } + if len(advisories) != 1 || advisories[0].GHSAID != "GHSA-recovered" { + t.Errorf("Unexpected advisories after retry: %+v", advisories) + } + if attempts.Load() < 2 { + t.Errorf("Expected at least 2 attempts, got %d", attempts.Load()) + } +} + +func TestCollectRepos(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "repos.txt") + fileContent := "# Comment\nrepo/from-file-1\n\nrepo/from-file-2\n# Another comment\nrepo/from-csv-1\n" + if err := os.WriteFile(filePath, []byte(fileContent), 0644); err != nil { + t.Fatalf("Failed writing temp repos file: %v", err) + } + + reposCSV := "repo/from-csv-1, repo/from-csv-2" + positional := []string{"repo/positional", "repo/from-file-1"} + + ctx := context.Background() + repos, err := collectRepos(ctx, reposCSV, filePath, "", positional, nil) + if err != nil { + t.Fatalf("collectRepos failed: %v", err) + } + + want := []string{ + "repo/from-csv-1", + "repo/from-csv-2", + "repo/from-file-1", + "repo/from-file-2", + "repo/positional", + } + + if diff := cmp.Diff(want, repos); diff != "" { + t.Errorf("collectRepos mismatch (-want +got):\n%s", diff) + } + + // Test non-existent file returns error + _, err = collectRepos(ctx, "", filepath.Join(tmpDir, "does-not-exist.txt"), "", nil, nil) + if err == nil { + t.Errorf("collectRepos with non-existent file expected error, got nil") + } +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/jobdata.go b/vulnfeeds/cmd/converters/repository-ghsa/jobdata.go new file mode 100644 index 00000000000..1cd984430a5 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/jobdata.go @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "cloud.google.com/go/datastore" +) + +const ( + // jobDataKind is the Datastore entity kind for job metadata. + jobDataKind = "JobData" + // defaultJobDataKey is the default entity ID in Datastore for this cron job. + defaultJobDataKey = "repository_ghsa_last_run" +) + +// jobDataLastRunEntity mirrors the Datastore JobData entity schema used across osv.dev. +type jobDataLastRunEntity struct { + Value *time.Time `datastore:"value,noindex"` +} + +// JobDataStore abstracts storage operations for job execution metadata. +type JobDataStore interface { + GetLastRun(ctx context.Context, jobID string) (*time.Time, error) + SetLastRun(ctx context.Context, jobID string, t time.Time) error + Close() error +} + +// DatastoreJobStore implements JobDataStore using Google Cloud Datastore. +type DatastoreJobStore struct { + client *datastore.Client +} + +// NewDatastoreJobStore creates a new DatastoreJobStore for the given GCP project. +func NewDatastoreJobStore(ctx context.Context, projectID string) (*DatastoreJobStore, error) { + client, err := datastore.NewClient(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("datastore.NewClient: %w", err) + } + + return &DatastoreJobStore{client: client}, nil +} + +// GetLastRun retrieves the last execution timestamp for the given job ID. +func (s *DatastoreJobStore) GetLastRun(ctx context.Context, jobID string) (*time.Time, error) { + key := datastore.NameKey(jobDataKind, jobID, nil) + var entity jobDataLastRunEntity + err := s.client.Get(ctx, key, &entity) + if err != nil { + if errors.Is(err, datastore.ErrNoSuchEntity) { + //nolint:nilnil // A nil pointer with nil error signifies that no prior execution record exists. + return nil, nil + } + + return nil, fmt.Errorf("failed to get %s from Datastore: %w", jobID, err) + } + + return entity.Value, nil +} + +// SetLastRun records the execution timestamp for the given job ID. +func (s *DatastoreJobStore) SetLastRun(ctx context.Context, jobID string, t time.Time) error { + key := datastore.NameKey(jobDataKind, jobID, nil) + utcTime := t.UTC() + entity := jobDataLastRunEntity{Value: &utcTime} + _, err := s.client.Put(ctx, key, &entity) + if err != nil { + return fmt.Errorf("failed to write %s to Datastore: %w", jobID, err) + } + + return nil +} + +// Close closes the underlying Datastore client. +func (s *DatastoreJobStore) Close() error { + if s.client != nil { + return s.client.Close() + } + + return nil +} + +// MemJobStore implements JobDataStore in-memory for testing and dry-run executions. +type MemJobStore struct { + mu sync.RWMutex + data map[string]time.Time +} + +// NewMemJobStore creates a new in-memory JobDataStore. +func NewMemJobStore() *MemJobStore { + return &MemJobStore{ + data: make(map[string]time.Time), + } +} + +// GetLastRun retrieves the timestamp from memory. +func (m *MemJobStore) GetLastRun(_ context.Context, jobID string) (*time.Time, error) { + m.mu.RLock() + defer m.mu.RUnlock() + t, ok := m.data[jobID] + if !ok { + //nolint:nilnil // A nil pointer with nil error signifies that no prior execution record exists. + return nil, nil + } + tCopy := t.UTC() + + return &tCopy, nil +} + +// SetLastRun stores the timestamp in memory. +func (m *MemJobStore) SetLastRun(_ context.Context, jobID string, t time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + m.data[jobID] = t.UTC() + + return nil +} + +// Close is a no-op for MemJobStore. +func (m *MemJobStore) Close() error { + return nil +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/jobdata_test.go b/vulnfeeds/cmd/converters/repository-ghsa/jobdata_test.go new file mode 100644 index 00000000000..098d53c2ed8 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/jobdata_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + "time" +) + +func TestMemJobStore(t *testing.T) { + ctx := context.Background() + store := NewMemJobStore() + defer store.Close() + + // Initial retrieval should return nil + lastRun, err := store.GetLastRun(ctx, "test_job") + if err != nil { + t.Fatalf("unexpected error getting last run: %v", err) + } + if lastRun != nil { + t.Fatalf("expected nil last run, got %v", lastRun) + } + + // Record a timestamp + now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + if err := store.SetLastRun(ctx, "test_job", now); err != nil { + t.Fatalf("unexpected error setting last run: %v", err) + } + + // Retrieve recorded timestamp + got, err := store.GetLastRun(ctx, "test_job") + if err != nil { + t.Fatalf("unexpected error getting last run after set: %v", err) + } + if got == nil { + t.Fatal("expected non-nil last run, got nil") + } + if !got.Equal(now) { + t.Fatalf("expected time %v, got %v", now, *got) + } + + // Overwrite timestamp + later := now.Add(6 * time.Hour) + if err := store.SetLastRun(ctx, "test_job", later); err != nil { + t.Fatalf("unexpected error updating last run: %v", err) + } + + gotLater, err := store.GetLastRun(ctx, "test_job") + if err != nil { + t.Fatalf("unexpected error getting updated last run: %v", err) + } + if gotLater == nil || !gotLater.Equal(later) { + t.Fatalf("expected time %v, got %v", later, gotLater) + } + + // Another job ID remains empty + other, err := store.GetLastRun(ctx, "other_job") + if err != nil { + t.Fatalf("unexpected error getting other job: %v", err) + } + if other != nil { + t.Fatalf("expected other_job to be nil, got %v", other) + } +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/main.go b/vulnfeeds/cmd/converters/repository-ghsa/main.go new file mode 100644 index 00000000000..85c6c0eec31 --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/main.go @@ -0,0 +1,502 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main fetches GitHub repository-level security advisories and converts them to OSV format. +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "cloud.google.com/go/storage" + gcs "github.com/google/osv.dev/vulnfeeds/gcs-tools" + "github.com/google/osv.dev/vulnfeeds/git" + "github.com/google/osv.dev/vulnfeeds/utility/logger" + "github.com/google/osv.dev/vulnfeeds/vulns" + "golang.org/x/sync/semaphore" +) + +var ( + reposFlag = flag.String("repos", "", "Comma-separated list of GitHub repositories (e.g. owner/repo or https://github.com/owner/repo)") + reposFileFlag = flag.String("repos-file", "", "Path to a file (or gs://bucket/path.json) containing repositories") + reposGCSPathFlag = flag.String("repos-gcs-path", "", "Google Cloud Storage URI (gs://bucket/path.json) containing repository list") + outDirFlag = flag.String("out-dir", "output", "Directory to output converted OSV JSON files") + gitterHostFlag = flag.String("gitter-host", "", "URL of the Gitter caching service (defaults to GITTER_HOST env var)") + githubTokenFlag = flag.String("github-token", "", "GitHub API token (defaults to GITHUB_TOKEN or GH_TOKEN env var)") + stateFlag = flag.String("state", "published", "Filter advisories by state (published, closed, withdrawn, or all)") + workersFlag = flag.Int("workers", 8, "Number of concurrent workers for processing repositories") + uploadToGCSFlag = flag.Bool("upload-to-gcs", false, "Whether to upload output OSV records directly to Google Cloud Storage") + outputBucketFlag = flag.String("output-bucket", "", "Destination GCS bucket name (defaults to OUTPUT_GCS_BUCKET or OUTPUT_BUCKET env var)") + gcsPrefixFlag = flag.String("gcs-prefix", "ghsa-repo-osv", "Prefix path in GCS bucket") + datastoreProjectFlag = flag.String("datastore-project", "", "Google Cloud project ID for Datastore JobData tracking (defaults to GOOGLE_CLOUD_PROJECT or DATASTORE_PROJECT_ID env var)") + datastoreJobIDFlag = flag.String("datastore-job-id", defaultJobDataKey, "Entity ID in Datastore JobData kind") + saveLastRunFlag = flag.Bool("save-last-run", false, "Whether to record last_run_time in Datastore upon successful completion") +) + +func main() { + flag.Parse() + + logger.InitGlobalLogger() + defer logger.Close() + + ctx := context.Background() + + var storageClient *storage.Client + defer func() { + if storageClient != nil { + _ = storageClient.Close() + } + }() + + repos, err := collectRepos(ctx, *reposFlag, *reposFileFlag, *reposGCSPathFlag, flag.Args(), storageClient) + if err != nil { + logger.Fatal("Failed to collect repositories", slog.Any("error", err)) + } + if len(repos) == 0 { + logger.Fatal("No repositories specified. Use -repos, -repos-file, -repos-gcs-path, or positional arguments.") + } + + // Configure Gitter host if specified via flag + if *gitterHostFlag != "" { + _ = os.Setenv("GITTER_HOST", *gitterHostFlag) + } + + // Configure GitHub token + token := *githubTokenFlag + if token == "" { + token = os.Getenv("GITHUB_TOKEN") + if token == "" { + token = os.Getenv("GH_TOKEN") + } + } + + if *outDirFlag != "" { + if err := os.MkdirAll(*outDirFlag, 0755); err != nil { + logger.Fatal("Failed to create output directory", slog.String("dir", *outDirFlag), slog.Any("error", err)) + } + } + + // Configure GCS Upload + uploadToGCS := *uploadToGCSFlag + if !uploadToGCS && os.Getenv("UPLOAD_TO_GCS") == "true" { + uploadToGCS = true + } + + bucketName := *outputBucketFlag + if bucketName == "" { + bucketName = os.Getenv("OUTPUT_GCS_BUCKET") + if bucketName == "" { + bucketName = os.Getenv("OUTPUT_BUCKET") + if bucketName == "" { + bucketName = "osv-test-ghsa-repo-conversion" + } + } + } + + var gcsHelper *gcs.Helper + if uploadToGCS { + var err error + gcsHelper, err = gcs.InitUploadPool(ctx, *workersFlag, bucketName) + if err != nil { + logger.Fatal("Failed to initialize GCS helper", slog.String("bucket", bucketName), slog.Any("error", err)) + } + defer gcsHelper.CloseAndWait() + } + + // Configure Datastore JobData tracking + dsProject := *datastoreProjectFlag + if dsProject == "" { + dsProject = os.Getenv("GOOGLE_CLOUD_PROJECT") + if dsProject == "" { + dsProject = os.Getenv("DATASTORE_PROJECT_ID") + } + } + + var jobStore JobDataStore + shouldSaveLastRun := *saveLastRunFlag || (dsProject != "" && *datastoreProjectFlag != "") + if dsProject != "" { + var err error + jobStore, err = NewDatastoreJobStore(ctx, dsProject) + if err != nil { + logger.Warn("Failed to initialize Datastore client for JobData tracking", slog.String("project", dsProject), slog.Any("error", err)) + } else { + defer jobStore.Close() + if lastRun, err := jobStore.GetLastRun(ctx, *datastoreJobIDFlag); err == nil && lastRun != nil { + logger.Info("Previous job run time retrieved from Datastore", slog.String("job_id", *datastoreJobIDFlag), slog.Time("last_run", *lastRun)) + } + } + } + + httpClient := &http.Client{ + Timeout: 60 * time.Second, + } + + ghClient := NewGitHubClient(token, httpClient) + tagsCache := git.NewRepoTagsCache() + + processAllRepositories(ctx, repos, ghClient, tagsCache, httpClient, gcsHelper) + + // Record execution timestamp in Datastore upon completion + if shouldSaveLastRun && jobStore != nil { + now := time.Now().UTC() + if err := jobStore.SetLastRun(ctx, *datastoreJobIDFlag, now); err != nil { + logger.Error("Failed to save last_run_time in Datastore", slog.String("job_id", *datastoreJobIDFlag), slog.Any("error", err)) + } else { + logger.Info("Successfully saved last_run_time to Datastore", slog.String("job_id", *datastoreJobIDFlag), slog.Time("timestamp", now)) + } + } +} + +// parseRepoFile parses repository identifiers from raw bytes supporting JSON arrays of strings, +// JSON arrays of objects with repo keys, JSON maps/dictionaries, and plain newline-delimited text. +func parseRepoFile(data []byte) ([]string, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 { + return nil, nil + } + + // If data starts with [ or {, attempt JSON parsing + if trimmed[0] == '[' || trimmed[0] == '{' { + // 1. Try []string + var strList []string + if err := json.Unmarshal(trimmed, &strList); err == nil { + var result []string + for _, s := range strList { + s = strings.TrimSpace(s) + if s != "" { + result = append(result, s) + } + } + + return result, nil + } + + // 2. Try []map[string]any + var objList []map[string]any + if err := json.Unmarshal(trimmed, &objList); err == nil { + var result []string + for _, obj := range objList { + if r := extractRepoFromMap(obj); r != "" { + result = append(result, r) + } + } + if len(result) > 0 { + return result, nil + } + } + + // 3. Try map[string]json.RawMessage + var rawMap map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &rawMap); err == nil { + // Check for wrapper keys like "repos", "repositories", "items", "data" + for _, key := range []string{"repos", "repositories", "items", "data"} { + if raw, ok := rawMap[key]; ok { + subList, err := parseRepoFile(raw) + if err == nil && len(subList) > 0 { + return subList, nil + } + } + } + + // If no known wrapper key, keys themselves might be repo names (e.g. owner/repo) + var result []string + for k := range rawMap { + k = strings.TrimSpace(k) + if strings.Contains(k, "/") { + result = append(result, k) + } + } + if len(result) > 0 { + return result, nil + } + } + } + + // Fallback to line-by-line plain text scanning + scanner := bufio.NewScanner(bytes.NewReader(trimmed)) + var result []string + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" && !strings.HasPrefix(line, "#") { + result = append(result, line) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading plain text repo list: %w", err) + } + + return result, nil +} + +func extractRepoFromMap(m map[string]any) string { + candidateKeys := []string{"repo", "url", "name", "repository", "canonical_url", "git"} + for _, k := range candidateKeys { + if val, ok := m[k]; ok { + if s, ok := val.(string); ok { + s = strings.TrimSpace(s) + if s != "" { + return s + } + } + } + } + + return "" +} + +// readRepoFile reads repository identifiers from either a local file or a Google Cloud Storage URI (gs://bucket/object). +func readRepoFile(ctx context.Context, pathStr string, gcsClient *storage.Client) ([]string, error) { + if u, ok := strings.CutPrefix(pathStr, "gs://"); ok { + parts := strings.SplitN(u, "/", 2) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return nil, fmt.Errorf("invalid GCS path %q: must be in format gs://bucket/object", pathStr) + } + bucketName := parts[0] + objectName := parts[1] + + clientToUse := gcsClient + var createdClient bool + if clientToUse == nil { + var err error + clientToUse, err = storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCS client for %s: %w", pathStr, err) + } + createdClient = true + } + if createdClient { + defer clientToUse.Close() + } + + rc, err := clientToUse.Bucket(bucketName).Object(objectName).NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("opening GCS object %s: %w", pathStr, err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("reading GCS object %s: %w", pathStr, err) + } + + return parseRepoFile(data) + } + + data, err := os.ReadFile(pathStr) + if err != nil { + return nil, fmt.Errorf("reading local file %s: %w", pathStr, err) + } + + return parseRepoFile(data) +} + +func collectRepos(ctx context.Context, reposCSV, reposFile, reposGCSPath string, positional []string, gcsClient *storage.Client) ([]string, error) { + var list []string + + if reposCSV != "" { + for r := range strings.SplitSeq(reposCSV, ",") { + r = strings.TrimSpace(r) + if r != "" { + list = append(list, r) + } + } + } + + if reposGCSPath != "" { + gcsList, err := readRepoFile(ctx, reposGCSPath, gcsClient) + if err != nil { + return nil, fmt.Errorf("failed to read repos from GCS path %s: %w", reposGCSPath, err) + } + list = append(list, gcsList...) + } + + if reposFile != "" { + fileList, err := readRepoFile(ctx, reposFile, gcsClient) + if err != nil { + return nil, fmt.Errorf("failed to read repos from repos-file %s: %w", reposFile, err) + } + list = append(list, fileList...) + } + + for _, p := range positional { + p = strings.TrimSpace(p) + if p != "" { + list = append(list, p) + } + } + + // Deduplicate repositories preserving order + seen := make(map[string]bool) + var deduped []string + for _, r := range list { + if !seen[r] { + seen[r] = true + deduped = append(deduped, r) + } + } + + return deduped, nil +} + +func processAllRepositories(ctx context.Context, repos []string, ghClient *GitHubClient, tagsCache git.RepoTagsCache, httpClient *http.Client, gcsHelper *gcs.Helper) { + numWorkers := *workersFlag + if numWorkers <= 0 { + numWorkers = 1 + } + + sem := semaphore.NewWeighted(int64(numWorkers)) + var wg sync.WaitGroup + + var ( + totalAdvisoriesCount atomic.Uint64 + successCount atomic.Uint64 + failCount atomic.Uint64 + ) + + logger.Info("Starting processing repositories", slog.Int("repo_count", len(repos)), slog.Int("workers", numWorkers)) + + for _, rawRepo := range repos { + target, err := ParseRepoTarget(rawRepo) + if err != nil { + logger.Warn("Failed parsing repository identifier", slog.String("raw", rawRepo), slog.Any("error", err)) + failCount.Add(1) + + continue + } + + if err := sem.Acquire(ctx, 1); err != nil { + logger.Error("Context cancelled while acquiring semaphore", slog.Any("error", err)) + break + } + + wg.Add(1) + go func(t RepoTarget) { + defer sem.Release(1) + defer wg.Done() + + count, err := processRepository(ctx, t, ghClient, tagsCache, httpClient, gcsHelper) + if err != nil { + logger.Error("Failed processing repository", slog.String("repo", t.CanonicalURL), slog.Any("error", err)) + failCount.Add(1) + } else { + successCount.Add(1) + totalAdvisoriesCount.Add(uint64(count)) + } + }(target) + } + + wg.Wait() + + logger.Info("Processing complete", + slog.Uint64("successful_repos", successCount.Load()), + slog.Uint64("failed_repos", failCount.Load()), + slog.Uint64("total_advisories_converted", totalAdvisoriesCount.Load()), + ) +} + +func processRepository(ctx context.Context, target RepoTarget, ghClient *GitHubClient, tagsCache git.RepoTagsCache, httpClient *http.Client, gcsHelper *gcs.Helper) (int, error) { + logger.Info("Fetching advisories for repository", slog.String("owner", target.Owner), slog.String("repo", target.Repo)) + + advisories, err := ghClient.FetchAdvisories(ctx, target.Owner, target.Repo, *stateFlag) + if err != nil { + return 0, fmt.Errorf("fetching advisories: %w", err) + } + + if len(advisories) == 0 { + logger.Info("No advisories found for repository", slog.String("repo", target.CanonicalURL)) + return 0, nil + } + + logger.Info("Found advisories for repository", slog.String("repo", target.CanonicalURL), slog.Int("count", len(advisories))) + + // Fetch normalized Git tags for commit resolution + //nolint:contextcheck // git.NormalizeRepoTags does not accept a Context parameter. + normalizedTags, err := git.NormalizeRepoTags(target.CanonicalURL, tagsCache, httpClient) + if err != nil { + logger.Warn("Failed to normalize tags for repository; proceeding with partial commit resolution", + slog.String("repo", target.CanonicalURL), slog.Any("error", err)) + normalizedTags = make(map[string]git.NormalizedTag) + } + + convertedCount := 0 + for _, advisory := range advisories { + vuln, err := ConvertAdvisoryToOSV(advisory, target, normalizedTags) + if err != nil { + logger.Warn("Failed converting advisory to OSV", + slog.String("id", advisory.GHSAID), slog.String("repo", target.CanonicalURL), slog.Any("error", err)) + + continue + } + + if err := writeOSVRecord(vuln, *outDirFlag, gcsHelper, *gcsPrefixFlag); err != nil { + logger.Error("Failed writing OSV record", + slog.String("id", advisory.GHSAID), slog.Any("error", err)) + + continue + } + + convertedCount++ + } + + return convertedCount, nil +} + +func writeOSVRecord(vuln *vulns.Vulnerability, outDir string, gcsHelper *gcs.Helper, prefix string) error { + vulnID := vuln.GetId() + if vulnID == "" { + return errors.New("vulnerability ID is empty") + } + + var buf bytes.Buffer + if err := vuln.ToJSON(&buf); err != nil { + return fmt.Errorf("serializing vulnerability %s to JSON: %w", vulnID, err) + } + + // Write to local out-dir if provided + if outDir != "" { + localPath := filepath.Join(outDir, vulnID+".json") + if err := os.WriteFile(localPath, buf.Bytes(), 0644); err != nil { + return fmt.Errorf("writing local file %s: %w", localPath, err) + } + } + + // Upload to GCS if configured + if gcsHelper != nil { + gcsObjName := vulnID + ".json" + if prefix != "" { + gcsObjName = strings.Trim(prefix, "/") + "/" + vulnID + ".json" + } + hash := sha256.Sum256(buf.Bytes()) + hexHash := hex.EncodeToString(hash[:]) + gcsHelper.Upload(gcsObjName, bytes.NewReader(buf.Bytes()), hexHash, "application/json") + } + + return nil +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/main_test.go b/vulnfeeds/cmd/converters/repository-ghsa/main_test.go new file mode 100644 index 00000000000..c8357233dcf --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/main_test.go @@ -0,0 +1,354 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/osv.dev/vulnfeeds/git" + "github.com/google/osv.dev/vulnfeeds/vulns" +) + +func TestProcessRepository_EndToEnd(t *testing.T) { + tmpDir := t.TempDir() + origOutDir := *outDirFlag + origState := *stateFlag + *outDirFlag = tmpDir + *stateFlag = "published" + defer func() { + *outDirFlag = origOutDir + *stateFlag = origState + }() + + cveID := "CVE-2026-8888" + desc := "End to end test advisory" + pkgName := "e2e-pkg" + vRange := ">= 1.0.0, < 2.0.0" + pubTime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + advisories := []GHSAAdvisory{ + { + GHSAID: "GHSA-e2e-test-1234", + CVEID: &cveID, + HTMLURL: "https://github.com/test-owner/test-repo/security/advisories/GHSA-e2e-test-1234", + Summary: "E2E Advisory", + Description: &desc, + State: "published", + PublishedAt: &pubTime, + Vulnerabilities: []GHSAVulnerability{ + { + Package: &GHSAPackage{ + Ecosystem: "npm", + Name: &pkgName, + }, + VulnerableVersionRange: &vRange, + }, + }, + }, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(advisories) + })) + defer ts.Close() + + ghClient := NewGitHubClient("", ts.Client()) + ghClient.SetBaseURL(ts.URL) + + target := RepoTarget{ + Owner: "test-owner", + Repo: "test-repo", + CanonicalURL: "https://github.com/test-owner/test-repo", + } + + tagsCache := git.NewRepoTagsCache() + tagsCache.Set(target.CanonicalURL, git.RepoTagsMap{ + NormalizedTag: map[string]git.NormalizedTag{ + "1-0-0": {Commit: "1111111111111111111111111111111111111111", OriginalTag: "v1.0.0"}, + "2-0-0": {Commit: "2222222222222222222222222222222222222222", OriginalTag: "v2.0.0"}, + }, + }) + + count, err := processRepository(context.Background(), target, ghClient, tagsCache, ts.Client(), nil) + if err != nil { + t.Fatalf("processRepository failed: %v", err) + } + if count != 1 { + t.Errorf("processRepository converted %d advisories; want 1", count) + } + + // Verify output record + recordPath := filepath.Join(tmpDir, "GHSA-e2e-test-1234.json") + data, err := os.ReadFile(recordPath) + if err != nil { + t.Fatalf("expected output file %s: %v", recordPath, err) + } + + var parsed map[string]any + if err := json.Unmarshal(data, &parsed); err != nil { + t.Fatalf("failed unmarshaling generated OSV JSON: %v", err) + } + + if parsed["id"] != "GHSA-e2e-test-1234" { + t.Errorf("expected ID GHSA-e2e-test-1234, got %v", parsed["id"]) + } + if parsed["summary"] != "E2E Advisory" { + t.Errorf("expected summary E2E Advisory, got %v", parsed["summary"]) + } + + affectedList, ok := parsed["affected"].([]any) + if !ok || len(affectedList) == 0 { + t.Fatalf("expected affected list in generated JSON, got: %v", parsed["affected"]) + } + for i, aff := range affectedList { + affMap, ok := aff.(map[string]any) + if !ok { + t.Fatalf("affected[%d] is not a map: %v", i, aff) + } + if pkg, exists := affMap["package"]; exists { + t.Errorf("affected[%d] has unexpected 'package' field: %v", i, pkg) + } + rangesList, ok := affMap["ranges"].([]any) + if !ok || len(rangesList) == 0 { + t.Fatalf("affected[%d] missing ranges: %v", i, affMap) + } + for j, r := range rangesList { + rMap, ok := r.(map[string]any) + if !ok { + t.Fatalf("affected[%d].ranges[%d] is not a map: %v", i, j, r) + } + if rMap["type"] != "GIT" { + t.Errorf("affected[%d].ranges[%d] type = %v, want GIT", i, j, rMap["type"]) + } + dbSpec, ok := rMap["database_specific"].(map[string]any) + if !ok { + t.Fatalf("affected[%d].ranges[%d] missing database_specific: %v", i, j, rMap) + } + if dbSpec["source"] != "AFFECTED_FIELD" { + t.Errorf("affected[%d].ranges[%d] database_specific source = %v, want AFFECTED_FIELD", i, j, dbSpec["source"]) + } + extracted, ok := dbSpec["extracted_events"].([]any) + if !ok || len(extracted) == 0 { + t.Fatalf("affected[%d].ranges[%d] database_specific missing extracted_events: %v", i, j, dbSpec) + } + } + } +} + +func TestProcessRepository_EmptyAdvisories(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]GHSAAdvisory{}) + })) + defer ts.Close() + + ghClient := NewGitHubClient("", ts.Client()) + ghClient.SetBaseURL(ts.URL) + + target := RepoTarget{ + Owner: "owner", + Repo: "empty", + CanonicalURL: "https://github.com/owner/empty", + } + + tagsCache := git.NewRepoTagsCache() + count, err := processRepository(context.Background(), target, ghClient, tagsCache, ts.Client(), nil) + if err != nil { + t.Fatalf("processRepository unexpectedly failed on empty advisories: %v", err) + } + if count != 0 { + t.Errorf("expected 0 converted advisories, got %d", count) + } +} + +func TestParseRepoFile_Formats(t *testing.T) { + tests := []struct { + name string + input string + want []string + wantErr bool + }{ + { + name: "json string array", + input: `["google/osv.dev", "https://github.com/torvalds/linux", "pallets/flask"]`, + want: []string{"google/osv.dev", "https://github.com/torvalds/linux", "pallets/flask"}, + }, + { + name: "json object array with repo and url keys", + input: `[{"repo": "google/osv.dev"}, {"url": "https://github.com/golang/go"}, {"name": "gin-gonic/gin"}]`, + want: []string{"google/osv.dev", "https://github.com/golang/go", "gin-gonic/gin"}, + }, + { + name: "json dict with repos list wrapper", + input: `{"repos": ["google/osv.dev", "facebook/react"]}`, + want: []string{"google/osv.dev", "facebook/react"}, + }, + { + name: "json dict with repositories object list wrapper", + input: `{"repositories": [{"repo": "google/osv.dev"}, {"url": "expressjs/express"}]}`, + want: []string{"google/osv.dev", "expressjs/express"}, + }, + { + name: "json map where keys are repositories", + input: `{"google/osv.dev": {"active": true}, "gin-gonic/gin": {"active": false}}`, + want: []string{"gin-gonic/gin", "google/osv.dev"}, + }, + { + name: "plain text with comments and empty lines", + input: ` +# Core repositories +google/osv.dev +https://github.com/torvalds/linux + +# Another one +pallets/flask +`, + want: []string{"google/osv.dev", "https://github.com/torvalds/linux", "pallets/flask"}, + }, + { + name: "empty input", + input: "", + want: nil, + }, + { + name: "whitespace only", + input: " \n\t\n ", + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseRepoFile([]byte(tc.input)) + if (err != nil) != tc.wantErr { + t.Fatalf("parseRepoFile() err = %v, wantErr = %v", err, tc.wantErr) + } + // For map keys, order may vary; sort for comparison if needed + if tc.name == "json map where keys are repositories" { + if len(got) != len(tc.want) { + t.Fatalf("got %v, want %v", got, tc.want) + } + for _, w := range tc.want { + if !slices.Contains(got, w) { + t.Errorf("missing expected repo %s in %v", w, got) + } + } + + return + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("parseRepoFile mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestReadRepoFile_Local(t *testing.T) { + tmpDir := t.TempDir() + filePath := filepath.Join(tmpDir, "repos.json") + content := `["google/osv.dev", "golang/go"]` + if err := os.WriteFile(filePath, []byte(content), 0644); err != nil { + t.Fatalf("failed writing test file: %v", err) + } + + ctx := context.Background() + repos, err := readRepoFile(ctx, filePath, nil) + if err != nil { + t.Fatalf("readRepoFile failed: %v", err) + } + + want := []string{"google/osv.dev", "golang/go"} + if diff := cmp.Diff(want, repos); diff != "" { + t.Errorf("readRepoFile mismatch (-want +got):\n%s", diff) + } +} + +func TestReadRepoFile_InvalidGCSPath(t *testing.T) { + ctx := context.Background() + // Malformed gs:// without object + _, err := readRepoFile(ctx, "gs://onlybucket", nil) + if err == nil { + t.Error("expected error for malformed GCS URI, got nil") + } + + // Empty object + _, err = readRepoFile(ctx, "gs://bucket/", nil) + if err == nil { + t.Error("expected error for empty GCS object path, got nil") + } +} + +func TestCollectRepos_Integration(t *testing.T) { + tmpDir := t.TempDir() + localFile := filepath.Join(tmpDir, "repos.txt") + if err := os.WriteFile(localFile, []byte("repo/from-file-1\nrepo/from-file-2\n"), 0644); err != nil { + t.Fatalf("failed writing local file: %v", err) + } + + ctx := context.Background() + repos, err := collectRepos(ctx, "repo/from-csv-1, repo/from-csv-2", localFile, "", []string{"repo/positional", "repo/from-csv-1"}, nil) + if err != nil { + t.Fatalf("collectRepos failed: %v", err) + } + + want := []string{ + "repo/from-csv-1", + "repo/from-csv-2", + "repo/from-file-1", + "repo/from-file-2", + "repo/positional", + } + + if diff := cmp.Diff(want, repos); diff != "" { + t.Errorf("collectRepos mismatch (-want +got):\n%s", diff) + } +} + +func TestWriteOSVRecord_EmptyID(t *testing.T) { + vuln := &vulns.Vulnerability{} + err := writeOSVRecord(vuln, "", nil, "") + if err == nil { + t.Error("expected error for empty vuln ID, got nil") + } +} + +func TestExtractRepoFromMap_Keys(t *testing.T) { + tests := []struct { + input map[string]any + want string + }{ + {input: map[string]any{"canonical_url": "https://github.com/org/repo"}, want: "https://github.com/org/repo"}, + {input: map[string]any{"git": "git@github.com:org/repo.git"}, want: "git@github.com:org/repo.git"}, + {input: map[string]any{"other": "value"}, want: ""}, + {input: map[string]any{"repo": 123}, want: ""}, + } + + for _, tc := range tests { + got := extractRepoFromMap(tc.input) + if got != tc.want { + t.Errorf("extractRepoFromMap(%v) = %q, want %q", tc.input, got, tc.want) + } + } +} diff --git a/vulnfeeds/cmd/converters/repository-ghsa/run_repository_ghsa.sh b/vulnfeeds/cmd/converters/repository-ghsa/run_repository_ghsa.sh new file mode 100755 index 00000000000..00cf321073e --- /dev/null +++ b/vulnfeeds/cmd/converters/repository-ghsa/run_repository_ghsa.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +echo "Commencing GHSA repository-specific advisory conversion run" + +NUM_WORKERS="${NUM_WORKERS:=8}" +OUTPUT_BUCKET="${OUTPUT_BUCKET:=osv-test-ghsa-repo-conversion}" +REPOS_FILE="${REPOS_FILE:=}" +REPOS_GCS_PATH="${REPOS_GCS_PATH:=}" +LOCAL_OUT_DIR="${LOCAL_OUT_DIR:=output}" +GCS_PREFIX="${GCS_PREFIX:=ghsa-repo-osv}" +GOOGLE_CLOUD_PROJECT="${GOOGLE_CLOUD_PROJECT:=}" +DATASTORE_JOB_ID="${DATASTORE_JOB_ID:=repository_ghsa_last_run}" +GITHUB_TOKEN="${GITHUB_TOKEN:=}" + +ARGS=( + "--workers=${NUM_WORKERS}" + "--out-dir=${LOCAL_OUT_DIR}" + "--output-bucket=${OUTPUT_BUCKET}" + "--gcs-prefix=${GCS_PREFIX}" + "--upload-to-gcs=true" +) + +if [[ -n "${REPOS_GCS_PATH}" ]]; then + ARGS+=("--repos-gcs-path=${REPOS_GCS_PATH}") +elif [[ -n "${REPOS_FILE}" ]]; then + ARGS+=("--repos-file=${REPOS_FILE}") +fi + +if [[ -n "${GOOGLE_CLOUD_PROJECT}" ]]; then + ARGS+=("--datastore-project=${GOOGLE_CLOUD_PROJECT}") + ARGS+=("--datastore-job-id=${DATASTORE_JOB_ID}") + ARGS+=("--save-last-run=true") +fi + +if [[ -n "${GITHUB_TOKEN}" ]]; then + ARGS+=("--github-token=${GITHUB_TOKEN}") +fi + +echo "Running repository-ghsa with arguments: ${ARGS[*]}" +exec /root/repository-ghsa "${ARGS[@]}" "$@" diff --git a/vulnfeeds/go.mod b/vulnfeeds/go.mod index fefede2b000..5117ff22143 100644 --- a/vulnfeeds/go.mod +++ b/vulnfeeds/go.mod @@ -4,6 +4,7 @@ go 1.27.0 require ( charm.land/lipgloss/v2 v2.0.6 + cloud.google.com/go/datastore v1.25.0 cloud.google.com/go/secretmanager v1.21.0 cloud.google.com/go/storage v1.66.0 github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 diff --git a/vulnfeeds/go.sum b/vulnfeeds/go.sum index 33e65e7cb4d..f256969ccf9 100644 --- a/vulnfeeds/go.sum +++ b/vulnfeeds/go.sum @@ -11,6 +11,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= +cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q=