From 5b6a3b0020899d1f5315aef29af36fe41055b21d Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Thu, 8 Oct 2026 06:25:39 +0200 Subject: [PATCH] ci: share gosec policy across local and CI scans --- .github/workflows/ci.yml | 51 ++-------- .github/workflows/pre-commit.yml | 22 +--- .pre-commit-config.yaml | 12 +-- Makefile | 7 +- scripts/gosec-hook.sh | 168 +++---------------------------- scripts/run-gosec-test.sh | 75 ++++++++++++++ scripts/run-gosec.sh | 65 ++++++++++++ 7 files changed, 170 insertions(+), 230 deletions(-) create mode 100644 scripts/run-gosec-test.sh create mode 100644 scripts/run-gosec.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ccd0799d..780658e59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -375,50 +375,17 @@ jobs: # Still requires checkout and Go setup to have succeeded. if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' run: | - # Install pinned gosec using the job's existing setup-go. - # The securego/gosec Docker action bundles its own Go toolchain which - # cannot satisfy the module's go directive, causing a toolchain mismatch. - go install github.com/securego/gosec/v2/cmd/gosec@v2.28.0 - # Scan each owned module independently. - set -e + set -euo pipefail status=0 - for mod in .; do - tag=$(echo "$mod" | tr './' '--' | sed 's/^-/root/') - out="$RUNNER_TEMP/gosec-${tag}.sarif" - echo "==> gosec in $mod" - if ! (cd "$mod" && gosec -fmt sarif -out "$out" ./...); then - echo "::warning::gosec exited non-zero in $mod (findings or a scan error)" - status=1 - fi - if [ ! -f "$out" ]; then - echo "::error::gosec produced no SARIF output for $mod" - status=1 - continue - fi - # Code scanning rejects a SARIF file whose runs share a category - # (github.blog changelog 2025-07-21), so give each module's run a - # unique automationDetails.id before merging. - jq --arg id "gosec-${tag}/" '.runs |= map(.automationDetails = {id: $id})' \ - "$out" > "$out.tmp" && mv "$out.tmp" "$out" - done - # Merge per-module SARIF runs into one file for the upload step. - # Only modules that actually produced a file are included; if - # gosec crashed before writing any of them, fail loud here rather - # than uploading an empty result silently. - shopt -s nullglob - sarif_files=("$RUNNER_TEMP"/gosec-*.sarif) - if [ "${#sarif_files[@]}" -eq 0 ]; then - echo "::error::no gosec SARIF output was produced by any module" >&2 - exit 1 + bash scripts/run-gosec-test.sh || status=$? + out="$RUNNER_TEMP/gosec-results.sarif" + bash scripts/run-gosec.sh --format sarif --output "$out" || status=$? + if ! jq -e '.version == "2.1.0" and (.runs | type == "array" and length > 0) and all(.runs[]; (.tool.driver | type == "object") and (.results | type == "array"))' "$out" >/dev/null; then + echo "::error::gosec produced missing or malformed SARIF" >&2 + exit 2 fi - # jq is preinstalled on the GitHub Ubuntu runner image (no new deps). - # Written to $RUNNER_TEMP, not the checkout root: never save working - # files in the repository root (repo coding guideline), and this - # file is purely a hand-off to the upload step below. - merged="$RUNNER_TEMP/gosec-results.sarif" - jq -s '{version: "2.1.0", "$schema": "https://json.schemastore.org/sarif-2.1.0.json", runs: [.[].runs[]]}' \ - "${sarif_files[@]}" > "$merged" - echo "Merged $(jq '.runs | length' "$merged") SARIF runs" + jq '.runs |= map(.automationDetails = {id: "gosec-root/"})' "$out" > "$out.tmp" + mv "$out.tmp" "$out" exit "$status" - name: Upload gosec results to GitHub Security diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 2e079028f..b5fddfa29 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -44,27 +44,13 @@ jobs: with: node-version: "24" - # Cache the installed tool binaries (gosec, gocyclo). Keyed on the - # pinned version strings so a tool-version bump still triggers a - # fresh install. Both binaries land in ~/go/bin which setup-go@v6 - # already adds to PATH. Restored BEFORE the install steps so the - # `if: cache-hit != 'true'` guards below can short-circuit them on - # cache-hit runs (the `go install` invocations cost ~3-5s each - # even when the module cache is warm; skipping them on cache-hit - # is worth the extra `if`). - - name: Cache Go-installed tools (gosec, gocyclo) + # Cache the pinned gocyclo binary before installation. + - name: Cache Go-installed tools (gocyclo) id: cache-go-tools uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/go/bin - key: go-tools-${{ runner.os }}-gosec-v2.28.0-gocyclo-v0.6.0 - - - name: Install gosec - if: steps.cache-go-tools.outputs.cache-hit != 'true' - # Pinned to the same version ci.yml's `securego/gosec` Action uses, - # so an upstream gosec release with rule changes can't silently - # downgrade the gate between the two workflows. - run: go install github.com/securego/gosec/v2/cmd/gosec@v2.28.0 + key: go-tools-${{ runner.os }}-gocyclo-v0.6.0 - name: Install gocyclo if: steps.cache-go-tools.outputs.cache-hit != 'true' @@ -160,7 +146,7 @@ jobs: - name: Run pre-commit # SKIP the local per-changed-package gosec hook in CI: --all-files # feeds every Go file at once, while the dedicated Security Scanning - # job remains the authoritative per-module gosec v2.28.0 gate. + # job runs the shared strict policy across the root module. # nick-fields/retry wraps the run with up to 3 attempts and a # 90-second wait so transient flakes do not require a manual rerun. uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d38176221..36e3ab7da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -157,16 +157,8 @@ repos: types: [file] - id: gosec - name: Go security scanner (per-module, per-changed-package) - # Scans only the Go packages that contain staged files, resolved to - # their owning module (root, pkg/, providers/aws, providers/azure, - # providers/gcp). Fast: never whole-repo, always per-changed-package. - # - # gosec v2.28.0 is auto-installed to - # ~/.cache/pre-commit-gosec/v2.28.0/gosec on first use. - # - # Exclusion rationale and flag list live in scripts/gosec-hook.sh. - # Keep in sync with the exclude= flags there. + name: Go security scanner (changed packages) + # Version, installation and strict policy are shared with Make and CI. entry: bash scripts/gosec-hook.sh language: system pass_filenames: true diff --git a/Makefile b/Makefile index 01d32092a..f5ade94f4 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ VERSION?=dev GOLANGCI_LINT_VERSION?=v2.10.1 -GOSEC_VERSION?=v2.28.0 GOCYCLO_VERSION?=v0.6.0 STATICCHECK_VERSION?=v0.7.0 @@ -85,8 +84,7 @@ complexity-report: security-scan: security-scan-go security-scan-go: - @command -v gosec >/dev/null || { echo "gosec not installed. Install: make install-dev-tools" >&2; exit 1; } - gosec -fmt=json -out=gosec-report.json -exclude=G101,G104,G115,G204,G301,G304,G402,G505 ./... + bash scripts/run-gosec.sh --format json --output gosec-report.json security-scan-snyk: @command -v snyk >/dev/null || { echo "snyk not installed. Install: npm install -g snyk" >&2; exit 1; } @@ -105,8 +103,7 @@ setup-git-secrets: install-dev-tools: @echo "Installing golangci-lint $(GOLANGCI_LINT_VERSION)..." @go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) - @echo "Installing gosec $(GOSEC_VERSION)..." - @go install github.com/securego/gosec/v2/cmd/gosec@$(GOSEC_VERSION) + @bash scripts/run-gosec.sh --install-only @echo "Installing staticcheck $(STATICCHECK_VERSION)..." @go install honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VERSION) @echo "Installing gocyclo $(GOCYCLO_VERSION)..." diff --git a/scripts/gosec-hook.sh b/scripts/gosec-hook.sh index 2334c76e3..35cf9d665 100755 --- a/scripts/gosec-hook.sh +++ b/scripts/gosec-hook.sh @@ -1,160 +1,18 @@ #!/usr/bin/env bash -# scripts/gosec-hook.sh -# -# Pre-commit gosec hook: runs gosec on only the Go packages that contain staged -# files, resolved to their owning module. Fast by design: never scans the whole -# repo; each commit triggers at most one gosec invocation per affected module. -# -# Version pin: gosec v2.28.0 (matches the CI pin in ci.yml, bumped by #1384). -# Installed on first use to ~/.cache/pre-commit-gosec/v2.28.0/gosec; never -# modifies the system-wide gosec binary. -# -# Called by pre-commit with pass_filenames: true and files: \.go$. -# Exits 0 when no staged .go files survive filtering (deleted / testdata). -# Exits 1 on any gosec finding; exits 2 on setup failure. -# -# Exclusion rationale (kept in sync with .pre-commit-config.yaml): -# G101 - variable names containing password/secret/token -> false positives -# G104 - unchecked errors -> covered by errcheck -# G115 - integer overflow -> safe conversions flagged -# G117 - unsafe pointer arithmetic -> vendor/generated code -# G118 - net/http serve without timeout -> timeouts set at handler level -# G122 - unsafe operations -> low-level helpers, pre-existing -# G204 - subprocess with variable -> CLI tool needs dynamic commands -# G301 - dir permissions > 0750 -> acceptable for dev tooling -# G304 - file path from variable -> CLI reads user-specified paths -# G402 - TLS MinVersion not set -> handled by cloud SDK defaults -# G505 - import of crypto/sha1 -> checksums, not security primitives -# G702 - TLS InsecureSkipVerify -> test helpers only, pre-existing -# G703 - unhandled defer error -> deferred close errors logged separately -# G705 - unhandled goroutine error -> pre-existing pattern -# G706 - ignored errors -> pre-existing; covered by go vet errcheck - set -euo pipefail -GOSEC_VERSION="2.28.0" -GOSEC_BIN="${HOME}/.cache/pre-commit-gosec/v${GOSEC_VERSION}/gosec" - -GOSEC_EXCLUDE="G101,G104,G115,G117,G118,G122,G204,G301,G304,G402,G505,G702,G703,G705,G706" - -# This repository has a single Go module, so every changed Go file belongs to -# the root module. -MODULE_DIRS="" - -# ---- helpers ---------------------------------------------------------------- - -ensure_gosec() { - local need_install=0 - if [[ -x "$GOSEC_BIN" ]]; then - # gosec built via `go install` embeds "dev" in -h regardless of tag; - # read the real module version from the binary's build info instead. - local installed_ver - installed_ver=$(go version -m "$GOSEC_BIN" 2>/dev/null \ - | awk '$1=="mod" && $2~/gosec/{print $3}') - # installed_ver is e.g. "v2.28.0"; compare against "v$GOSEC_VERSION". - if [[ "$installed_ver" != "v${GOSEC_VERSION}" ]]; then - echo "pre-commit/gosec: cached binary is ${installed_ver:-unknown}, need v${GOSEC_VERSION}; reinstalling" >&2 - need_install=1 - fi - else - need_install=1 - fi - - if [[ $need_install -eq 1 ]]; then - echo "pre-commit/gosec: installing gosec@v${GOSEC_VERSION} -> $(dirname "$GOSEC_BIN")" >&2 - mkdir -p "$(dirname "$GOSEC_BIN")" - GOBIN="$(dirname "$GOSEC_BIN")" go install \ - "github.com/securego/gosec/v2/cmd/gosec@v${GOSEC_VERSION}" || { - echo "pre-commit/gosec: install failed (is Go on PATH?)" >&2 - exit 2 - } - fi -} - -# Print the module root (relative to repo root) that owns a given relative file -# path, or empty string when the file belongs to the root module. -module_for() { - local f="$1" mod - for mod in $MODULE_DIRS; do - case "$f" in - "$mod"/*) printf '%s' "$mod"; return ;; - esac +packages=() +for file in "$@"; do + [[ -f "$file" ]] || continue + case "$file" in testdata/*|*/testdata/*) continue ;; esac + directory=$(dirname "$file") + package="./$directory" + [[ "$directory" != . ]] || package=. + duplicate=0 + for existing in "${packages[@]+"${packages[@]}"}"; do + if [[ "$existing" == "$package" ]]; then duplicate=1; break; fi done - printf '' -} - -# ---- main ------------------------------------------------------------------- - -[[ $# -eq 0 ]] && exit 0 - -REPO_ROOT="$(git rev-parse --show-toplevel)" - -# Filter: skip deleted files and files under testdata/. -live_files=() -for f in "$@"; do - [[ -f "$f" ]] || continue - case "$f" in - */testdata/*) continue ;; - testdata/*) continue ;; - esac - live_files+=("$f") -done - -[[ ${#live_files[@]} -eq 0 ]] && exit 0 - -ensure_gosec - -# Build "module|package" pairs from the live file list. -# Package path is relative to the owning module root, in ./pkg notation. -pairs_raw=() -for f in "${live_files[@]}"; do - mod=$(module_for "$f") - pkg_dir=$(dirname "$f") - - if [[ -n "$mod" ]]; then - # Strip the module prefix (plus the separating slash). - rel="${pkg_dir:$((${#mod}+1))}" - [[ -z "$rel" ]] && rel="." # file sits directly in the module root - else - rel="$pkg_dir" # root module; pkg_dir is already relative - fi - - # Normalise to go-tool notation. - if [[ "$rel" == "." ]]; then - pkg="." - else - pkg="./$rel" - fi - - pairs_raw+=("${mod}|${pkg}") + [[ "$duplicate" -eq 1 ]] || packages+=("$package") done - -# Deduplicate. -pairs_sorted=$(printf '%s\n' "${pairs_raw[@]}" | sort -u) - -# Enumerate unique module roots. -mods=$(printf '%s\n' "$pairs_sorted" | cut -d'|' -f1 | sort -u) - -fail=0 -while IFS= read -r mod; do - pkgs=$(printf '%s\n' "$pairs_sorted" \ - | awk -F'|' -v m="$mod" '$1==m{print $2}' \ - | tr '\n' ' ') - mod_dir="${REPO_ROOT}${mod:+/${mod}}" - - echo "gosec [${mod:-.}]: scanning package(s): $pkgs" >&2 - - # pkgs intentionally unquoted: space-separated package paths, no glob chars. - # shellcheck disable=SC2086 - if ! (cd "$mod_dir" && "$GOSEC_BIN" \ - -quiet \ - -exclude-dir=.legacy \ - -exclude-dir=.dev-notes \ - -exclude-dir=vendor \ - "-exclude=${GOSEC_EXCLUDE}" \ - $pkgs); then - fail=1 - fi -done <<< "$mods" - -exit $fail +[[ ${#packages[@]} -gt 0 ]] || exit 0 +exec bash "$(dirname "${BASH_SOURCE[0]}")/run-gosec.sh" -- "${packages[@]}" diff --git a/scripts/run-gosec-test.sh b/scripts/run-gosec-test.sh new file mode 100644 index 000000000..f4e0198a4 --- /dev/null +++ b/scripts/run-gosec-test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +scratch=$(mktemp -d) +export XDG_CACHE_HOME="$scratch/cache" +export GOSEC_TEST_DIR="$scratch" +mkdir -p "$scratch/bin" "$scratch/repo/scripts" "$scratch/repo/pkg space" "$scratch/repo/pkg*glob" "$scratch/repo/testdata" +cp "$root/scripts/run-gosec.sh" "$root/scripts/gosec-hook.sh" "$scratch/repo/scripts/" +cat > "$scratch/bin/go" <<'GO' +#!/usr/bin/env bash +set -euo pipefail +if [[ "$1" == version ]]; then + version=$(cat "$GOSEC_TEST_DIR/version") + printf 'mod\t%s\t%s\n' "${GOSEC_TEST_MODULE:-github.com/securego/gosec/v2}" "$version" +else + [[ "${GOSEC_TEST_INSTALL_FAIL:-0}" == 0 ]] || exit 1 + printf '%s\n' "${2##*@}" > "$GOSEC_TEST_DIR/version" + cp "$GOSEC_TEST_DIR/scanner" "$GOBIN/gosec" + chmod +x "$GOBIN/gosec" + echo installed >> "$GOSEC_TEST_DIR/installs" +fi +GO +cat > "$scratch/scanner" <<'SCANNER' +#!/usr/bin/env bash +set -euo pipefail +printf '<%s>\n' "$@" > "$GOSEC_TEST_DIR/argv" +while [[ $# -gt 0 ]]; do + if [[ "$1" == -out && "${GOSEC_TEST_REPORT:-1}" == 1 ]]; then + printf '{"Issues":[]}\n' > "$2" + fi + shift +done +exit "${GOSEC_TEST_STATUS:-0}" +SCANNER +chmod +x "$scratch/bin/go" +export PATH="$scratch/bin:$PATH" +cd "$scratch/repo" +expect_status() { + local expected="$1" actual=0 + shift + "$@" > "$scratch/stdout" 2> "$scratch/stderr" || actual=$? + [[ "$actual" == "$expected" ]] || { cat "$scratch/stderr" >&2; echo "expected $expected, got $actual" >&2; exit 1; } +} +expect_status 0 bash scripts/gosec-hook.sh missing.go testdata/ignored.go +[[ ! -f "$scratch/installs" ]] +touch "pkg space/a.go" "pkg space/b.go" "pkg*glob/a.go" testdata/ignored.go +expect_status 0 bash scripts/gosec-hook.sh "pkg space/a.go" "pkg space/b.go" "pkg*glob/a.go" testdata/ignored.go missing.go +printf '<%s>\n' -fmt text -- './pkg space' './pkg*glob' > "$scratch/expected" +diff -u "$scratch/expected" "$scratch/argv" +expect_status 0 bash scripts/run-gosec.sh -- -exclude=G304 +printf '<%s>\n' -fmt text -- -exclude=G304 > "$scratch/expected" +diff -u "$scratch/expected" "$scratch/argv" +expect_status 0 bash scripts/run-gosec.sh --format json --output "$scratch/report.json" +[[ -s "$scratch/report.json" && $(wc -l < "$scratch/installs") -eq 1 ]] +expect_status 0 bash scripts/run-gosec.sh --install-only +export GOSEC_TEST_STATUS=7 +expect_status 7 bash scripts/run-gosec.sh --format json --output "$scratch/finding.json" +[[ -s "$scratch/finding.json" ]] +export GOSEC_TEST_REPORT=0 +expect_status 2 bash scripts/run-gosec.sh --format json --output "$scratch/report.json" +export GOSEC_TEST_STATUS=0 GOSEC_TEST_REPORT=1 +expect_status 2 bash scripts/run-gosec.sh --format sarif +expect_status 2 bash scripts/run-gosec.sh --quiet +expect_status 2 bash scripts/run-gosec.sh --format json --output "$scratch/missing/report.json" +expect_status 2 bash scripts/run-gosec.sh --format json --output testdata +printf 'wrong\n' > "$scratch/version" +export GOSEC_TEST_INSTALL_FAIL=1 +expect_status 2 bash scripts/run-gosec.sh --install-only +export GOSEC_TEST_INSTALL_FAIL=0 +expect_status 0 bash scripts/run-gosec.sh --install-only +[[ $(wc -l < "$scratch/installs") -eq 2 ]] +export GOSEC_TEST_MODULE=github.com/unrelated/gosec/v2 +expect_status 2 bash scripts/run-gosec.sh --install-only +echo "gosec contract checks passed ($scratch)" diff --git a/scripts/run-gosec.sh b/scripts/run-gosec.sh new file mode 100644 index 000000000..1df217094 --- /dev/null +++ b/scripts/run-gosec.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly GOSEC_VERSION="v2.28.0" +readonly GOSEC_MODULE="github.com/securego/gosec/v2" +readonly GOSEC_BIN="${XDG_CACHE_HOME:-${HOME}/.cache}/pre-commit-gosec/${GOSEC_VERSION}/gosec" +format=text +output="" +install_only=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --format|--output) + [[ $# -ge 2 ]] || { echo "gosec: $1 needs a value" >&2; exit 2; } + if [[ "$1" == --format ]]; then format="$2"; else output="$2"; fi + shift 2 ;; + --install-only) install_only=1; shift ;; + --) shift; break ;; + *) echo "gosec: unsupported option: $1" >&2; exit 2 ;; + esac +done +case "$format" in text|json|sarif) ;; *) echo "gosec: unsupported format: $format" >&2; exit 2 ;; esac +if [[ "$format" != text && -z "$output" ]]; then + echo "gosec: $format requires --output" >&2 + exit 2 +fi + +installed_version() { + go version -m "$GOSEC_BIN" 2>/dev/null | awk -v module="$GOSEC_MODULE" '$1 == "mod" && $2 == module { print $3 }' +} +if [[ ! -x "$GOSEC_BIN" ]] || [[ "$(installed_version)" != "$GOSEC_VERSION" ]]; then + echo "gosec: installing ${GOSEC_MODULE}/cmd/gosec@${GOSEC_VERSION}" >&2 + if ! mkdir -p "$(dirname "$GOSEC_BIN")" || ! GOBIN="$(dirname "$GOSEC_BIN")" go install "${GOSEC_MODULE}/cmd/gosec@${GOSEC_VERSION}"; then + echo "gosec: install failed" >&2 + exit 2 + fi +fi +if [[ ! -x "$GOSEC_BIN" ]] || [[ "$(installed_version)" != "$GOSEC_VERSION" ]]; then + echo "gosec: installed binary does not match ${GOSEC_VERSION}" >&2 + exit 2 +fi +[[ "$install_only" -eq 0 ]] || exit 0 +cd "$(dirname "${BASH_SOURCE[0]}")/.." +[[ $# -gt 0 ]] || set -- ./... +args=(-fmt "$format") +if [[ -n "$output" ]]; then + [[ ! -d "$output" ]] || { echo "gosec: output is a directory: $output" >&2; exit 2; } + if ! report=$(mktemp "${output}.XXXXXX"); then + echo "gosec: cannot create report beside $output" >&2 + exit 2 + fi + args+=(-out "$report") +fi +status=0 +"$GOSEC_BIN" "${args[@]}" -- "$@" || status=$? +if [[ -n "$output" ]]; then + if [[ ! -s "$report" ]]; then + echo "gosec: scanner produced no report" >&2 + exit 2 + fi + if ! mv "$report" "$output"; then + echo "gosec: cannot publish report to $output" >&2 + exit 2 + fi +fi +exit "$status"