diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6cccc6d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: + pull_request: + branches: [master] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: tests and coverage + runs-on: cx-public-ubuntu-x64 + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 22 + # Runner cannot reach registry.npmjs.org; install through the echohq registry, as in test_action.yaml + - name: Install dependencies through echohq registry + env: + ECHO_LIBRARIES_ACCESS_KEY: ${{ secrets.ECHO_LIBRARIES_ACCESS_KEY }} + run: | + NPMRC="${RUNNER_TEMP}/npmrc" + umask 077 + printf 'registry=https://npm.echohq.com/\n//npm.echohq.com/:_authToken=%s\n' "${ECHO_LIBRARIES_ACCESS_KEY}" > "${NPMRC}" + npm ci --ignore-scripts --userconfig "${NPMRC}" + rm -f "${NPMRC}" + - name: Run tests with coverage + run: npm run test:coverage + + shellcheck: + name: shellcheck entrypoint + runs-on: cx-public-ubuntu-x64 + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # entrypoint.sh runs under busybox ash in the action image + - name: Lint entrypoint.sh + run: docker run --rm -v "${PWD}:/mnt:ro" koalaman/shellcheck@sha256:2097951f02e735b613f4a34de20c40f937a6c8f18ecb170612c88c34517221fb --shell=busybox --severity=error entrypoint.sh # v0.10.0 diff --git a/.github/workflows/coverage-badge.yml b/.github/workflows/coverage-badge.yml new file mode 100644 index 0000000..4dd5f45 --- /dev/null +++ b/.github/workflows/coverage-badge.yml @@ -0,0 +1,72 @@ +name: coverage-badge + +on: + push: + branches: [master] + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + coverage: + name: generate-coverage + runs-on: cx-public-ubuntu-x64 + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 22 + # Runner cannot reach registry.npmjs.org; install through the echohq registry, as in test_action.yaml + - name: Install dependencies through echohq registry + env: + ECHO_LIBRARIES_ACCESS_KEY: ${{ secrets.ECHO_LIBRARIES_ACCESS_KEY }} + run: | + NPMRC="${RUNNER_TEMP}/npmrc" + umask 077 + printf 'registry=https://npm.echohq.com/\n//npm.echohq.com/:_authToken=%s\n' "${ECHO_LIBRARIES_ACCESS_KEY}" > "${NPMRC}" + npm ci --ignore-scripts --userconfig "${NPMRC}" + rm -f "${NPMRC}" + - name: Run tests with coverage + run: npm run test:coverage + - name: Generate badge + run: node scripts/coverage-badge.js coverage/lcov.info coverage/badge.json + - name: Upload coverage badge Artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-badge + path: coverage/badge.json + + publish: + name: publish-coverage + needs: coverage + runs-on: cx-public-ubuntu-x64 + timeout-minutes: 5 + permissions: + contents: write # for git push to the badges branch + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: badges + persist-credentials: true + - name: Download Badge json + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: coverage-badge + path: latest-coverage + - name: Push badge to the badges branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + mv latest-coverage/badge.json coverage.json + git add coverage.json + if ! git status | grep "nothing to commit"; then + git commit -m 'chore(tests): updating test coverage badge' + git push origin badges + fi diff --git a/.gitignore b/.gitignore index 55371e5..71e0ed0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules -.vscode \ No newline at end of file +.vscode +coverage diff --git a/README.md b/README.md index 02de873..41a0c93 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![License: GPL-3.0](https://img.shields.io/badge/License-GPL3.0-yellow.svg)](https://www.gnu.org/licenses) [![Latest Release](https://img.shields.io/github/v/release/checkmarx/kics-github-action)](https://github.com/checkmarx/kics-github-action/releases) [![Open Issues](https://img.shields.io/github/issues-raw/checkmarx/kics-github-action)](https://github.com/checkmarx/kics-github-action/issues) +[![Coverage](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2FCheckmarx%2Fkics-github-action%2Fbadges%2Fcoverage.json)](https://github.com/Checkmarx/kics-github-action/actions/workflows/coverage-badge.yml) - [KICS GitHub Action](#kics-github-action) - [Integrate KICS into your GitHub workflows](#integrate-kics-into-your-github-workflows) @@ -20,6 +21,7 @@ - [Uploading SARIF report](#uploading-sarif-report) - [Using configuration file](#using-configuration-file) - [How To Contribute](#how-to-contribute) + - [Running the tests](#running-the-tests) - [License](#license) ## Integrate KICS into your GitHub workflows @@ -448,6 +450,25 @@ jobs: We welcome [issues](https://github.com/checkmarx/kics-github-action/issues) to and [pull requests](https://github.com/checkmarx/kics-github-action/pulls) against this repository! +### Running the tests + +Requires Node.js 22 or newer. + +```sh +npm ci +npm test # unit and end-to-end tests +npm run test:coverage # same, plus a coverage report and the coverage gate used by CI +``` + +Tests live under `test/`: + +- `test/unit` - the PR comment and job summary logic against a fake GitHub API +- `test/e2e` - the action run as a workflow would run it: `src/main.js` as a process, `entrypoint.sh` against a fake `kics`, and the `action.yml` contract +- `test/helpers` and `test/fixtures` - the fake GitHub API, KICS report builders and process runners + +Tests marked with `knownBug` document known bugs: they are reported as `todo` and are expected to fail until the bug is fixed. +They are skipped in the coverage run, so coverage only counts verified behaviour. + # License KICS Github Action diff --git a/package-lock.json b/package-lock.json index a6131cf..faf8ea9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,15 +10,11 @@ "license": "GNU GENERAL PUBLIC LICENSE", "dependencies": { "@actions/core": "^1.10.1", - "@actions/exec": "^1.1.0", "@actions/github": "^5.0.0", "@actions/io": "^1.1.1", - "@actions/tool-cache": "^2.0.1", - "moment": "^2.29.4", - "uuid": "^8.3.2" + "moment": "^2.29.4" }, "devDependencies": { - "@types/uuid": "^8.3.4", "@vercel/ncc": "^0.36.1", "prettier": "^2.4.1" } @@ -40,14 +36,6 @@ "tunnel": "^0.0.6" } }, - "node_modules/@actions/exec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.0.tgz", - "integrity": "sha512-LImpN9AY0J1R1mEYJjVJfSZWU4zYOlEcwSTgPve1rFQqK5AwrEs6uWW5Rv70gbDIQIAUwI86z6B+9mPK4w9Sbg==", - "dependencies": { - "@actions/io": "^1.0.1" - } - }, "node_modules/@actions/github": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.0.0.tgz", @@ -72,53 +60,6 @@ "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.1.tgz", "integrity": "sha512-Qi4JoKXjmE0O67wAOH6y0n26QXhMKMFo7GD/4IXNVcrtLjUlGjGuVys6pQgwF3ArfGTQu0XpqaNr0YhED2RaRA==" }, - "node_modules/@actions/tool-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz", - "integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==", - "dependencies": { - "@actions/core": "^1.2.6", - "@actions/exec": "^1.0.0", - "@actions/http-client": "^2.0.1", - "@actions/io": "^1.1.1", - "semver": "^6.1.0", - "uuid": "^3.3.2" - } - }, - "node_modules/@actions/tool-cache/node_modules/@actions/http-client": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz", - "integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==", - "dependencies": { - "tunnel": "^0.0.6", - "undici": "^5.25.4" - } - }, - "node_modules/@actions/tool-cache/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/@actions/tool-cache/node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/@fastify/busboy": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", - "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", - "engines": { - "node": ">=14" - } - }, "node_modules/@octokit/auth-token": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", @@ -220,12 +161,6 @@ "@octokit/openapi-types": "^11.2.0" } }, - "node_modules/@types/uuid": { - "version": "8.3.4", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", - "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", - "dev": true - }, "node_modules/@vercel/ncc": { "version": "0.36.1", "resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.1.tgz", @@ -313,17 +248,6 @@ "node": ">=0.6.11 <=0.7.0 || >=0.7.3" } }, - "node_modules/undici": { - "version": "5.28.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", - "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", - "dependencies": { - "@fastify/busboy": "^2.0.0" - }, - "engines": { - "node": ">=14.0" - } - }, "node_modules/universal-user-agent": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", @@ -377,14 +301,6 @@ } } }, - "@actions/exec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.0.tgz", - "integrity": "sha512-LImpN9AY0J1R1mEYJjVJfSZWU4zYOlEcwSTgPve1rFQqK5AwrEs6uWW5Rv70gbDIQIAUwI86z6B+9mPK4w9Sbg==", - "requires": { - "@actions/io": "^1.0.1" - } - }, "@actions/github": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.0.0.tgz", @@ -409,45 +325,6 @@ "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.1.tgz", "integrity": "sha512-Qi4JoKXjmE0O67wAOH6y0n26QXhMKMFo7GD/4IXNVcrtLjUlGjGuVys6pQgwF3ArfGTQu0XpqaNr0YhED2RaRA==" }, - "@actions/tool-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz", - "integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==", - "requires": { - "@actions/core": "^1.2.6", - "@actions/exec": "^1.0.0", - "@actions/http-client": "^2.0.1", - "@actions/io": "^1.1.1", - "semver": "^6.1.0", - "uuid": "^3.3.2" - }, - "dependencies": { - "@actions/http-client": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz", - "integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==", - "requires": { - "tunnel": "^0.0.6", - "undici": "^5.25.4" - } - }, - "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" - }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, - "@fastify/busboy": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", - "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==" - }, "@octokit/auth-token": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", @@ -543,12 +420,6 @@ "@octokit/openapi-types": "^11.2.0" } }, - "@types/uuid": { - "version": "8.3.4", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", - "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", - "dev": true - }, "@vercel/ncc": { "version": "0.36.1", "resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.1.tgz", @@ -607,14 +478,6 @@ "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, - "undici": { - "version": "5.28.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", - "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", - "requires": { - "@fastify/busboy": "^2.0.0" - } - }, "universal-user-agent": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", diff --git a/package.json b/package.json index 7b04ac0..249bef1 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,9 @@ "main": "index.js", "scripts": { "build": "ncc build src/main.js", - "format": "prettier --write **/*.ts" + "format": "prettier --write **/*.ts", + "test": "node --test \"test/**/*.test.js\"", + "test:coverage": "mkdir -p coverage && KNOWN_BUGS=skip node --test --experimental-test-coverage --test-coverage-lines=90 --test-coverage-branches=85 --test-coverage-functions=90 --test-coverage-include=\"src/**/*.js\" --test-reporter=spec --test-reporter-destination=stdout --test-reporter=lcov --test-reporter-destination=coverage/lcov.info \"test/**/*.test.js\"" }, "repository": { "type": "git", @@ -20,16 +22,12 @@ "homepage": "https://github.com/Checkmarx/kics-github-action#readme", "dependencies": { "@actions/core": "^1.10.1", - "@actions/exec": "^1.1.0", "@actions/github": "^5.0.0", "@actions/io": "^1.1.1", - "@actions/tool-cache": "^2.0.1", - "moment": "^2.29.4", - "uuid": "^8.3.2" + "moment": "^2.29.4" }, "devDependencies": { "@vercel/ncc": "^0.36.1", - "prettier": "^2.4.1", - "@types/uuid": "^8.3.4" + "prettier": "^2.4.1" } } diff --git a/scripts/coverage-badge.js b/scripts/coverage-badge.js new file mode 100644 index 0000000..cff211d --- /dev/null +++ b/scripts/coverage-badge.js @@ -0,0 +1,40 @@ +'use strict' + +const fs = require('node:fs') + +function lineCoverage(lcov) { + let found = 0 + let hit = 0 + for (const line of lcov.split('\n')) { + if (line.startsWith('LF:')) found += Number(line.slice(3)) + else if (line.startsWith('LH:')) hit += Number(line.slice(3)) + } + if (found === 0) throw new Error('lcov report contains no instrumented lines') + return Math.round((hit / found) * 1000) / 10 +} + +function colorFor(percent) { + if (percent >= 90) return 'brightgreen' + if (percent >= 80) return 'green' + if (percent >= 70) return 'yellowgreen' + if (percent >= 60) return 'yellow' + if (percent >= 50) return 'orange' + return 'red' +} + +function badge(percent) { + return { schemaVersion: 1, label: 'coverage', message: `${percent}%`, color: colorFor(percent) } +} + +if (require.main === module) { + const [lcovPath, outPath] = process.argv.slice(2) + if (!lcovPath || !outPath) { + console.error('usage: coverage-badge.js ') + process.exit(2) + } + const percent = lineCoverage(fs.readFileSync(lcovPath, 'utf8')) + fs.writeFileSync(outPath, JSON.stringify(badge(percent)) + '\n') + console.log(percent) +} + +module.exports = { lineCoverage, colorFor, badge } diff --git a/test/e2e/action-contract.test.js b/test/e2e/action-contract.test.js new file mode 100644 index 0000000..910ef31 --- /dev/null +++ b/test/e2e/action-contract.test.js @@ -0,0 +1,64 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') +const { ROOT, loadManifest, runnerInputEnv } = require('../helpers/action-manifest') +const { knownBug } = require('../helpers/known-bug') + +const { inputs, env } = loadManifest() +const read = (file) => fs.readFileSync(path.join(ROOT, file), 'utf8') +const reads = (source, pattern) => [...source.matchAll(pattern)].map((m) => m[1]) +const consumed = new Set([ + ...reads(read('entrypoint.sh'), /\$\{?(INPUT_[A-Z_]+)/g), + ...reads(read('src/main.js'), /process\.env\.(INPUT_[A-Z_]+)/g), +]) +consumed.delete('INPUT_PARAM') // entrypoint.sh's own "-p " variable, not an action input + +const KNOWN_UNUSED_INPUTS = { type: 'declared in action.yml but entrypoint.sh only reads platform_type, so `type:` is silently ignored' } + +test('action.yml parsing assumptions hold', async (t) => { + await t.test('declares inputs and the runs.env mappings the tests rely on', () => { + assert.ok(Object.keys(inputs).length > 20, 'inputs section parsed') + assert.equal(env.INPUT_EXCLUDED_COLUMNS_FOR_COMMENTS_WITH_QUERIES, 'excluded_column_for_comments_with_queries') + }) + + await t.test('path is the only required input', () => { + assert.deepEqual(Object.keys(inputs).filter((n) => inputs[n].required === 'true'), ['path']) + }) + + await t.test('runs as a docker action built from the repository Dockerfile', () => { + assert.match(read('action.yml'), /^runs:\n {2}using: "docker"\n {2}image: Dockerfile$/m) + }) +}) + +test('every INPUT_* variable the code reads is provided by the runner', async (t) => { + const provided = new Set(Object.keys(runnerInputEnv())) + for (const name of [...consumed].sort()) { + await t.test(name, () => assert.ok(provided.has(name), `${name} is read but no input or runs.env entry in action.yml provides it`)) + } +}) + +test('every declared input is used by the code', async (t) => { + const viaEnvMapping = new Set(Object.entries(env).filter(([key]) => consumed.has(key)).map(([, input]) => input)) + for (const name of Object.keys(inputs)) { + const used = consumed.has(`INPUT_${name.toUpperCase()}`) || viaEnvMapping.has(name) + const options = name in KNOWN_UNUSED_INPUTS ? knownBug(KNOWN_UNUSED_INPUTS[name]) : {} + await t.test(name, options, () => assert.ok(used, `input "${name}" is declared but never read`)) + } +}) + +test('defaults a workflow gets without configuration', async (t) => { + await t.test('annotations on; comments, job summary and queries table off', () => { + const defaults = runnerInputEnv() + assert.equal(defaults.INPUT_ENABLE_ANNOTATIONS, 'true') + assert.equal(defaults.INPUT_ENABLE_COMMENTS, 'false') + assert.equal(defaults.INPUT_ENABLE_JOBS_SUMMARY, 'false') + assert.equal(defaults.INPUT_COMMENTS_WITH_QUERIES, 'false') + }) + + await t.test('the token defaults to the workflow GITHUB_TOKEN', () => { + assert.match(read('action.yml'), /token:[\s\S]*?default: \$\{\{\s*github\.token\s*\}\}/) + }) +}) diff --git a/test/e2e/action.test.js b/test/e2e/action.test.js new file mode 100644 index 0000000..c407b31 --- /dev/null +++ b/test/e2e/action.test.js @@ -0,0 +1,215 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const { runAction } = require('../helpers/run-action') +const { startFakeGitHub, humanComment } = require('../helpers/fake-github') +const { kicsResults, query, finding } = require('../helpers/fixtures') +const { knownBug } = require('../helpers/known-bug') + +const KICS_FOUND_RESULTS = '50' + +const SILENT_FAILURE = + 'main() swallows every exception, so a KICS failure is never reported when GitHub or results.json cannot be used' + +test('pull request comment', async (t) => { + await t.test('is not posted unless enable_comments is on (the default)', async (t) => { + const fake = await startFakeGitHub(t) + const run = await runAction(t, { results: kicsResults(), apiUrl: fake.url }) + assert.equal(run.code, 0) + assert.deepEqual(fake.calls(), [], 'no API traffic at all') + }) + + await t.test('reports the scan on the pull request when enable_comments is on', async (t) => { + const fake = await startFakeGitHub(t) + const run = await runAction(t, { results: kicsResults(), inputs: { enable_comments: 'true' }, apiUrl: fake.url, prNumber: 7 }) + + assert.equal(run.code, 0, run.stdout + run.stderr) + assert.deepEqual(fake.calls(), ['GET /repos/o/r/issues/7/comments', 'POST /repos/o/r/issues/7/comments']) + const body = fake.comments[0].body + assert.match(body, /KICS version: v2\.1\.20/) + assert.match(body, /\| HIGH \| 1 \|/) + assert.match(body, /\| TOTAL \| 4 \|/) + assert.ok(!body.includes('### Queries Results'), 'queries are opt-in') + }) + + await t.test('updates its own comment when the PR is scanned again', async (t) => { + const fake = await startFakeGitHub(t, { comments: [humanComment(1), humanComment(2)] }) + const inputs = { enable_comments: 'true' } + + await runAction(t, { results: kicsResults(), inputs, apiUrl: fake.url }) + await runAction(t, { results: kicsResults({ kics_version: 'v9.9.9' }), inputs, apiUrl: fake.url }) + + const kicsComments = fake.comments.filter((c) => c.body.startsWith('![kics-logo](')) + assert.equal(kicsComments.length, 1) + assert.match(kicsComments[0].body, /v9\.9\.9/) + }) + + await t.test('includes the queries table with comments_with_queries, minus the excluded columns', async (t) => { + const fake = await startFakeGitHub(t) + await runAction(t, { + results: kicsResults(), + inputs: { enable_comments: 'true', comments_with_queries: 'true', excluded_column_for_comments_with_queries: 'similarity_id,search_line' }, + apiUrl: fake.url, + }) + + const body = fake.comments[0].body + assert.match(body, /### Queries Results/) + assert.match(body, /\[S3 Bucket Without Versioning\]\(https:\/\/docs\.kics\.io/) + assert.ok(!body.includes('Similarity Id')) + assert.ok(!body.includes('Search Line')) + assert.match(body, /\| Severity/, 'non-excluded columns remain') + }) + + await t.test('uses the default excluded columns from action.yml when none are given', async (t) => { + const fake = await startFakeGitHub(t) + await runAction(t, { results: kicsResults(), inputs: { enable_comments: 'true', comments_with_queries: 'true' }, apiUrl: fake.url }) + const body = fake.comments[0].body + for (const hidden of ['Description Id', 'Similarity Id', 'Search Line', 'Search Value']) assert.ok(!body.includes(hidden), hidden) + }) + + await t.test('input values are case-insensitive booleans', async (t) => { + const fake = await startFakeGitHub(t) + await runAction(t, { results: kicsResults(), inputs: { enable_comments: 'TRUE', comments_with_queries: 'True' }, apiUrl: fake.url }) + assert.match(fake.comments[0].body, /### Queries Results/) + }) + + await t.test('is skipped without failing the step when not running for a pull request', knownBug('on push events prNumber is "", so the action calls /issues//comments, gets a 404 and swallows it'), async (t) => { + const fake = await startFakeGitHub(t) + const run = await runAction(t, { results: kicsResults(), inputs: { enable_comments: 'true' }, apiUrl: fake.url, event: 'push' }) + assert.equal(run.code, 0) + assert.deepEqual(fake.calls(), [], 'there is no PR to talk to') + }) +}) + +test('annotations', async (t) => { + await t.test('one warning per finding, pointing at file and line', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { enable_annotations: 'true' } }) + + assert.equal(run.warnings.length, 4) + assert.deepEqual(run.warnings[0], { + message: 'S3 bucket should have versioning enabled', + properties: { file: 'test/samples/positive1.tf', line: '12', endLine: '12', title: '[MEDIUM] S3 Bucket Without Versioning' }, + }) + assert.ok(run.warnings.some((w) => w.properties.file === 'test/samples/positive2.tf' && w.properties.line === '30' && w.properties.title.startsWith('[HIGH]'))) + }) + + await t.test('a query that hits several files is annotated on each of them', async (t) => { + const files = ['a.tf', 'b.tf', 'c.tf'].map((file_name, i) => finding({ file_name, line: i + 1 })) + const run = await runAction(t, { results: kicsResults({ queries: [query({ files })] }), inputs: { enable_annotations: 'true' } }) + assert.deepEqual(run.warnings.map((w) => [w.properties.file, w.properties.line]), [['a.tf', '1'], ['b.tf', '2'], ['c.tf', '3']]) + }) + + await t.test('a clean scan produces no annotations', async (t) => { + const run = await runAction(t, { results: kicsResults({ queries: [] }), inputs: { enable_annotations: 'true' } }) + assert.equal(run.warnings.length, 0) + assert.equal(run.code, 0) + }) + + await t.test('enable_annotations: false produces none', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { enable_annotations: 'false' } }) + assert.equal(run.warnings.length, 0) + }) + + await t.test('annotates by default as declared in action.yml', async (t) => { + const run = await runAction(t, { results: kicsResults() }) + assert.equal(run.warnings.length, 4) + }) +}) + +test('job summary', async (t) => { + await t.test('is written when enable_jobs_summary is on', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { enable_jobs_summary: 'true', comments_with_queries: 'true' } }) + assert.match(run.summary, /KICS version: v2\.1\.20/) + assert.match(run.summary, /\| TOTAL \| 4 \|/) + assert.match(run.summary, /### Queries Results/) + }) + + await t.test('is left empty by default', async (t) => { + const run = await runAction(t, { results: kicsResults() }) + assert.equal(run.summary, '') + }) + + await t.test('works without any pull request or API access (push events)', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { enable_jobs_summary: 'true' }, event: 'push' }) + assert.equal(run.code, 0) + assert.match(run.summary, /TOTAL/) + }) +}) + +test('workflow status follows the KICS exit code', async (t) => { + await t.test('passes when KICS exits 0', async (t) => { + const run = await runAction(t, { results: kicsResults({ queries: [] }), kicsExitCode: '0' }) + assert.equal(run.code, 0) + assert.deepEqual(run.errors, []) + assert.match(run.stdout, /KICS scan status code: 0/) + }) + + await t.test('fails with the KICS exit code when KICS reports results', async (t) => { + const run = await runAction(t, { results: kicsResults(), kicsExitCode: KICS_FOUND_RESULTS }) + assert.equal(run.code, 1) + assert.deepEqual(run.errors.map((e) => e.message), ['KICS scan failed with exit code 50']) + }) + + await t.test('still publishes the report before failing the step', async (t) => { + const fake = await startFakeGitHub(t) + const run = await runAction(t, { + results: kicsResults(), + kicsExitCode: KICS_FOUND_RESULTS, + inputs: { enable_comments: 'true', enable_jobs_summary: 'true' }, + apiUrl: fake.url, + }) + assert.equal(run.code, 1) + assert.equal(fake.comments.length, 1, 'PR comment posted') + assert.match(run.summary, /TOTAL/, 'job summary written') + assert.equal(run.warnings.length, 4, 'annotations emitted') + }) + + await t.test('a clean scan still passes when commenting is forbidden, as on fork pull requests', async (t) => { + const fake = await startFakeGitHub(t, { failWith: 403 }) + const run = await runAction(t, { results: kicsResults({ queries: [] }), kicsExitCode: '0', inputs: { enable_comments: 'true' }, apiUrl: fake.url }) + assert.equal(run.code, 0) + }) + + await t.test('fails when KICS fails and the PR comment cannot be posted', knownBug(SILENT_FAILURE), async (t) => { + const fake = await startFakeGitHub(t, { failWith: 500 }) + const run = await runAction(t, { results: kicsResults(), kicsExitCode: KICS_FOUND_RESULTS, inputs: { enable_comments: 'true' }, apiUrl: fake.url }) + assert.equal(run.code, 1, 'a GitHub API outage must not turn a failed scan into a green build') + }) + + await t.test('fails when KICS failed and wrote no results.json', knownBug(SILENT_FAILURE), async (t) => { + const run = await runAction(t, { kicsExitCode: '126' }) + assert.equal(run.code, 1, run.stdout + run.stderr) + assert.ok(run.errors.length > 0) + }) + + await t.test('reports a missing results.json instead of passing silently', knownBug(SILENT_FAILURE), async (t) => { + const run = await runAction(t, { kicsExitCode: '0', inputs: { enable_comments: 'true' } }) + assert.notEqual(run.code, 0, 'the scan produced no report, which is not a success') + }) +}) + +test('results location and cleanup', async (t) => { + await t.test('reads results.json from output_path', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { output_path: 'reports', enable_jobs_summary: 'true' } }) + assert.equal(run.code, 0) + assert.match(run.summary, /TOTAL/) + }) + + await t.test('removes the results.json it asked KICS for when the user did not request json', async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { output_formats: 'sarif' } }) + assert.equal(run.resultsFileExists, false) + }) + + await t.test('removes it when no output format was chosen at all', async (t) => { + const run = await runAction(t, { results: kicsResults() }) + assert.equal(run.resultsFileExists, false) + }) + + for (const formats of ['json', 'sarif,json', 'JSON,sarif']) { + await t.test(`keeps results.json when output_formats is "${formats}"`, async (t) => { + const run = await runAction(t, { results: kicsResults(), inputs: { output_formats: formats } }) + assert.equal(run.resultsFileExists, true) + }) + } +}) diff --git a/test/e2e/entrypoint.test.js b/test/e2e/entrypoint.test.js new file mode 100644 index 0000000..6322738 --- /dev/null +++ b/test/e2e/entrypoint.test.js @@ -0,0 +1,224 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') +const { runEntrypoint, flagValue } = require('../helpers/run-entrypoint') +const { startFakeGitHub } = require('../helpers/fake-github') +const { kicsResults } = require('../helpers/fixtures') +const { knownBug } = require('../helpers/known-bug') + +const DEFAULT_QUERIES = '/app/bin/assets/queries' + +test('required input', async (t) => { + await t.test('refuses to scan without a path', async (t) => { + const run = await runEntrypoint(t, { inputs: { path: '' } }) + assert.equal(run.code, 1) + assert.match(run.stdout, /ERR input path can't be empty/) + assert.equal(run.kicsRan, false) + assert.equal(run.nodeCall, undefined, 'the JS stage must not run either') + }) + + await t.test('scans the given path', async (t) => { + const run = await runEntrypoint(t, { inputs: { path: 'terraform,k8s/deploy.yaml' } }) + assert.equal(flagValue(run.args, '-p'), 'terraform,k8s/deploy.yaml') + }) + + await t.test('tolerates a path wrapped in double quotes', async (t) => { + const run = await runEntrypoint(t, { inputs: { path: '"terraform,modules"' } }) + assert.equal(flagValue(run.args, '-p'), 'terraform,modules') + }) + + await t.test('keeps a path containing spaces as one argument', knownBug('the unquoted $INPUT_PATH is word-split, so "my infra" becomes two arguments'), async (t) => { + const run = await runEntrypoint(t, { inputs: { path: 'my infra' } }) + assert.deepEqual(run.args.slice(run.args.indexOf('-p'), run.args.indexOf('-p') + 2), ['-p', 'my infra']) + }) +}) + +test('baseline command line', async (t) => { + await t.test('runs `kics scan` without the progress bar, which would pollute CI logs', async (t) => { + const run = await runEntrypoint(t) + assert.deepEqual(run.args.slice(0, 2), ['scan', '--no-progress']) + }) + + await t.test('with no optional inputs, passes only path, output, report format and bundled queries', async (t) => { + const run = await runEntrypoint(t) + assert.deepEqual(run.args, ['scan', '--no-progress', '-p', 'infra', '-o', './', '--report-formats', 'json', '-q', DEFAULT_QUERIES]) + }) + + await t.test('uses the queries bundled in the image unless told otherwise', async (t) => { + assert.equal(flagValue((await runEntrypoint(t)).args, '-q'), DEFAULT_QUERIES) + assert.equal(flagValue((await runEntrypoint(t, { inputs: { queries: 'my/queries' } })).args, '-q'), 'my/queries') + }) +}) + +test('inputs mapped to KICS flags', async (t) => { + const valued = [ + ['payload_path', 'payload.json', '-d'], + ['config_path', 'kics.config', '--config'], + ['exclude_paths', './vendor/*,legacy.tf', '-e'], + ['exclude_results', 'abc123,def456', '-x'], + ['exclude_severities', 'info,low', '--exclude-severities'], + ['exclude_queries', '11111111-2222-3333-4444-555555555555', '--exclude-queries'], + ['exclude_categories', 'Observability,Backup', '--exclude-categories'], + ['platform_type', 'Terraform,Dockerfile', '--type'], + ['fail_on', 'high,medium', '--fail-on'], + ['timeout', '60', '--timeout'], + ['profiling', 'CPU', '--profiling'], + ['libraries_path', 'libs', '-b'], + ['secrets_regexes_path', 'secrets.json', '-r'], + ['ignore_on_exit', 'results', '--ignore-on-exit'], + ['cloud_provider', 'aws,gcp', '--cloud-provider'], + ['queries', 'queries/custom,queries/team', '-q'], + ] + for (const [input, value, flag] of valued) { + await t.test(`${input} -> ${flag} ${value}`, async (t) => { + const run = await runEntrypoint(t, { inputs: { [input]: value } }) + assert.equal(flagValue(run.args, flag), value, run.args.join(' ')) + }) + } + + const switches = [ + ['exclude_gitignore', '--exclude-gitignore'], + ['disable_secrets', '--disable-secrets'], + ['disable_full_descriptions', '--disable-full-descriptions'], + ['verbose', '-v'], + ] + for (const [input, flag] of switches) { + await t.test(`${input}: true -> ${flag}`, async (t) => { + const run = await runEntrypoint(t, { inputs: { [input]: 'true' } }) + assert.ok(run.args.includes(flag), run.args.join(' ')) + }) + + await t.test(`${input} left unset -> no ${flag}`, async (t) => { + const run = await runEntrypoint(t) + assert.ok(!run.args.includes(flag)) + }) + + await t.test(`${input}: false -> no ${flag}`, knownBug('any non-empty value, including "false", enables the flag (#121)'), async (t) => { + const run = await runEntrypoint(t, { inputs: { [input]: 'false' } }) + assert.ok(!run.args.includes(flag), `${input}: false must not enable ${flag}: ${run.args.join(' ')}`) + }) + } + + await t.test('include_queries -> -i ', knownBug('the flag is built from $INPUT_PROFILING instead of $INPUT_INCLUDE_QUERIES (#98)'), async (t) => { + const run = await runEntrypoint(t, { inputs: { include_queries: '229588ef-8fde-40c8-8756-f4f2b5825ded' } }) + assert.equal(flagValue(run.args, '-i'), '229588ef-8fde-40c8-8756-f4f2b5825ded') + }) + + await t.test('bom: true -> -m', async (t) => { + const run = await runEntrypoint(t, { inputs: { bom: 'true' } }) + assert.ok(run.args.includes('-m'), run.args.join(' ')) + }) + + await t.test('bom and include_queries do not borrow the value of profiling', knownBug('both flags are built from $INPUT_PROFILING (see #98)'), async (t) => { + const run = await runEntrypoint(t, { inputs: { bom: 'true', include_queries: 'q-1', profiling: 'CPU' } }) + assert.equal(flagValue(run.args, '--profiling'), 'CPU') + assert.notEqual(flagValue(run.args, '-m'), 'CPU', 'bom is a switch, it takes no value') + assert.equal(flagValue(run.args, '-i'), 'q-1') + }) +}) + +test('report formats', async (t) => { + const cases = [ + [undefined, 'json'], + ['json', 'json'], + ['sarif', 'sarif,json'], + ['sarif,json', 'sarif,json'], + ['json,sarif', 'json,sarif'], + ] + for (const [requested, passed] of cases) { + await t.test(`output_formats ${requested === undefined ? 'unset' : `"${requested}"`} -> --report-formats ${passed}`, async (t) => { + const run = await runEntrypoint(t, { inputs: requested === undefined ? {} : { output_formats: requested } }) + assert.equal(flagValue(run.args, '--report-formats'), passed) + }) + } +}) + +test('handover to the JavaScript stage', async (t) => { + await t.test('passes KICS exit code through KICS_EXIT_CODE', async (t) => { + for (const exit of [0, 20, 50]) { + const run = await runEntrypoint(t, { results: kicsResults(), kicsExitCode: exit }) + assert.equal(run.nodeCall.KICS_EXIT_CODE, String(exit)) + } + }) + + await t.test('still runs the JavaScript stage when KICS exits non-zero, so results get reported', async (t) => { + const run = await runEntrypoint(t, { results: kicsResults(), kicsExitCode: 50 }) + assert.ok(run.nodeCall, 'node must run') + assert.equal(run.nodeCall.args, 'dist/index.js') + }) + + await t.test('runs the JavaScript stage from /app, where the bundle and the results copy live', async (t) => { + const run = await runEntrypoint(t, { results: kicsResults() }) + assert.equal(run.nodeCall.cwd, run.appDir) + }) + + await t.test('copies results.json next to the bundle', async (t) => { + const run = await runEntrypoint(t, { results: kicsResults() }) + const copy = JSON.parse(fs.readFileSync(path.join(run.appDir, 'results.json'), 'utf8')) + assert.equal(copy.total_counter, 4) + }) + + await t.test('asks KICS to write into output_path and copies that directory', async (t) => { + const run = await runEntrypoint(t, { results: kicsResults(), inputs: { output_path: 'myoutput/' } }) + assert.equal(flagValue(run.args, '-o'), 'myoutput/') + assert.ok(fs.existsSync(path.join(run.appDir, 'myoutput', 'results.json')), 'main.js reads /results.json relative to /app') + }) + + await t.test('scans relative to the workflow workspace', async (t) => { + const run = await runEntrypoint(t, { results: kicsResults() }) + assert.ok(fs.existsSync(path.join(run.workspace, 'results.json')), 'KICS wrote into GITHUB_WORKSPACE') + }) +}) + +test('whole action: entrypoint, KICS, JavaScript stage and GitHub', async (t) => { + await t.test('a scan with findings comments on the PR, annotates, and fails the step', async (t) => { + const fake = await startFakeGitHub(t) + const run = await runEntrypoint(t, { + runMain: true, + apiUrl: fake.url, + results: kicsResults(), + kicsExitCode: 50, + inputs: { enable_comments: 'true', enable_jobs_summary: 'true', output_formats: 'sarif' }, + }) + + assert.equal(run.code, 1, run.stdout + run.stderr) + assert.match(run.stdout, /::error::KICS scan failed with exit code 50/) + assert.equal((run.stdout.match(/^::warning /gm) || []).length, 4, 'one annotation per finding') + assert.equal(fake.comments.length, 1) + assert.match(fake.comments[0].body, /\| TOTAL \| 4 \|/) + assert.match(run.summary, /\| TOTAL \| 4 \|/) + assert.equal(flagValue(run.args, '--report-formats'), 'sarif,json') + }) + + await t.test('a clean scan passes quietly', async (t) => { + const run = await runEntrypoint(t, { runMain: true, results: kicsResults({ queries: [] }), kicsExitCode: 0, inputs: { enable_annotations: 'true' } }) + assert.equal(run.code, 0, run.stdout + run.stderr) + assert.doesNotMatch(run.stdout, /^::(warning|error)/m) + }) + + await t.test('the workspace is left without a results.json nobody asked for', knownBug('output_formats: sarif still leaves results.json in the workspace; main.js only removes the copy inside /app'), async (t) => { + const run = await runEntrypoint(t, { runMain: true, results: kicsResults(), inputs: { output_formats: 'sarif' } }) + assert.equal(fs.existsSync(path.join(run.workspace, 'results.json')), false) + }) + + await t.test('reports from a config file with a custom output-name are still picked up', knownBug('main.js only reads results.json, so a config with output-name makes the scan fail with ENOENT (#96, #106)'), async (t) => { + const run = await runEntrypoint(t, { + runMain: true, + results: kicsResults(), + resultsName: 'my-results', + workspaceFiles: { 'kics.json': JSON.stringify({ 'output-name': 'my-results' }) }, + inputs: { config_path: 'kics.json', enable_jobs_summary: 'true' }, + }) + assert.match(run.summary, /\| TOTAL \| 4 \|/, run.stdout + run.stderr) + }) + + await t.test('results.json is left readable for the next workflow steps', knownBug('KICS runs as root and writes mode 600, so later steps and upload-sarif cannot read it (#130)'), async (t) => { + const run = await runEntrypoint(t, { results: kicsResults(), resultsMode: '600' }) + const mode = fs.statSync(path.join(run.workspace, 'results.json')).mode + assert.notEqual(mode & 0o044, 0, `mode is ${(mode & 0o777).toString(8)}`) + }) +}) + diff --git a/test/e2e/repository-contract.test.js b/test/e2e/repository-contract.test.js new file mode 100644 index 0000000..0c8a0eb --- /dev/null +++ b/test/e2e/repository-contract.test.js @@ -0,0 +1,60 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') +const { knownBug } = require('../helpers/known-bug') +const { ROOT, loadManifest } = require('../helpers/action-manifest') + +const read = (file) => fs.readFileSync(path.join(ROOT, file), 'utf8') + +test('nothing is downloaded or installed when the action runs (#160, #58)', () => { + const forbidden = /\b(apk|apt|apt-get|npm|npx|yarn|pip|pip3|curl|wget)\b/ + const offending = read('entrypoint.sh').split('\n').filter((line) => forbidden.test(line)) + assert.deepEqual(offending, []) +}) + +test('base images are pinned by digest (#84, #85, #101)', () => { + const stages = new Set() + const unpinned = [] + for (const line of read('Dockerfile').split('\n')) { + const from = line.match(/^FROM\s+(\S+)(?:\s+AS\s+(\S+))?/i) + if (!from) continue + if (!stages.has(from[1]) && !from[1].includes('@sha256:')) unpinned.push(from[1]) + if (from[2]) stages.add(from[2]) + } + assert.deepEqual(unpinned, []) +}) + +test('every third-party action in the workflows is pinned to a full commit SHA', () => { + const dir = path.join(ROOT, '.github', 'workflows') + const unpinned = [] + for (const file of fs.readdirSync(dir)) { + for (const [, ref] of fs.readFileSync(path.join(dir, file), 'utf8').matchAll(/^\s*-?\s*uses:\s*(\S+)/gm)) { + if (!ref.startsWith('./') && !/@[0-9a-f]{40}$/.test(ref)) unpinned.push(`${file}: ${ref}`) + } + } + assert.deepEqual(unpinned, []) +}) + +test('README', async (t) => { + const readme = read('README.md') + + await t.test('documents only inputs that exist in action.yml (#94)', () => { + const table = readme.split('## Inputs')[1].split('\n## ')[0] + const documented = [...table.matchAll(/^\|\s*([a-z_]+)\s*\|/gm)].map((m) => m[1]) + const declared = Object.keys(loadManifest().inputs) + assert.ok(documented.length > 10, 'inputs table parsed') + assert.deepEqual(documented.filter((name) => !declared.includes(name)), []) + }) + + const sarifExamples = [...readme.matchAll(/```yaml\n([\s\S]*?)```/g)].map((m) => m[1]).filter((b) => b.includes('upload-sarif')) + + await t.test('has SARIF upload examples', () => assert.ok(sarifExamples.length > 0)) + + for (const [i, block] of sarifExamples.entries()) { + const options = block.includes('config_path') ? knownBug('the config file example sets neither ignore_on_exit nor ignore-on-exit, so the job fails before the SARIF upload (#76)') : {} + await t.test(`SARIF upload example ${i + 1} does not fail before the upload`, options, () => assert.match(block, /ignore[_-]on[_-]exit/)) + } +}) diff --git a/test/helpers/action-manifest.js b/test/helpers/action-manifest.js new file mode 100644 index 0000000..c891ee0 --- /dev/null +++ b/test/helpers/action-manifest.js @@ -0,0 +1,62 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') + +const ROOT = path.resolve(__dirname, '..', '..') +const ACTION_YML = path.join(ROOT, 'action.yml') + +function unquote(value) { + const v = value.trim() + return /^(["']).*\1$/.test(v) ? v.slice(1, -1) : v +} + +function loadManifest() { + const lines = fs.readFileSync(ACTION_YML, 'utf8').split('\n') + const inputs = {} + const env = {} + let section = '' + let current = null + + for (const line of lines) { + const top = line.match(/^([a-z]+):/) + if (top) { + section = top[1] + continue + } + if (section === 'inputs') { + const name = line.match(/^ {2}([a-z_]+):\s*$/) + if (name) { + current = inputs[name[1]] = { required: false } + continue + } + const prop = line.match(/^ {4}(default|required):\s*(.*)$/) + if (prop && current) current[prop[1]] = unquote(prop[2]) + } + if (section === 'runs') { + const mapping = line.match(/^ {4}(INPUT_[A-Z_]+):\s*\$\{\{\s*inputs\.([a-z_]+)\s*\}\}/) + if (mapping) env[mapping[1]] = mapping[2] + } + } + return { inputs, env } +} + +function runnerInputEnv(overrides = {}, { token = 'test-token' } = {}) { + const { inputs, env } = loadManifest() + const unknown = Object.keys(overrides).filter((name) => !(name in inputs)) + if (unknown.length) throw new Error(`not an input of action.yml: ${unknown.join(', ')}`) + + const value = (name) => { + if (name in overrides) return String(overrides[name]) + const def = inputs[name].default + if (def === undefined) return '' + return def.includes('github.token') ? token : def + } + + const result = {} + for (const name of Object.keys(inputs)) result[`INPUT_${name.toUpperCase()}`] = value(name) + for (const [key, name] of Object.entries(env)) result[key] = value(name) + return result +} + +module.exports = { ROOT, loadManifest, runnerInputEnv } diff --git a/test/helpers/fake-github.js b/test/helpers/fake-github.js new file mode 100644 index 0000000..cfdfbfa --- /dev/null +++ b/test/helpers/fake-github.js @@ -0,0 +1,106 @@ +'use strict' + +const http = require('node:http') +const github = require('@actions/github') + +const MAX_COMMENT_LENGTH = 65536 +const DEFAULT_PER_PAGE = 30 +const MAX_PER_PAGE = 100 + +function json(res, status, body, headers = {}) { + res.writeHead(status, { 'content-type': 'application/json', ...headers }) + res.end(JSON.stringify(body)) +} + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = [] + req.on('data', (c) => chunks.push(c)) + req.on('end', () => resolve(Buffer.concat(chunks).toString())) + req.on('error', reject) + }) +} + +async function startFakeGitHub(t, { comments = [], failWith } = {}) { + const store = comments.map((c) => ({ ...c })) + const requests = [] + let nextId = Math.max(1_000_000, ...store.map((c) => c.id + 1)) + + const server = http.createServer(async (req, res) => { + const url = new URL(req.url, 'http://fake') + const raw = await readBody(req) + const body = raw ? JSON.parse(raw) : undefined + requests.push({ + method: req.method, + path: url.pathname, + query: Object.fromEntries(url.searchParams), + body, + authorization: req.headers.authorization, + }) + + if (failWith) return json(res, failWith, { message: 'simulated outage' }) + + const list = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/issues\/(\d+)\/comments$/) + const single = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/issues\/comments\/(\d+)$/) + + if (list && req.method === 'GET') { + const perPage = Math.min(Number(url.searchParams.get('per_page') || DEFAULT_PER_PAGE), MAX_PER_PAGE) + const page = Number(url.searchParams.get('page') || 1) + const data = store.slice((page - 1) * perPage, page * perPage) + const headers = {} + if (page * perPage < store.length) { + const next = new URL(req.url, `http://${req.headers.host}`) + next.searchParams.set('page', String(page + 1)) + headers.link = `<${next}>; rel="next"` + } + return json(res, 200, data, headers) + } + + if (list && req.method === 'POST') { + if (body.body.length > MAX_COMMENT_LENGTH) return tooLong(res) + const comment = { id: nextId++, user: { login: 'github-actions[bot]' }, body: body.body } + store.push(comment) + return json(res, 201, comment) + } + + if (single && req.method === 'PATCH') { + const comment = store.find((c) => c.id === Number(single[3])) + if (!comment) return json(res, 404, { message: 'Not Found' }) + if (body.body.length > MAX_COMMENT_LENGTH) return tooLong(res) + comment.body = body.body + return json(res, 200, comment) + } + + return json(res, 404, { message: 'Not Found' }) + }) + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const url = `http://127.0.0.1:${server.address().port}` + + const close = () => { + server.closeAllConnections() + return new Promise((resolve) => server.close(resolve)) + } + t.after(close) + + return { + url, + // @actions/github fixes an https proxy agent at import time, so a plain http agent is supplied + octokit: github.getOctokit('token', { baseUrl: url, request: { agent: new http.Agent() } }), + comments: store, + requests, + calls: () => requests.map((r) => `${r.method} ${r.path}`), + writes: () => requests.filter((r) => r.method !== 'GET'), + } +} + +function tooLong(res) { + json(res, 422, { + message: 'Validation Failed', + errors: [{ resource: 'IssueComment', code: 'unprocessable', field: 'data', message: `Body is too long (maximum is ${MAX_COMMENT_LENGTH} characters)` }], + }) +} + +const humanComment = (id) => ({ id, user: { login: 'someone' }, body: `comment ${id}` }) + +module.exports = { startFakeGitHub, humanComment, MAX_COMMENT_LENGTH } diff --git a/test/helpers/fixtures.js b/test/helpers/fixtures.js new file mode 100644 index 0000000..73960ae --- /dev/null +++ b/test/helpers/fixtures.js @@ -0,0 +1,93 @@ +'use strict' + +const SEVERITIES = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'INFO', 'TRACE'] + +function finding(overrides = {}) { + return { + file_name: 'test/samples/positive1.tf', + similarity_id: 'a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90', + line: 12, + issue_type: 'MissingAttribute', + search_key: 'aws_s3_bucket[bucket]', + search_line: 12, + search_value: '', + expected_value: "'versioning' should be defined and enabled", + actual_value: "'versioning' is undefined", + ...overrides, + } +} + +function query(overrides = {}) { + return { + query_name: 'S3 Bucket Without Versioning', + query_id: '568a4d22-3517-44a6-a7ad-6a7eed88722c', + query_url: 'https://docs.kics.io/latest/queries/terraform-queries/aws/568a4d22-3517-44a6-a7ad-6a7eed88722c', + severity: 'MEDIUM', + platform: 'Terraform', + cloud_provider: 'AWS', + category: 'Backup', + experimental: false, + description: 'S3 bucket should have versioning enabled', + description_id: '8c7c3a2b', + files: [finding()], + ...overrides, + } +} + +function defaultQueries() { + return [ + query({ + files: [finding(), finding({ file_name: 'test/samples/positive2.tf', line: 3 })], + }), + query({ + query_name: 'Security Group With Unrestricted Ingress', + query_id: '4728cd65-a20c-49da-8b31-9c08b423e4db', + query_url: 'https://docs.kics.io/latest/queries/terraform-queries/aws/4728cd65-a20c-49da-8b31-9c08b423e4db', + severity: 'HIGH', + category: 'Networking and Firewall', + description: 'Security group allows ingress from 0.0.0.0/0', + files: [finding({ file_name: 'test/samples/positive2.tf', line: 30 })], + }), + query({ + query_name: 'Resource Without Tags', + query_id: '0a3b9a1e-7a52-4b5b-8c1e-3f7e2f2f6a10', + query_url: 'https://docs.kics.io/latest/queries/terraform-queries/aws/0a3b9a1e-7a52-4b5b-8c1e-3f7e2f2f6a10', + severity: 'LOW', + category: 'Best Practices', + description: 'Resources should be tagged', + files: [finding({ line: 45 })], + }), + ] +} + +function kicsResults({ queries = defaultQueries(), ...overrides } = {}) { + const severity_counters = Object.fromEntries(SEVERITIES.map((s) => [s, 0])) + let total = 0 + for (const q of queries) { + severity_counters[q.severity] += q.files.length + total += q.files.length + } + return { + kics_version: 'v2.1.20', + files_scanned: 2, + lines_scanned: 120, + files_parsed: 2, + lines_parsed: 120, + lines_ignored: 0, + files_failed_to_scan: 0, + queries_total: 280, + queries_failed_to_execute: 0, + queries_failed_to_compute_similarity_id: 0, + scan_id: 'console', + severity_counters, + total_counter: total, + total_bom_resources: 0, + start: '2026-03-01T10:00:00.123456+00:00', + end: '2026-03-01T10:00:07.123456+00:00', + paths: ['test/samples/positive1.tf', 'test/samples/positive2.tf'], + queries, + ...overrides, + } +} + +module.exports = { kicsResults, query, finding, SEVERITIES } diff --git a/test/helpers/known-bug.js b/test/helpers/known-bug.js new file mode 100644 index 0000000..f0f159d --- /dev/null +++ b/test/helpers/known-bug.js @@ -0,0 +1,5 @@ +'use strict' + +const knownBug = (reason) => (process.env.KNOWN_BUGS === 'skip' ? { skip: reason } : { todo: reason }) + +module.exports = { knownBug } diff --git a/test/helpers/run-action.js b/test/helpers/run-action.js new file mode 100644 index 0000000..ac9d359 --- /dev/null +++ b/test/helpers/run-action.js @@ -0,0 +1,93 @@ +'use strict' + +const { spawn } = require('node:child_process') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { ROOT, runnerInputEnv } = require('./action-manifest') + +const MAIN_JS = path.join(ROOT, 'src', 'main.js') + +function run(command, args, options) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, options) + let stdout = '' + let stderr = '' + child.stdout.on('data', (d) => (stdout += d)) + child.stderr.on('data', (d) => (stderr += d)) + child.on('error', reject) + child.on('close', (code) => resolve({ code, stdout, stderr })) + }) +} + +function baseEnv() { + const env = { PATH: process.env.PATH, HOME: os.tmpdir() } + // keep collecting coverage from the spawned process when the suite runs with coverage enabled + if (process.env.NODE_V8_COVERAGE) env.NODE_V8_COVERAGE = process.env.NODE_V8_COVERAGE + return env +} + +function githubContext(dir, { event = 'pull_request', prNumber = 1, apiUrl, repository = 'o/r' } = {}) { + const eventPath = path.join(dir, 'event.json') + const payload = event === 'pull_request' ? { pull_request: { number: prNumber } } : { ref: 'refs/heads/master' } + fs.writeFileSync(eventPath, JSON.stringify(payload)) + const summaryPath = path.join(dir, 'step-summary.md') + fs.writeFileSync(summaryPath, '') + const env = { + GITHUB_REPOSITORY: repository, + GITHUB_EVENT_NAME: event, + GITHUB_EVENT_PATH: eventPath, + GITHUB_STEP_SUMMARY: summaryPath, + } + if (apiUrl) env.GITHUB_API_URL = apiUrl + return { env, summaryPath } +} + +function workflowCommands(stdout, name) { + return stdout + .split('\n') + .map((line) => line.match(new RegExp(`^::${name}(?: ([^:]*))?::(.*)$`))) + .filter(Boolean) + .map(([, props = '', message]) => ({ + message: decodeURIComponent(message.replace(/%0A/gi, '\n')), + properties: Object.fromEntries( + props.split(',').filter(Boolean).map((p) => { + const i = p.indexOf('=') + return [p.slice(0, i), decodeURIComponent(p.slice(i + 1))] + }) + ), + })) +} + +async function runAction(t, { results, inputs = {}, kicsExitCode = '0', apiUrl, event, prNumber } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kics-action-')) + const outputDir = inputs.output_path ? path.join(dir, inputs.output_path) : dir + fs.mkdirSync(outputDir, { recursive: true }) + const resultsFile = path.join(outputDir, 'results.json') + if (results !== undefined) fs.writeFileSync(resultsFile, JSON.stringify(results)) + + const context = githubContext(dir, { event, prNumber, apiUrl }) + const env = { + ...baseEnv(), + ...runnerInputEnv(inputs), + ...context.env, + KICS_EXIT_CODE: kicsExitCode, + } + + const { code, stdout, stderr } = await run(process.execPath, [MAIN_JS], { cwd: dir, env }) + + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + + return { + code, + stdout, + stderr, + warnings: workflowCommands(stdout, 'warning'), + errors: workflowCommands(stdout, 'error'), + summary: fs.readFileSync(context.summaryPath, 'utf8'), + resultsFileExists: fs.existsSync(resultsFile), + dir, + } +} + +module.exports = { runAction, run, baseEnv, githubContext, workflowCommands, MAIN_JS } diff --git a/test/helpers/run-entrypoint.js b/test/helpers/run-entrypoint.js new file mode 100644 index 0000000..1bddb81 --- /dev/null +++ b/test/helpers/run-entrypoint.js @@ -0,0 +1,116 @@ +'use strict' + +// Redirects the hard-coded /app paths of entrypoint.sh into a sandbox; throws if those lines change shape. + +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { ROOT, runnerInputEnv } = require('./action-manifest') +const { run, baseEnv, githubContext, MAIN_JS } = require('./run-action') + +const ENTRYPOINT = path.join(ROOT, 'entrypoint.sh') + +const CONTAINER_PATH_REWRITES = [ + [/^\/app\/bin\/kics scan/m, (sandbox) => `${sandbox}/app/bin/kics scan`], + [/^cp -r "\$\{CP_PATH\}" "\/app\/"$/m, (sandbox) => `cp -r "\${CP_PATH}" "${sandbox}/app/"`], + [/^cd \/app$/m, (sandbox) => `cd ${sandbox}/app`], +] + +function sandboxedEntrypoint(sandbox) { + let script = fs.readFileSync(ENTRYPOINT, 'utf8') + for (const [pattern, replacement] of CONTAINER_PATH_REWRITES) { + if (!pattern.test(script)) throw new Error(`entrypoint.sh changed: no line matches ${pattern}; update test/helpers/run-entrypoint.js`) + script = script.replace(pattern, replacement(sandbox)) + } + return script +} + +const FAKE_KICS = `#!/bin/bash +printf '%s\\n' "$@" > "$SANDBOX/kics-args.txt" +out=./ +while [ $# -gt 0 ]; do + if [ "$1" = "-o" ]; then out="$2"; fi + shift +done +if [ -n "$FAKE_KICS_RESULTS" ]; then + mkdir -p "$out" + file="$out/\${FAKE_KICS_NAME:-results}.json" + cp "$FAKE_KICS_RESULTS" "$file" + if [ -n "$FAKE_KICS_MODE" ]; then chmod "$FAKE_KICS_MODE" "$file"; fi +fi +exit "\${FAKE_KICS_EXIT:-0}" +` + +const NODE_SHIM = `#!/bin/bash +{ echo "cwd=$(pwd)"; echo "args=$*"; echo "KICS_EXIT_CODE=$KICS_EXIT_CODE"; } > "$SANDBOX/node-call.txt" +if [ -n "$RUN_MAIN" ]; then exec "$REAL_NODE" "$MAIN_JS"; fi +` + +function write(file, content, mode) { + fs.mkdirSync(path.dirname(file), { recursive: true }) + fs.writeFileSync(file, content, { mode }) +} + +async function runEntrypoint(t, { inputs = {}, results, kicsExitCode = 0, runMain = false, apiUrl, workspaceFiles = {}, resultsName, resultsMode } = {}) { + const sandbox = fs.mkdtempSync(path.join(os.tmpdir(), 'kics-entrypoint-')) + t.after(() => fs.rmSync(sandbox, { recursive: true, force: true })) + + const workspace = path.join(sandbox, 'workspace') + fs.mkdirSync(workspace) + for (const [name, content] of Object.entries(workspaceFiles)) write(path.join(workspace, name), content) + + write(path.join(sandbox, 'app', 'bin', 'kics'), FAKE_KICS, 0o755) + fs.mkdirSync(path.join(sandbox, 'app', 'dist')) + write(path.join(sandbox, 'bin', 'node'), NODE_SHIM, 0o755) + write(path.join(sandbox, 'entrypoint.sh'), sandboxedEntrypoint(sandbox), 0o755) + + let resultsFile = '' + if (results !== undefined) { + resultsFile = path.join(sandbox, 'canned-results.json') + fs.writeFileSync(resultsFile, JSON.stringify(results)) + } + + const context = githubContext(sandbox, { apiUrl }) + const env = { + ...baseEnv(), + PATH: `${path.join(sandbox, 'bin')}:${process.env.PATH}`, + ...runnerInputEnv({ path: 'infra', ...inputs }), + ...context.env, + GITHUB_WORKSPACE: workspace, + SANDBOX: sandbox, + FAKE_KICS_RESULTS: resultsFile, + FAKE_KICS_EXIT: String(kicsExitCode), + FAKE_KICS_NAME: resultsName ?? '', + FAKE_KICS_MODE: resultsMode ?? '', + REAL_NODE: process.execPath, + MAIN_JS, + } + if (runMain) env.RUN_MAIN = '1' + + // bash, not the container's ash: the script only uses constructs both support + const { code, stdout, stderr } = await run('bash', [path.join(sandbox, 'entrypoint.sh')], { cwd: workspace, env }) + + const read = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : undefined) + const argsText = read(path.join(sandbox, 'kics-args.txt')) + const nodeCall = read(path.join(sandbox, 'node-call.txt')) + + return { + code, + stdout, + stderr, + kicsRan: argsText !== undefined, + args: argsText === undefined ? [] : argsText.trimEnd().split('\n'), + nodeCall: nodeCall && Object.fromEntries(nodeCall.trimEnd().split('\n').map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)])), + sandbox, + workspace, + appDir: path.join(sandbox, 'app'), + summary: read(context.summaryPath), + } +} + +function flagValue(args, flag) { + const i = args.indexOf(flag) + return i === -1 ? undefined : args[i + 1] +} + +module.exports = { runEntrypoint, flagValue } diff --git a/test/unit/commenter.test.js b/test/unit/commenter.test.js new file mode 100644 index 0000000..f91abeb --- /dev/null +++ b/test/unit/commenter.test.js @@ -0,0 +1,286 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { postPRComment, postJobSummary } = require('../../src/commenter') +const { startFakeGitHub, humanComment, MAX_COMMENT_LENGTH } = require('../helpers/fake-github') +const { kicsResults, query, finding } = require('../helpers/fixtures') +const { knownBug } = require('../helpers/known-bug') + +const repo = { owner: 'o', repo: 'r' } +const PR = 1 +const COMMENTS_PATH = '/repos/o/r/issues/1/comments' +const kicsBody = '![kics-logo](https://example.com/logo.png)\nold report' +const noExclusions = [] + +// Pending fix: PR #163 (fix-duplicated-comment). Remove the `knownBug` markers once it is merged. +const PR_163 = 'PR #163: comment lookup must paginate and match on the KICS marker, not the author' + +async function post(fake, results = kicsResults(), { withQueries = false, excluded = noExclusions, prNumber = PR } = {}) { + await postPRComment(results, repo, prNumber, fake.octokit, withQueries, excluded) +} + +const postedBody = (fake) => fake.writes().at(-1).body.body + +test('PR comment lifecycle', async (t) => { + await t.test('creates a comment when none exists', async (t) => { + const fake = await startFakeGitHub(t, { comments: [humanComment(1)] }) + await post(fake) + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), [`POST ${COMMENTS_PATH}`]) + }) + + await t.test('updates the existing KICS comment instead of adding another', async (t) => { + const fake = await startFakeGitHub(t, { + comments: [humanComment(1), { id: 2, user: { login: 'github-actions[bot]' }, body: kicsBody }, humanComment(3)], + }) + await post(fake, kicsResults({ kics_version: 'v9.9.9' })) + + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), ['PATCH /repos/o/r/issues/comments/2']) + assert.match(fake.comments.find((c) => c.id === 2).body, /KICS version: v9\.9\.9/, 'the stale report is replaced') + assert.ok(!fake.comments.find((c) => c.id === 2).body.includes('old report')) + }) + + await t.test('re-running on the same PR keeps a single KICS comment', async (t) => { + const fake = await startFakeGitHub(t, { comments: [humanComment(1)] }) + await post(fake, kicsResults({ kics_version: 'v1.0.0' })) + await post(fake, kicsResults({ kics_version: 'v2.0.0' })) + await post(fake, kicsResults({ kics_version: 'v3.0.0' })) + + const kicsComments = fake.comments.filter((c) => c.body.startsWith('![kics-logo](')) + assert.equal(kicsComments.length, 1, fake.calls().join('\n')) + assert.match(kicsComments[0].body, /v3\.0\.0/) + }) + + await t.test('only touches the pull request it was asked about', async (t) => { + const fake = await startFakeGitHub(t) + await post(fake, kicsResults(), { prNumber: 42 }) + assert.deepEqual(fake.calls(), ['GET /repos/o/r/issues/42/comments', 'POST /repos/o/r/issues/42/comments']) + }) + + await t.test('authenticates with the provided token', async (t) => { + const fake = await startFakeGitHub(t) + await post(fake) + assert.ok(fake.requests.every((r) => r.authorization === 'token token')) + }) + + await t.test('does not treat a quoted KICS report as the KICS comment', async (t) => { + const fake = await startFakeGitHub(t, { comments: [{ id: 7, user: { login: 'someone' }, body: '> ' + kicsBody }] }) + await post(fake) + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), [`POST ${COMMENTS_PATH}`]) + }) + + await t.test('does not touch a human comment that merely mentions KICS', async (t) => { + const fake = await startFakeGitHub(t, { comments: [{ id: 7, user: { login: 'someone' }, body: 'KICS found nothing, nice!' }] }) + await post(fake) + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), [`POST ${COMMENTS_PATH}`]) + assert.equal(fake.comments.find((c) => c.id === 7).body, 'KICS found nothing, nice!') + }) +}) + +test('finding the existing KICS comment in a busy thread', async (t) => { + await t.test('updates it when buried past the first 30 comments', knownBug(PR_163), async (t) => { + const comments = Array.from({ length: 40 }, (_, i) => humanComment(i)) + comments.push({ id: 1000, user: { login: 'github-actions[bot]' }, body: kicsBody }) + const fake = await startFakeGitHub(t, { comments }) + + await post(fake) + + assert.ok(fake.calls().includes('PATCH /repos/o/r/issues/comments/1000'), fake.calls().join('\n')) + assert.ok(!fake.calls().some((c) => c.startsWith('POST')), 'must not create a duplicate') + }) + + await t.test('updates it when it is on page 3 of 100+ comments per page', knownBug(PR_163), async (t) => { + const comments = Array.from({ length: 250 }, (_, i) => humanComment(i)) + comments.push({ id: 1000, user: { login: 'github-actions[bot]' }, body: kicsBody }) + const fake = await startFakeGitHub(t, { comments }) + + await post(fake) + + assert.ok(fake.calls().includes('PATCH /repos/o/r/issues/comments/1000'), fake.calls().join('\n')) + assert.ok(!fake.calls().some((c) => c.startsWith('POST')), 'must not create a duplicate') + assert.ok(fake.requests.filter((r) => r.method === 'GET').length <= 3, 'lists 100 comments per page, not 30') + }) + + await t.test('updates it when it was posted by an author other than github-actions[bot]', knownBug(PR_163), async (t) => { + const fake = await startFakeGitHub(t, { comments: [{ id: 5, user: { login: 'my-app[bot]' }, body: kicsBody }] }) + await post(fake) + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), ['PATCH /repos/o/r/issues/comments/5']) + }) + + await t.test('updates the oldest KICS comment when duplicates already exist', knownBug(PR_163), async (t) => { + const fake = await startFakeGitHub(t, { + comments: [ + { id: 10, user: { login: 'someone' }, body: kicsBody }, + { id: 11, user: { login: 'someone' }, body: kicsBody }, + ], + }) + await post(fake) + assert.deepEqual(fake.writes().map((r) => `${r.method} ${r.path}`), ['PATCH /repos/o/r/issues/comments/10']) + }) +}) + +test('GitHub comment size limit', async (t) => { + await t.test('a normal report is well under the limit', async (t) => { + const fake = await startFakeGitHub(t) + await post(fake) + assert.ok(postedBody(fake).length < MAX_COMMENT_LENGTH) + }) + + await t.test('a huge report with queries still gets posted', knownBug('GitHub rejects bodies over 65536 characters with 422; the report must be truncated or split'), async (t) => { + const fake = await startFakeGitHub(t) + const many = Array.from({ length: 400 }, (_, i) => finding({ file_name: `modules/service-${i}/main.tf`, line: i + 1 })) + const results = kicsResults({ queries: [query({ files: many })] }) + + await post(fake, results, { withQueries: true, excluded: ['description_id', 'similarity_id'] }) + + assert.equal(fake.comments.length, 1, 'a comment must exist even when the full report is too large') + assert.ok(fake.comments[0].body.length <= MAX_COMMENT_LENGTH) + }) +}) + +test('report content', async (t) => { + const reportFor = async (t, results, options) => { + const fake = await startFakeGitHub(t) + await post(fake, results, options) + return postedBody(fake) + } + + await t.test('starts with the KICS marker used to find the comment again', async (t) => { + const body = await reportFor(t, kicsResults()) + assert.ok(body.startsWith('![kics-logo](')) + }) + + await t.test('shows the KICS version, severity counts and total', async (t) => { + const body = await reportFor(t, kicsResults()) + assert.match(body, /\*\*KICS version: v2\.1\.20\*\*/) + assert.match(body, /\| HIGH \| 1 \|/) + assert.match(body, /\| MEDIUM \| 2 \|/) + assert.match(body, /\| LOW \| 1 \|/) + assert.match(body, /\| TOTAL \| 4 \|/) + }) + + await t.test('lists severities from most to least severe, only those the report contains', async (t) => { + const results = kicsResults({ severity_counters: { LOW: 3, HIGH: 1, INFO: 2 }, total_counter: 6 }) + const body = await reportFor(t, results) + const order = [...body.matchAll(/\| (CRITICAL|HIGH|MEDIUM|LOW|INFO|TRACE) \| \d+ \|/g)].map((m) => m[1]) + assert.deepEqual(order, ['HIGH', 'LOW', 'INFO']) + }) + + await t.test('has a row, with its icon, for every severity KICS can report', async (t) => { + const severities = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'INFO', 'TRACE'] + const queries = severities.map((severity) => query({ severity, query_name: `${severity} query` })) + const body = await reportFor(t, kicsResults({ queries })) + for (const severity of severities) assert.match(body, new RegExp(`\\| !\\[${severity}\\]\\(https://[^)]+\\) \\| ${severity} \\| 1 \\|`), severity) + assert.match(body, /\| TOTAL \| 6 \|/) + assert.ok(!body.includes('undefined')) + }) + + await t.test('shows scan metrics, with execution time computed from start and end', async (t) => { + const body = await reportFor(t, kicsResults({ files_scanned: 12, files_parsed: 11, files_failed_to_scan: 1, queries_total: 280, queries_failed_to_execute: 2 })) + assert.match(body, /Files scanned .*\| 12\n/) + assert.match(body, /Files parsed .*\| 11\n/) + assert.match(body, /Files failed to scan .*\| 1\n/) + assert.match(body, /Total executed queries .*\| 280\n/) + assert.match(body, /Queries failed to execute .*\| 2\n/) + assert.match(body, /Execution time .*\| 7\n/) + }) + + await t.test('a clean scan reports zero findings and no query table', async (t) => { + const body = await reportFor(t, kicsResults({ queries: [] }), { withQueries: true }) + assert.match(body, /\| TOTAL \| 0 \|/) + assert.ok(!/\| HIGH \| [1-9]/.test(body)) + }) + + await t.test('omits the queries table unless requested', async (t) => { + const body = await reportFor(t, kicsResults(), { withQueries: false }) + assert.ok(!body.includes('### Queries Results')) + assert.ok(!body.includes('Security Group With Unrestricted Ingress')) + }) +}) + +test('report content with queries', async (t) => { + const withQueries = async (t, results, excluded = noExclusions) => { + const fake = await startFakeGitHub(t) + await post(fake, results, { withQueries: true, excluded }) + const body = postedBody(fake) + const table = body.split('### Queries Results\n')[1] + const lines = table.split('')[0].split('\n').filter((l) => l.trim() && !l.startsWith('<')) + const rows = lines.filter((l) => l.startsWith('|')) + return { body, header: rows[0], rows: rows.slice(2), lines } + } + + await t.test('has one row per finding, not per query', async (t) => { + const { rows } = await withQueries(t, kicsResults()) + assert.equal(rows.length, 4) + }) + + await t.test('links each query name to its documentation', async (t) => { + const { body } = await withQueries(t, kicsResults()) + assert.ok(body.includes('[Security Group With Unrestricted Ingress](https://docs.kics.io/latest/queries/terraform-queries/aws/4728cd65-a20c-49da-8b31-9c08b423e4db)')) + }) + + await t.test('titles columns from the snake_case report keys', async (t) => { + const { header } = await withQueries(t, kicsResults()) + assert.match(header, /\| Query Name/) + assert.match(header, /\| Similarity Id/) + assert.match(header, /\| File Name/) + }) + + await t.test('never shows the raw query_url column', async (t) => { + const { header } = await withQueries(t, kicsResults()) + assert.ok(!/Query Url/.test(header)) + }) + + await t.test('hides the columns the workflow excluded', async (t) => { + const { header, body } = await withQueries(t, kicsResults(), ['description_id', 'similarity_id', 'search_line', 'search_value']) + for (const hidden of ['Description Id', 'Similarity Id', 'Search Line', 'Search Value']) assert.ok(!header.includes(hidden), hidden) + assert.match(header, /\| Severity/) + assert.ok(!body.includes('a1b2c3d4e5f60718'), 'excluded values are not leaked in rows') + }) + + await t.test('keeps rows aligned with the header when a finding lacks a column', async (t) => { + const sparse = finding({ file_name: 'a.tf' }) + delete sparse.search_value + delete sparse.actual_value + const { header, rows } = await withQueries(t, kicsResults({ queries: [query({ files: [finding(), sparse] })] })) + const columns = (line) => line.replace(/\|$/, '').split('|').length + assert.equal(columns(rows[1]), columns(header)) + }) + + await t.test('keeps a multi-line value inside its table cell', knownBug('only the first newline is replaced, so a second one breaks the markdown table'), async (t) => { + const multiline = finding({ actual_value: 'line one\nline two\nline three' }) + const { lines } = await withQueries(t, kicsResults({ queries: [query({ files: [multiline] })] })) + assert.ok(lines.every((l) => l.startsWith('|')), `stray lines outside the table:\n${lines.join('\n')}`) + }) + + await t.test('keeps a value containing a pipe inside its table cell', knownBug('unescaped | in a value (e.g. a regex in search_value) shifts every following column'), async (t) => { + const piped = finding({ search_value: 'a|b' }) + const { header, rows } = await withQueries(t, kicsResults({ queries: [query({ files: [piped] })] })) + const columns = (line) => line.replace(/\\\|/g, '').replace(/\|$/, '').split('|').length + assert.equal(columns(rows[0]), columns(header)) + }) +}) + +test('job summary', async (t) => { + await t.test('writes the same report to the workflow run summary', async (t) => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'kics-summary-')), 'summary.md') + fs.writeFileSync(file, '') + const previous = process.env.GITHUB_STEP_SUMMARY + process.env.GITHUB_STEP_SUMMARY = file + t.after(() => { + if (previous === undefined) delete process.env.GITHUB_STEP_SUMMARY + else process.env.GITHUB_STEP_SUMMARY = previous + fs.rmSync(path.dirname(file), { recursive: true, force: true }) + }) + + await postJobSummary(kicsResults(), true, noExclusions) + + const written = fs.readFileSync(file, 'utf8') + assert.ok(written.startsWith('![kics-logo](')) + assert.match(written, /\| TOTAL \| 4 \|/) + assert.match(written, /### Queries Results/) + }) +}) diff --git a/test/unit/coverage-badge.test.js b/test/unit/coverage-badge.test.js new file mode 100644 index 0000000..b0b8155 --- /dev/null +++ b/test/unit/coverage-badge.test.js @@ -0,0 +1,26 @@ +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const { lineCoverage, colorFor, badge } = require('../../scripts/coverage-badge') + +const lcov = (...files) => files.map(([lf, lh]) => `SF:x.js\nLF:${lf}\nLH:${lh}\nend_of_record`).join('\n') + +test('coverage badge', async (t) => { + await t.test('weights files by their number of lines, not by file count', () => { + assert.equal(lineCoverage(lcov([100, 90], [10, 0])), 81.8) + }) + + await t.test('refuses an empty report instead of publishing a misleading badge', () => { + assert.throws(() => lineCoverage(''), /no instrumented lines/) + }) + + await t.test('colour follows the coverage bands', () => { + const bands = [[100, 'brightgreen'], [90, 'brightgreen'], [89.9, 'green'], [80, 'green'], [79.9, 'yellowgreen'], [70, 'yellowgreen'], [60, 'yellow'], [50, 'orange'], [49.9, 'red'], [0, 'red']] + for (const [percent, color] of bands) assert.equal(colorFor(percent), color, `${percent}%`) + }) + + await t.test('produces a shields.io endpoint document', () => { + assert.deepEqual(badge(87.5), { schemaVersion: 1, label: 'coverage', message: '87.5%', color: 'green' }) + }) +})