From da0b7d1a495571d34132678e2c18c46dfb278e6b Mon Sep 17 00:00:00 2001 From: monoxgas Date: Wed, 26 Aug 2026 15:25:54 -0600 Subject: [PATCH 1/2] fix(web-security): complete the install without reaching the network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install script runs on every sandbox boot where the capability changed, not just the first. Every download was unguarded, so a runtime that already carried the tooling fetched it again anyway — and on a disconnected install the first fetch failed, `set -e` aborted, and the entire provision died at line 15 before installing anything. Measured against a runtime image with the tooling pre-baked: `SCRIPT EXIT=2`, nothing installed. Guard every fetch on the artefact it produces, and pin every version: - `go install ...@latest` x5 pinned (pdtm v0.1.5, protoscope, interactsh v1.3.1, 2fa v1.2.0, surf v0.0.5) and guarded. `@latest` re-resolves against the module proxy even when the binary is present, and produces a different tool set on different days — which no SBOM can describe. - `pdtm -install` now installs only the tools actually missing, rather than re-fetching all nine every run. - katana, agent-browser, its browser binaries, and the caido-mode npm deps guarded the same way. - The Go toolchain is a ~150 MB download whose only purpose is building the tools above, so it is now requested only when one of them is missing. That was the line the whole script died on. Three latent bugs surfaced once execution got that far, all of which aborted the run on a non-root runtime regardless of connectivity: - `mkdir -p /opt/burp` and the `/usr/local/bin/burp` wrapper assumed root. Now escalate through `as_root` and degrade to a warning. The wrapper is only written when the jar actually arrived — a `burp` on PATH pointing at nothing is worse than no `burp`. - `apt-get install` for exiftool and Node.js assumed root, same treatment. - `pip install` assumed a `pip` binary. A uv-managed virtualenv has none, so these died with "command not found" on exactly the images the SDK ships. `py_install` prefers `uv pip`, falls back to pip, then `python3 -m pip`. Burp, Caido, jxscout and the browser binaries are not redistributable, so a disconnected deployment will not have them. Those now degrade to warnings rather than taking the rest of the tooling down with them. Also: the `caido-cli` check asserted a local server binary the capability never uses — it talks to Caido over `CAIDO_URL` with the Python client (mcp/caido.py). An enclave running its own Caido was reported degraded while working correctly. Now satisfied by a local binary *or* a reachable endpoint; verified all three ways. Verified in a runtime image with the tooling pre-baked, `--network none`: `SCRIPT EXIT=0`, 13 checks passing, every unavailable vendor tool warning rather than aborting. New tests pin both properties — every fetch guarded, every version pinned — and all seven fail against the previous script. --- capabilities/web-security/capability.yaml | 9 +- .../web-security/scripts/install_tools.sh | 183 ++++++++++++++---- .../tests/test_caido_mode_skill.py | 19 +- .../tests/test_install_tools_offline.py | 91 +++++++++ 4 files changed, 256 insertions(+), 46 deletions(-) create mode 100644 capabilities/web-security/tests/test_install_tools_offline.py diff --git a/capabilities/web-security/capability.yaml b/capabilities/web-security/capability.yaml index 8528ffb..325f83f 100644 --- a/capabilities/web-security/capability.yaml +++ b/capabilities/web-security/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: web-security -version: "1.13.0" +version: "1.14.0" description: > Web application penetration testing with 83 attack technique playbooks covering HTTP desync/request smuggling, cache poisoning, SSRF, SSTI, DOM @@ -174,8 +174,13 @@ checks: command: 'command -v interactsh-client >/dev/null 2>&1 || test -x "$HOME/go/bin/interactsh-client"' - name: protoscope command: 'command -v protoscope >/dev/null 2>&1 || test -x "$HOME/go/bin/protoscope"' + # The capability talks to Caido over CAIDO_URL using the Python client, not + # by driving the local binary (see mcp/caido.py). Asserting only the binary + # reports a correctly configured deployment as degraded — self-hosted + # installs may not ship caido-cli at all and instead point at a Caido the + # operator runs. Local binary or reachable endpoint, either satisfies it. - name: caido-cli - command: command -v caido-cli + command: 'command -v caido-cli >/dev/null 2>&1 || curl -fsS --max-time 3 -o /dev/null "${CAIDO_URL:-http://localhost:8080}"' - name: caido-mcp-server command: 'command -v caido-mcp-server >/dev/null 2>&1 || test -x "$HOME/bin/caido-mcp-server"' - name: caido-mode diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 2c3878a..1731a46 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -14,44 +14,121 @@ case "$OS" in ;; esac -# -- Go toolchain (needed for pdtm, protoscope, interactsh, surf) --------- -if ! command -v go &>/dev/null; then +export PATH="$HOME/.pdtm/go/bin:$HOME/go/bin:$PATH" + +# `have ` — is this already on PATH, or in one of the two directories the +# tools below install into? +# +# Every fetch in this script is guarded on the artefact it would produce, so a +# runtime image that already carries the tooling completes without a single +# outbound request. That matters beyond speed: self-hosted deployments run with +# no route to the internet, and this script runs on every sandbox boot where +# the capability changed, so an unguarded download is a repeated outbound +# attempt that cannot succeed. It is also why versions are pinned rather than +# `@latest` — an unpinned install re-resolves against the network even when the +# binary is present, and produces a different tool set on different days. +have() { + command -v "$1" >/dev/null 2>&1 \ + || [ -x "$HOME/.pdtm/go/bin/$1" ] \ + || [ -x "$HOME/go/bin/$1" ] +} + +# Some vendor tooling installs into root-owned paths. Whether this script runs +# as root depends on the image, so escalate only when needed and only when it +# is available — and let the caller decide what a failure means. +as_root() { + if [ "$(id -u)" = "0" ]; then + "$@" + else + sudo -n "$@" 2>/dev/null + fi +} + +# Install a Python package into whatever interpreter this runtime uses. +# `pip` is not always on PATH — a uv-managed virtualenv has no pip binary at +# all, which made the bare `pip install` calls below abort the run with +# "command not found" on exactly the images the SDK ships. +py_install() { + if command -v uv >/dev/null 2>&1; then + uv pip install --python "$(command -v python3)" "$@" + elif command -v pip >/dev/null 2>&1; then + pip install --break-system-packages "$@" + else + python3 -m pip install --break-system-packages "$@" + fi +} + +GO_TOOL_VERSIONS_pdtm="v0.1.5" +GO_TOOL_VERSIONS_protoscope="v0.0.0-20221109213918-8e7a6aafa2c9" +GO_TOOL_VERSIONS_interactsh="v1.3.1" +GO_TOOL_VERSIONS_2fa="v1.2.0" +GO_TOOL_VERSIONS_surf="v0.0.5" + +PD_TOOLS="nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap" + +# What is actually missing, before anything is fetched. +missing_go_tools="" +for tool in protoscope interactsh-client 2fa surf; do + have "$tool" || missing_go_tools="$missing_go_tools $tool" +done +missing_pd_tools="" +for tool in $PD_TOOLS; do + have "$tool" || missing_pd_tools="$missing_pd_tools,$tool" +done +missing_pd_tools="${missing_pd_tools#,}" + +# -- Go toolchain (only when something still has to be built) -------------- +# Deliberately last in the decision order: the toolchain is a ~150 MB download +# whose only purpose is building the tools above. If they are all present it is +# never needed, so it is never requested. +need_go=false +[ -n "$missing_go_tools" ] && need_go=true +[ -n "$missing_pd_tools" ] && ! have pdtm && need_go=true +if [ "$need_go" = true ] && ! command -v go &>/dev/null; then GO_VERSION="1.24.3" case "$ARCH" in aarch64|arm64) GOARCH="arm64" ;; *) GOARCH="amd64" ;; esac curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" | tar -xz -C /usr/local - export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" + export PATH="/usr/local/go/bin:$PATH" fi # -- PDTM + ProjectDiscovery tools ---------------------------------------- -go install github.com/projectdiscovery/pdtm/cmd/pdtm@latest -PDTM_BIN="$(go env GOPATH)/bin/pdtm" -"$PDTM_BIN" -install nuclei,httpx,subfinder,naabu,dnsx,uncover,alterx,tlsx,asnmap -export PATH="$HOME/.pdtm/go/bin:$PATH" +if [ -n "$missing_pd_tools" ]; then + if ! have pdtm; then + go install "github.com/projectdiscovery/pdtm/cmd/pdtm@${GO_TOOL_VERSIONS_pdtm}" + fi + PDTM_BIN="$(command -v pdtm || echo "$(go env GOPATH)/bin/pdtm")" + "$PDTM_BIN" -install "$missing_pd_tools" +fi # -- katana (pre-built binary, go-tree-sitter build issue) ----------------- -KATANA_VERSION="1.5.0" -DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)" -curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \ - -o /tmp/katana.zip -unzip -o /tmp/katana.zip -d /tmp/katana_extract -mv /tmp/katana_extract/katana "$HOME/.pdtm/go/bin/katana" -chmod +x "$HOME/.pdtm/go/bin/katana" -rm -rf /tmp/katana.zip /tmp/katana_extract +if ! have katana; then + KATANA_VERSION="1.5.0" + DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)" + mkdir -p "$HOME/.pdtm/go/bin" + curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \ + -o /tmp/katana.zip + unzip -o /tmp/katana.zip -d /tmp/katana_extract + mv /tmp/katana_extract/katana "$HOME/.pdtm/go/bin/katana" + chmod +x "$HOME/.pdtm/go/bin/katana" + rm -rf /tmp/katana.zip /tmp/katana_extract +fi # -- protoscope ------------------------------------------------------------ -go install github.com/protocolbuffers/protoscope/cmd/protoscope@latest +have protoscope || \ + go install "github.com/protocolbuffers/protoscope/cmd/protoscope@${GO_TOOL_VERSIONS_protoscope}" # -- interactsh-client ----------------------------------------------------- -go install github.com/projectdiscovery/interactsh/cmd/interactsh-client@latest +have interactsh-client || \ + go install "github.com/projectdiscovery/interactsh/cmd/interactsh-client@${GO_TOOL_VERSIONS_interactsh}" # -- 2fa (TOTP generator) -------------------------------------------------- -go install rsc.io/2fa@latest +have 2fa || go install "rsc.io/2fa@${GO_TOOL_VERSIONS_2fa}" # -- surf (SSRF target identification) ------------------------------------ -go install github.com/assetnote/surf/cmd/surf@latest +have surf || go install "github.com/assetnote/surf/cmd/surf@${GO_TOOL_VERSIONS_surf}" # -- kiterunner (API content discovery) ------------------------------------ if ! command -v kr &>/dev/null; then @@ -120,16 +197,25 @@ fi # Pro features require BURP_LICENSE_KEY at runtime. if [ ! -f /opt/burp/burpsuite.jar ]; then BURP_VERSION="2025.5" - mkdir -p /opt/burp - curl -fsSL "https://portswigger-cdn.net/burp/releases/download?product=community&version=${BURP_VERSION}&type=Jar" \ - -o /opt/burp/burpsuite.jar \ - || echo "WARN: Burp Suite download failed (check version), skipping" - # Wrapper script for convenience - cat > /usr/local/bin/burp <<'BURPEOF' + # /opt and /usr/local/bin are root-owned, and this script does not always run + # as root. Previously the unguarded `mkdir` aborted the entire provision under + # `set -e` on a non-root runtime, taking every tool below it down with it. + if as_root mkdir -p /opt/burp; then + as_root curl -fsSL "https://portswigger-cdn.net/burp/releases/download?product=community&version=${BURP_VERSION}&type=Jar" \ + -o /opt/burp/burpsuite.jar \ + || echo "WARN: Burp Suite download failed (check version), skipping" + # Only wrap a jar that actually arrived — a `burp` on PATH pointing at + # nothing is worse than no `burp` at all. + if [ -f /opt/burp/burpsuite.jar ]; then + as_root tee /usr/local/bin/burp >/dev/null <<'BURPEOF' #!/usr/bin/env bash exec java -jar /opt/burp/burpsuite.jar "$@" BURPEOF - chmod +x /usr/local/bin/burp + as_root chmod +x /usr/local/bin/burp + fi + else + echo "WARN: cannot create /opt/burp (requires root); skipping Burp Suite" + fi fi # -- jxscout ---------------------------------------------------------------- @@ -145,16 +231,27 @@ fi # -- exiftool (EXIF metadata manipulation) --------------------------------- if ! command -v exiftool &>/dev/null; then - apt-get install -y --no-install-recommends libimage-exiftool-perl + as_root apt-get install -y --no-install-recommends libimage-exiftool-perl \ + || echo "WARN: exiftool install failed, skipping" fi # -- Node.js + agent-browser ----------------------------------------------- if ! command -v node &>/dev/null; then - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - - apt-get install -y --no-install-recommends nodejs + curl -fsSL https://deb.nodesource.com/setup_22.x | as_root bash - \ + && as_root apt-get install -y --no-install-recommends nodejs \ + || echo "WARN: Node.js install failed, skipping" +fi +if ! have agent-browser; then + npm install -g agent-browser +fi +# `agent-browser install` downloads the browser binaries themselves. Guarded on +# its cache so a runtime that already has them makes no request, and left +# non-fatal because a disconnected deployment that cannot fetch a browser +# should still get the rest of this capability's tooling. +AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}" +if [ ! -d "$AGENT_BROWSER_CACHE" ]; then + agent-browser install || echo "WARN: agent-browser browser download failed, skipping" fi -npm install -g agent-browser -agent-browser install || true # -- caido-mode skill deps (Caido TypeScript SDK CLI) ----------------------- # The caido-mode skill bundles a tsx CLI built on @caido/sdk-client (caido-ts). @@ -162,7 +259,9 @@ agent-browser install || true # runtime. Path is relative to the capability root (CAPABILITY_ROOT if exported, # else the script's own location, which is /scripts). CAIDO_MODE_DIR="${CAPABILITY_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}/skills/caido-mode" -if [ -f "$CAIDO_MODE_DIR/package.json" ]; then +# Guarded on node_modules: without it this reaches the npm registry on every +# boot even when the dependencies are already installed. +if [ -f "$CAIDO_MODE_DIR/package.json" ] && [ ! -d "$CAIDO_MODE_DIR/node_modules" ]; then ( cd "$CAIDO_MODE_DIR" && npm install --no-audit --no-fund ) \ && echo "caido-mode skill deps installed (@caido/sdk-client / caido-ts)" \ || echo "WARN: caido-mode npm install failed, skipping" @@ -171,29 +270,37 @@ fi # -- ast-grep (AST-based code pattern search) --------------------------------- # Tree-sitter based structural code matching for JS/TS/HTML. Lightweight # alternative to semgrep for pattern matching (no taint analysis). -pip install --break-system-packages ast-grep-cli +have ast-grep || py_install ast-grep-cli || echo "WARN: ast-grep install failed, skipping" # -- waymore (Wayback Machine recon) ----------------------------------------- -pip install --break-system-packages waymore +have waymore || py_install waymore || echo "WARN: waymore install failed, skipping" # -- Pacu (AWS exploitation framework) ---------------------------------------- -pip install --break-system-packages pacu +have pacu || py_install pacu || echo "WARN: pacu install failed, skipping" # -- fireprox (AWS API Gateway IP rotation) --------------------------------- # Requires AWS credentials at runtime. Cloned to a predictable path so the # ip-rotation skill can reference it directly. FIREPROX_DIR="$HOME/git/fireprox" if [ ! -d "$FIREPROX_DIR" ]; then - git clone --depth 1 https://github.com/ustayready/fireprox "$FIREPROX_DIR" + # Requirements are installed only alongside a fresh clone. Re-running them on + # every boot re-resolves against PyPI for an environment that already + # satisfies them. + if git clone --depth 1 https://github.com/ustayready/fireprox "$FIREPROX_DIR"; then + py_install -r "$FIREPROX_DIR/requirements.txt" \ + || echo "WARN: fireprox requirements install failed, skipping" + else + echo "WARN: fireprox clone failed, skipping" + fi fi -pip install --break-system-packages -r "$FIREPROX_DIR/requirements.txt" # -- archivealchemist (malicious archive crafter) --------------------------- # Pure Python CLI for crafting Zip Slip, symlink, polyglot, and Unicode path # confusion archives. Cloned to a predictable path for the agent prompt. ARCHIVEALCHEMIST_DIR="$HOME/git/archivealchemist" if [ ! -d "$ARCHIVEALCHEMIST_DIR" ]; then - git clone --depth 1 https://github.com/avlidienbrunn/archivealchemist "$ARCHIVEALCHEMIST_DIR" + git clone --depth 1 https://github.com/avlidienbrunn/archivealchemist "$ARCHIVEALCHEMIST_DIR" \ + || echo "WARN: archivealchemist clone failed, skipping" fi # -- Clean up Go build cache ----------------------------------------------- diff --git a/capabilities/web-security/tests/test_caido_mode_skill.py b/capabilities/web-security/tests/test_caido_mode_skill.py index e8b5169..849171b 100644 --- a/capabilities/web-security/tests/test_caido_mode_skill.py +++ b/capabilities/web-security/tests/test_caido_mode_skill.py @@ -222,11 +222,18 @@ def test_resolves_skill_dir_from_capability_root(self) -> None: # skill dir is mounted, not baked into the image. assert ( 'CAIDO_MODE_DIR="${CAPABILITY_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}' - "/skills/caido-mode\"" in INSTALL_SCRIPT + '/skills/caido-mode"' in INSTALL_SCRIPT ) - def test_install_is_guarded_on_package_json(self) -> None: - assert 'if [ -f "$CAIDO_MODE_DIR/package.json" ]; then' in INSTALL_SCRIPT + def test_install_is_guarded_on_package_json_and_installed_deps(self) -> None: + # Guarded on both: package.json alone re-runs `npm install` on every + # boot, which reaches the registry even when the dependencies are + # already present — an outbound attempt a disconnected deployment + # cannot satisfy and does not need. + assert ( + 'if [ -f "$CAIDO_MODE_DIR/package.json" ] ' + '&& [ ! -d "$CAIDO_MODE_DIR/node_modules" ]; then' + ) in INSTALL_SCRIPT def test_install_failure_is_non_fatal(self) -> None: # A missing Node toolchain must not abort the whole provision run. @@ -404,9 +411,9 @@ def test_go_only_tools_are_marked_as_such(self) -> None: "caido_get_automate_entry", ): assert tool in self.SKILL - assert self.SKILL.index(tool) > go_section, ( - f"{tool} is a caido-go tool but appears before the caido-go section" - ) + assert ( + self.SKILL.index(tool) > go_section + ), f"{tool} is a caido-go tool but appears before the caido-go section" def test_documents_both_servers(self) -> None: assert "Two MCP servers, one instance" in self.SKILL diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py new file mode 100644 index 0000000..a84a016 --- /dev/null +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -0,0 +1,91 @@ +"""The install script must complete without reaching the network when its tools are present. + +Self-hosted deployments run with no route to the internet, and this script +executes on every sandbox boot where the capability changed — not just the +first. An unguarded download is therefore a repeated outbound attempt that +cannot succeed, and on a disconnected install one failed fetch aborts the whole +script under ``set -e``, taking the rest of the tooling with it. + +These pin the two properties that keep that from happening: every fetch is +guarded on the artefact it produces, and every version is pinned. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +INSTALL_SCRIPT = (ROOT / "scripts" / "install_tools.sh").read_text(encoding="utf-8") +LINES = INSTALL_SCRIPT.splitlines() + + +def _preceding_context(index: int, span: int = 6) -> str: + """The guard for a fetch sits on the same line or just above it.""" + return "\n".join(LINES[max(0, index - span) : index + 1]) + + +class TestVersionsArePinned: + def test_no_unpinned_go_installs(self) -> None: + # `go install ...@latest` re-resolves against the module proxy every + # run, so it reaches the network even when the binary is already + # present — and produces a different tool set on different days, which + # no SBOM can describe. + unpinned = [ + line.strip() + for line in LINES + if "@latest" in line and not line.strip().startswith("#") + ] + assert not unpinned, f"unpinned installs: {unpinned}" + + +class TestFetchesAreGuarded: + def test_every_go_install_is_guarded(self) -> None: + unguarded = [] + for i, line in enumerate(LINES): + if not line.strip().startswith(("go install", " go install")): + continue + if "have " not in _preceding_context(i): + unguarded.append(line.strip()) + assert not unguarded, f"unguarded go install: {unguarded}" + + def test_global_npm_install_is_guarded(self) -> None: + for i, line in enumerate(LINES): + if re.search(r"^\s*npm install -g", line): + assert "have " in _preceding_context( + i + ), f"unguarded global npm install at line {i + 1}: {line.strip()}" + + def test_pdtm_only_installs_missing_tools(self) -> None: + # `pdtm -install ` re-fetches every tool in the list. The + # set has to be narrowed to what is actually absent first. + assert "$missing_pd_tools" in INSTALL_SCRIPT + assert "-install nuclei,httpx" not in INSTALL_SCRIPT + + def test_katana_download_is_guarded(self) -> None: + idx = next( + i for i, line in enumerate(LINES) if "katana_${KATANA_VERSION}" in line + ) + assert "have katana" in _preceding_context(idx, span=8) + + def test_go_toolchain_is_only_fetched_when_something_needs_building(self) -> None: + # The toolchain is a ~150 MB download whose only purpose is building + # the tools above. A runtime that already carries them must never ask + # for it — which is what made the whole script abort at its first line + # on a disconnected install. + idx = next(i for i, line in enumerate(LINES) if "go.dev/dl/go" in line) + context = _preceding_context(idx, span=10) + assert "need_go" in context + + +class TestFailuresStayNonFatal: + def test_optional_vendor_downloads_do_not_abort_the_run(self) -> None: + # Burp, Caido and the browser binaries are not redistributable, so a + # disconnected deployment will not have them. That must degrade the + # capability, not take the rest of the tooling down with it. + for marker in ( + "WARN: Caido CLI install failed", + "WARN: Burp Suite download failed", + "WARN: agent-browser browser download failed", + ): + assert marker in INSTALL_SCRIPT, f"missing non-fatal fallback: {marker}" From bd6b790174f266d5a02e55d53258d4cd7e41de82 Mon Sep 17 00:00:00 2001 From: GangGreenTemperTatum <104169244+GangGreenTemperTatum@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:08:46 -0400 Subject: [PATCH 2/2] fix(web-security): as_root for all /usr/local/bin writes, expand offline guard tests. Guard py_install/curl/git-clone fetches, enforce as_root on every root-owned path write, gate go clean on need_go, and cover all 14 WARN fallback markers. --- .../web-security/scripts/install_tools.sh | 30 ++-- .../tests/test_install_tools_offline.py | 128 +++++++++++++++++- 2 files changed, 143 insertions(+), 15 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 1731a46..a0f71a7 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -131,12 +131,14 @@ have 2fa || go install "rsc.io/2fa@${GO_TOOL_VERSIONS_2fa}" have surf || go install "github.com/assetnote/surf/cmd/surf@${GO_TOOL_VERSIONS_surf}" # -- kiterunner (API content discovery) ------------------------------------ -if ! command -v kr &>/dev/null; then - git clone --depth 1 https://github.com/assetnote/kiterunner /tmp/kiterunner - cd /tmp/kiterunner && make build - mv /tmp/kiterunner/dist/kr /usr/local/bin/kr - rm -rf /tmp/kiterunner - cd - +if ! have kr; then + if git clone --depth 1 https://github.com/assetnote/kiterunner /tmp/kiterunner; then + ( cd /tmp/kiterunner && make build ) \ + && as_root mv /tmp/kiterunner/dist/kr /usr/local/bin/kr + rm -rf /tmp/kiterunner + else + echo "WARN: kiterunner clone failed, skipping" + fi fi # -- Caido CLI ------------------------------------------------------------- @@ -157,7 +159,7 @@ if ! command -v caido-cli &>/dev/null; then esac curl -fsSL "https://caido.download/releases/v${CAIDO_VERSION}/caido-cli-v${CAIDO_VERSION}-linux-${CAIDO_ARCH}.tar.gz" \ -o /tmp/caido-cli.tar.gz \ - && tar -xzf /tmp/caido-cli.tar.gz -C /usr/local/bin/ \ + && as_root tar -xzf /tmp/caido-cli.tar.gz -C /usr/local/bin/ \ && rm /tmp/caido-cli.tar.gz \ || echo "WARN: Caido CLI install failed (check version), skipping" fi @@ -181,7 +183,7 @@ if ! command -v caido-mcp-server &>/dev/null; then CAIDO_MCP_URL="https://github.com/c0tton-fluff/caido-mcp-server/releases/download/v${CAIDO_MCP_VERSION}/caido-mcp-server-linux-${CAIDO_MCP_ARCH}" if curl -fsSL "$CAIDO_MCP_URL" -o /tmp/caido-mcp-server; then if echo "${CAIDO_MCP_SHA256} /tmp/caido-mcp-server" | sha256sum -c - >/dev/null 2>&1; then - install -m 0755 /tmp/caido-mcp-server /usr/local/bin/caido-mcp-server + as_root install -m 0755 /tmp/caido-mcp-server /usr/local/bin/caido-mcp-server echo "caido-mcp-server v${CAIDO_MCP_VERSION} installed" else echo "WARN: caido-mcp-server checksum mismatch, skipping install" >&2 @@ -222,8 +224,9 @@ fi # Commercial binary — if JXSCOUT_BINARY_URL is set, download from there. # Otherwise skip; the MCP server falls back to PATH / ~/go/bin / ~/bin. if ! command -v jxscout-pro-v2 &>/dev/null && [ -n "${JXSCOUT_BINARY_URL:-}" ]; then - curl -fsSL "$JXSCOUT_BINARY_URL" -o /usr/local/bin/jxscout-pro-v2 - chmod +x /usr/local/bin/jxscout-pro-v2 + curl -fsSL "$JXSCOUT_BINARY_URL" -o /tmp/jxscout-pro-v2 \ + && as_root install -m 0755 /tmp/jxscout-pro-v2 /usr/local/bin/jxscout-pro-v2 + rm -f /tmp/jxscout-pro-v2 echo "jxscout installed from JXSCOUT_BINARY_URL" elif ! command -v jxscout-pro-v2 &>/dev/null; then echo "WARN: jxscout-pro-v2 not found. Set JXSCOUT_BINARY_URL to install, or place binary on PATH." @@ -242,7 +245,8 @@ if ! command -v node &>/dev/null; then || echo "WARN: Node.js install failed, skipping" fi if ! have agent-browser; then - npm install -g agent-browser + as_root npm install -g agent-browser \ + || echo "WARN: agent-browser install failed, skipping" fi # `agent-browser install` downloads the browser binaries themselves. Guarded on # its cache so a runtime that already has them makes no request, and left @@ -304,6 +308,8 @@ if [ ! -d "$ARCHIVEALCHEMIST_DIR" ]; then fi # -- Clean up Go build cache ----------------------------------------------- -go clean -cache -modcache 2>/dev/null || true +if [ "$need_go" = true ]; then + go clean -cache -modcache 2>/dev/null || true +fi echo "web-security tools installed successfully" diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index a84a016..e5274d2 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -25,6 +25,11 @@ def _preceding_context(index: int, span: int = 6) -> str: return "\n".join(LINES[max(0, index - span) : index + 1]) +def _surrounding_context(index: int, span: int = 4) -> str: + """Lines around a match — useful for checking as_root wrapping.""" + return "\n".join(LINES[max(0, index - span) : min(len(LINES), index + span + 1)]) + + class TestVersionsArePinned: def test_no_unpinned_go_installs(self) -> None: # `go install ...@latest` re-resolves against the module proxy every @@ -56,6 +61,26 @@ def test_global_npm_install_is_guarded(self) -> None: i ), f"unguarded global npm install at line {i + 1}: {line.strip()}" + def test_py_install_calls_are_guarded(self) -> None: + # Every `py_install` call must be preceded by a `have` check so that + # already-installed Python tools do not re-resolve against PyPI. + unguarded = [] + for i, line in enumerate(LINES): + stripped = line.strip() + if not stripped.startswith("py_install") and "py_install" not in stripped: + continue + # Skip the py_install function definition and requirement file + # installs (guarded by their parent clone check). + if ( + stripped.startswith(("if", "elif", "def", "#")) + or "-r " in stripped + or "py_install()" in stripped + ): + continue + if "have " not in _preceding_context(i): + unguarded.append(stripped) + assert not unguarded, f"unguarded py_install: {unguarded}" + def test_pdtm_only_installs_missing_tools(self) -> None: # `pdtm -install ` re-fetches every tool in the list. The # set has to be narrowed to what is actually absent first. @@ -68,6 +93,33 @@ def test_katana_download_is_guarded(self) -> None: ) assert "have katana" in _preceding_context(idx, span=8) + def test_caido_cli_download_is_guarded(self) -> None: + idx = next( + i for i, line in enumerate(LINES) if "caido.download/releases" in line + ) + assert "command -v caido-cli" in _preceding_context(idx, span=10) + + def test_caido_mcp_server_download_is_guarded(self) -> None: + idx = next( + i for i, line in enumerate(LINES) if "caido-mcp-server-linux" in line + ) + assert "command -v caido-mcp-server" in _preceding_context(idx, span=15) + + def test_kiterunner_build_is_guarded(self) -> None: + idx = next(i for i, line in enumerate(LINES) if "assetnote/kiterunner" in line) + assert "have kr" in _preceding_context(idx, span=4) + + def test_git_clones_are_guarded_on_target_dir(self) -> None: + # Clones to persistent paths (fireprox, archivealchemist) are guarded + # on the target directory existing. Clones to /tmp (kiterunner) are + # guarded on the binary they produce. + for i, line in enumerate(LINES): + if "git clone" not in line or line.strip().startswith("#"): + continue + ctx = _preceding_context(i, span=6) + has_dir_guard = "! -d " in ctx or "have " in ctx + assert has_dir_guard, f"unguarded git clone at line {i + 1}: {line.strip()}" + def test_go_toolchain_is_only_fetched_when_something_needs_building(self) -> None: # The toolchain is a ~150 MB download whose only purpose is building # the tools above. A runtime that already carries them must never ask @@ -77,15 +129,85 @@ def test_go_toolchain_is_only_fetched_when_something_needs_building(self) -> Non context = _preceding_context(idx, span=10) assert "need_go" in context + def test_go_cache_cleanup_only_runs_when_go_was_used(self) -> None: + idx = next(i for i, line in enumerate(LINES) if "go clean -cache" in line) + assert "need_go" in _preceding_context(idx, span=3) + + +class TestRootEscalation: + """Writes to root-owned paths (/usr/local/bin, /opt) must use as_root.""" + + def test_caido_cli_tar_uses_as_root(self) -> None: + idx = next( + i + for i, line in enumerate(LINES) + if "tar" in line and "caido-cli" in line and "/usr/local/bin" in line + ) + assert "as_root" in LINES[idx] + + def test_caido_mcp_server_install_uses_as_root(self) -> None: + idx = next( + i + for i, line in enumerate(LINES) + if "install -m" in line and "caido-mcp-server" in line + ) + assert "as_root" in LINES[idx] + + def test_kiterunner_mv_uses_as_root(self) -> None: + idx = next( + i + for i, line in enumerate(LINES) + if "/usr/local/bin/kr" in line and ("mv " in line or "install " in line) + ) + assert "as_root" in LINES[idx] + + def test_jxscout_install_uses_as_root(self) -> None: + idx = next( + i + for i, line in enumerate(LINES) + if "/usr/local/bin/jxscout" in line + and ("install " in line or "curl " not in line) + ) + assert "as_root" in LINES[idx] + + def test_burp_suite_uses_as_root(self) -> None: + idx = next(i for i, line in enumerate(LINES) if "mkdir -p /opt/burp" in line) + assert "as_root" in LINES[idx] + + def test_exiftool_apt_uses_as_root(self) -> None: + idx = next( + i + for i, line in enumerate(LINES) + if "apt-get" in line and "exiftool" in line + ) + assert "as_root" in LINES[idx] + + def test_nodejs_apt_uses_as_root(self) -> None: + idx = next( + i for i, line in enumerate(LINES) if "apt-get" in line and "nodejs" in line + ) + assert "as_root" in LINES[idx] + class TestFailuresStayNonFatal: def test_optional_vendor_downloads_do_not_abort_the_run(self) -> None: - # Burp, Caido and the browser binaries are not redistributable, so a - # disconnected deployment will not have them. That must degrade the - # capability, not take the rest of the tooling down with it. + # Vendor downloads, system packages, and optional tooling all degrade + # gracefully — a disconnected deployment must not have its entire + # provision aborted because one optional tool could not be fetched. for marker in ( "WARN: Caido CLI install failed", "WARN: Burp Suite download failed", "WARN: agent-browser browser download failed", + "WARN: exiftool install failed", + "WARN: Node.js install failed", + "WARN: kiterunner clone failed", + "WARN: fireprox clone failed", + "WARN: fireprox requirements install failed", + "WARN: archivealchemist clone failed", + "WARN: ast-grep install failed", + "WARN: waymore install failed", + "WARN: pacu install failed", + "WARN: agent-browser install failed", + "WARN: caido-mode npm install failed", ): assert marker in INSTALL_SCRIPT, f"missing non-fatal fallback: {marker}"