From 0c269055fdf7a90640f7d29e2a42a2a76020c199 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 13 Nov 2026 10:06:51 +0000 Subject: [PATCH 1/2] security: proven holders, per-site origins, sandboxed renders, signed installer extras (v0.3.65) Hosting: - a holder fills a replica slot only after passing a storage challenge (or a trusted receipt); announcements alone no longer stop a site being copied - giving up a copy needs three others with standing: three passes over six hours, so throwaway identities cannot trim honest holders quickly - a challenge is void when the challenged chunk was read from this node meanwhile, which catches a relay through the challenger; the last honest holder cannot be talked into dropping - the publisher waits for copies it has challenged itself Gateway: - hosted pages under /s/ are sandboxed into an opaque origin by default, and [gateway] site_host_suffix serves each site on its own host, where no other gateway route is reachable - storage API streams ranged reads block by block; delete keeps shards another object still names; shard indexes are bound-checked Workloads: - Blender: a constant script empties outside paths, removes linked data and File Output nodes and strips host stamps; on Linux the render runs inside bubblewrap when it works ([render] sandbox = auto | off | require) - apps: guarded egress network on root workers, a disk ceiling with a watchdog, state kept when an app is stopped for size - text jobs: base-model allowlist, validated job and hyperparameters, adapter cache filed under every file, free-space floor Installer and release: - bun is a pinned, checksummed download; mise is no longer installed - plugin installers and the TUI ship as signed release archives, verified by install.sh and by the CLI; desktop artifacts are signed - latest refuses to downgrade an installed binary; no sudo symlink into a user-writable directory - plugin install needs --unverified Co-Authored-By: Claude Fable 5.2 --- .github/workflows/release.yml | 113 ++- Cargo.lock | 34 +- Cargo.toml | 2 +- README.md | 4 +- SECURITY.md | 132 ++- apps/desktop/package.json | 2 +- apps/tui/package.json | 2 +- apps/web/package.json | 2 +- apps/web/src/app/docs/page.tsx | 8 +- apps/web/src/app/plugins/page.tsx | 2 +- .../web/src/app/releases/latest.json/route.ts | 2 +- docs/apps.md | 30 + docs/hosting.md | 98 +- docs/node-safety.md | 89 +- docs/private-storage.md | 13 +- node/crates/c0mpute-cli/src/main.rs | 422 +++++++-- node/crates/c0mpute-core/src/apps/mod.rs | 173 +++- node/crates/c0mpute-core/src/apps/runtime.rs | 317 ++++++- node/crates/c0mpute-core/src/config.rs | 41 + node/crates/c0mpute-core/src/container_run.rs | 35 +- .../c0mpute-core/src/hosting/edge_api.rs | 3 + node/crates/c0mpute-core/src/hosting/mod.rs | 693 +++++++++++++- .../crates/c0mpute-core/src/hosting/origin.rs | 318 +++++++ node/crates/c0mpute-core/src/hosting/proof.rs | 22 + node/crates/c0mpute-core/src/hosting/relay.rs | 243 +++++ .../c0mpute-core/src/hosting/replicas.rs | 370 +++++++- .../crates/c0mpute-core/src/hosting/stripe.rs | 61 +- node/crates/c0mpute-core/src/render.rs | 886 +++++++++++++++++- node/crates/c0mpute-core/src/supervisor.rs | 5 + node/crates/c0mpute-core/src/text.rs | 623 ++++++++++-- node/crates/c0mpute-core/tests/apps_e2e.rs | 6 + node/crates/c0mpute-gateway/src/lib.rs | 4 + node/crates/c0mpute-gateway/src/site_host.rs | 331 +++++++ .../crates/c0mpute-gateway/src/storage_api.rs | 138 ++- .../c0mpute-placement/src/distributed.rs | 68 +- node/crates/c0mpute-placement/src/repair.rs | 10 +- node/crates/c0mpute-store/src/lib.rs | 83 ++ node/crates/c0mpute-store/src/storage.rs | 446 ++++++++- package.json | 2 +- packages/captcha/package.json | 2 +- packages/shared/package.json | 2 +- plugins/ad-generator-agent/module.toml | 2 +- plugins/ads-manager/module.toml | 2 +- plugins/apps/module.toml | 2 +- plugins/captcha/module.toml | 2 +- plugins/coinpay/module.toml | 2 +- plugins/containers/module.toml | 2 +- plugins/crawl/module.toml | 2 +- plugins/crowdgpt/module.toml | 2 +- plugins/ctv-ads/module.toml | 2 +- plugins/diffusion/module.toml | 2 +- plugins/embed/module.toml | 2 +- plugins/gameserver/module.toml | 2 +- plugins/git/module.toml | 2 +- plugins/hosting/module.toml | 2 +- plugins/infernet/module.toml | 2 +- plugins/live-stream/module.toml | 2 +- plugins/music/module.toml | 2 +- plugins/ocr/module.toml | 2 +- plugins/podcast-ads/module.toml | 2 +- plugins/podcasting/module.toml | 2 +- plugins/podcasts/module.toml | 2 +- plugins/render/module.toml | 2 +- plugins/secure-chat/module.toml | 2 +- plugins/storage/module.toml | 2 +- plugins/threatcrush/module.toml | 2 +- plugins/train/module.toml | 2 +- plugins/transcode/module.toml | 2 +- plugins/tts/module.toml | 2 +- plugins/video/module.toml | 2 +- plugins/webhook/module.toml | 2 +- plugins/whisper/module.toml | 2 +- plugins/workflow-step/module.toml | 2 +- plugins/x402/module.toml | 2 +- scripts/install.sh | 613 ++++++++---- scripts/pack-release-extras.sh | 67 ++ 76 files changed, 6092 insertions(+), 491 deletions(-) create mode 100644 node/crates/c0mpute-core/src/hosting/origin.rs create mode 100644 node/crates/c0mpute-core/src/hosting/relay.rs create mode 100644 node/crates/c0mpute-gateway/src/site_host.rs create mode 100755 scripts/pack-release-extras.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 41323dc6..9c106fd4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,12 @@ name: release # production minisign key (repo secrets MINISIGN_SECRET_KEY + MINISIGN_PASSWORD; # public key minisign.pub) and uploads .minisig next to it, which # scripts/install.sh requires. A release is never published unsigned. +# The same goes for the two platform-independent archives the `extras` job +# packs from the tagged tree (scripts/pack-release-extras.sh): +# c0mpute-plugins.tar.gz (plugins//install.sh) and c0mpute-tui.tar.gz +# (apps/tui), which the installer and the CLI install from instead of fetching +# unsigned scripts. The desktop artifacts are built after publishing and signed +# by `desktop-sign`. # # A failed platform leaves the draft unpublished (Latest stays on the # previous release); fix and re-run the failed jobs. @@ -157,10 +163,49 @@ jobs: GH_TOKEN: ${{ github.token }} run: gh release upload "$TAG" "$ARTIFACT" "$ARTIFACT_SHA" --clobber + # Platform-independent archives the installer and the CLI install from: the + # plugin installers and the TUI source, packed from the tagged tree. Uploaded + # to the draft with a .sha256 like the binaries; `publish` signs them. + extras: + name: plugins + tui archives + needs: draft + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ env.TAG }} + path: tree + + # The packer comes from the commit this workflow runs at, so re-running + # an older tag (workflow_dispatch) from before the script existed still + # works. On a tag push that commit is the tag itself. + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + path: tools + sparse-checkout: scripts/pack-release-extras.sh + sparse-checkout-cone-mode: false + + - name: Pack and upload to the draft release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + sh tools/scripts/pack-release-extras.sh "$GITHUB_WORKSPACE/tree" "$RUNNER_TEMP/extras" + cd "$RUNNER_TEMP/extras" + sha256sum -c c0mpute-plugins.tar.gz.sha256 + sha256sum -c c0mpute-tui.tar.gz.sha256 + tar -tzf c0mpute-plugins.tar.gz + tar -tzf c0mpute-tui.tar.gz + gh release upload "$TAG" \ + c0mpute-plugins.tar.gz c0mpute-plugins.tar.gz.sha256 \ + c0mpute-tui.tar.gz c0mpute-tui.tar.gz.sha256 --clobber + publish: name: publish + mark latest - # Runs only when every platform job succeeded (needs + default success()). - needs: release + # Runs only when every platform job and the extras job succeeded (needs + + # default success()). + needs: [release, extras] runs-on: ubuntu-latest steps: # Only for minisign.pub, the public key every signature must verify against. @@ -200,7 +245,7 @@ jobs: work="$RUNNER_TEMP/sign"; mkdir -p "$work"; cd "$work" printf '%s\n' "$MINISIGN_SECRET_KEY" > "$RUNNER_TEMP/minisign.key" trap 'rm -f "$RUNNER_TEMP/minisign.key"' EXIT - want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip" + want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip c0mpute-plugins.tar.gz c0mpute-tui.tar.gz" for a in $want; do gh release download "$TAG" -p "$a" -p "$a.sha256" --clobber # Sign only what the platform build uploaded: check its checksum first. @@ -217,7 +262,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -eu - want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip" + want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip c0mpute-plugins.tar.gz c0mpute-tui.tar.gz" have=$(gh release view "$TAG" --json assets -q '.assets[].name') for a in $want; do for f in "$a" "$a.sha256" "$a.minisig"; do @@ -326,5 +371,65 @@ jobs: [ ${#files[@]} -gt 0 ] || { echo "no desktop artifacts built" >&2; ls -la; exit 1; } for f in "${files[@]}"; do if command -v sha256sum >/dev/null; then sha256sum "$f" > "$f.sha256"; else shasum -a 256 "$f" > "$f.sha256"; fi + # On a re-run the build differs from the one the old signature + # covers: drop it, `desktop-sign` uploads a fresh one. + gh release delete-asset "$TAG" "$f.minisig" -y >/dev/null 2>&1 || true gh release upload "$TAG" "$f" "$f.sha256" --clobber done + + # Sign the desktop artifacts with the same key as everything else. They are + # built after `publish` (a desktop failure must not hold back a CLI release), + # so this is its own job, and the key never enters a job that runs + # `bun install` or electron-builder. It signs whatever desktop builds are on + # the release, so one failed platform does not leave the others unsigned; + # after re-running a failed desktop job, re-run this one too. + # scripts/install.sh --desktop requires the AppImage's .minisig. + desktop-sign: + name: sign desktop artifacts + needs: [publish, desktop] + if: ${{ !cancelled() && needs.publish.result == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ env.TAG }} + sparse-checkout: minisign.pub + sparse-checkout-cone-mode: false + + - name: Install minisign (pinned) + run: | + set -eu + curl -fsSL -o "$RUNNER_TEMP/minisign.tar.gz" https://github.com/jedisct1/minisign/releases/download/0.12/minisign-0.12-linux.tar.gz + echo "9a599b48ba6eb7b1e80f12f36b94ceca7c00b7a5173c95c3efc88d9822957e73 $RUNNER_TEMP/minisign.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/minisign.tar.gz" -C "$RUNNER_TEMP" minisign-linux/x86_64/minisign + install -m 755 "$RUNNER_TEMP/minisign-linux/x86_64/minisign" /usr/local/bin/minisign 2>/dev/null \ + || sudo install -m 755 "$RUNNER_TEMP/minisign-linux/x86_64/minisign" /usr/local/bin/minisign + minisign -v + + - name: Sign desktop artifacts and upload .minisig files + env: + GH_TOKEN: ${{ github.token }} + MINISIGN_SECRET_KEY: ${{ secrets.MINISIGN_SECRET_KEY }} + MINISIGN_PASSWORD: ${{ secrets.MINISIGN_PASSWORD }} + run: | + set -eu + [ -n "$MINISIGN_SECRET_KEY" ] && [ -n "$MINISIGN_PASSWORD" ] || { echo "MINISIGN_SECRET_KEY / MINISIGN_PASSWORD secrets missing; cannot sign" >&2; exit 1; } + umask 077 + pub="$GITHUB_WORKSPACE/minisign.pub" + [ -s "$pub" ] || { echo "minisign.pub missing at $TAG; not signing" >&2; exit 1; } + work="$RUNNER_TEMP/sign-desktop"; mkdir -p "$work"; cd "$work" + printf '%s\n' "$MINISIGN_SECRET_KEY" > "$RUNNER_TEMP/minisign.key" + trap 'rm -f "$RUNNER_TEMP/minisign.key"' EXIT + names=$(gh release view "$TAG" --json assets -q '.assets[].name' \ + | grep -E '^c0mpute-desktop-[A-Za-z0-9._-]+\.(AppImage|deb|dmg|zip|exe)$' || true) + [ -n "$names" ] || { echo "no desktop artifacts on $TAG; nothing to sign" >&2; exit 1; } + for a in $names; do + gh release download "$TAG" -p "$a" -p "$a.sha256" --clobber + # Sign only what the desktop build uploaded: check its checksum first. + sha256sum -c "$a.sha256" + printf '%s\n' "$MINISIGN_PASSWORD" | minisign -S -s "$RUNNER_TEMP/minisign.key" -m "$a" \ + -t "timestamp:$(date +%s) version:$TAG file:$a" >/dev/null + minisign -V -p "$pub" -m "$a" + gh release upload "$TAG" "$a.minisig" --clobber + rm -f "$a" "$a.sha256" "$a.minisig" + done diff --git a/Cargo.lock b/Cargo.lock index 74aacb8b..f6313642 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "c0mpute-proto", @@ -501,7 +501,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "blake3", @@ -512,7 +512,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "async-trait", @@ -549,7 +549,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.3.64" +version = "0.3.65" dependencies = [ "aes-gcm", "anyhow", @@ -603,7 +603,7 @@ dependencies = [ [[package]] name = "c0mpute-doctor" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "serde", @@ -615,7 +615,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.3.64" +version = "0.3.65" dependencies = [ "blake3", "bs58", @@ -628,7 +628,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "axum", @@ -650,7 +650,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.3.64" +version = "0.3.65" dependencies = [ "c0mpute-envelope", "serde", @@ -660,7 +660,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "async-trait", @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "c0mpute-placement" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "async-trait", @@ -702,7 +702,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.3.64" +version = "0.3.65" dependencies = [ "blake3", "hex", @@ -714,7 +714,7 @@ dependencies = [ [[package]] name = "c0mpute-sealed" -version = "0.3.64" +version = "0.3.65" dependencies = [ "aes", "aes-gcm", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.3.64" +version = "0.3.65" dependencies = [ "aes-gcm", "anyhow", @@ -759,7 +759,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "blake3", @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "c0mpute-proto", @@ -791,7 +791,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "hex", @@ -806,7 +806,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.3.64" +version = "0.3.65" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index 73d5ed2f..8ed9cd48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ members = [ ] [workspace.package] -version = "0.3.64" +version = "0.3.65" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/README.md b/README.md index 122f3b4a..d29b57f2 100644 --- a/README.md +++ b/README.md @@ -508,7 +508,9 @@ cargo build --bin c0mpute - `c0mpute` Rust binary builds; clap surface for `doctor`, `worker`, `job`, `plugin`, `transcode`, `coinpay` (passthrough), `infernet` (passthrough), `tui`, `version` -- `c0mpute plugin install ` chain-calls upstream installers +- `c0mpute plugin install ` installs a first-party plugin from the signed + release archive; `c0mpute plugin install --unverified ` runs a + third-party installer - `c0mpute doctor` cross-checks `coinpay` and `infernet` on PATH - `c0mpute bench` runs fib / matmul / blake3 at 1…N threads, writes `bench.json` (same `metadata` + `results` layout as diff --git a/SECURITY.md b/SECURITY.md index f0f5215a..f56af0c2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,9 +8,16 @@ rather than a public issue. ## Release signing -Every artifact on a [c0mpute release](https://github.com/profullstack/c0mpute/releases) -(`c0mpute--.tar.gz`, `c0mpute-windows-x86_64.zip`) has a -[minisign](https://jedisct1.github.io/minisign/) signature next to it, `.minisig`. +Every artifact a [c0mpute release](https://github.com/profullstack/c0mpute/releases) is +installed from has a [minisign](https://jedisct1.github.io/minisign/) signature next to it, +`.minisig`: + +| Artifact | What it is | +| --- | --- | +| `c0mpute--.tar.gz`, `c0mpute-windows-x86_64.zip` | the `c0mpute` binary | +| `c0mpute-plugins.tar.gz` | every `plugins//install.sh` (and its `module.toml`) of that tag | +| `c0mpute-tui.tar.gz` | the terminal UI source (`apps/tui`: `src/`, `package.json`, `tsconfig.json`) | +| `c0mpute-desktop-*.{AppImage,deb,dmg,zip,exe}` | the desktop app builds | Public key ([`minisign.pub`](minisign.pub), key id `01A1B54137EC392A`): @@ -21,26 +28,111 @@ RWQqOew3QbWhAWH36TA5roreh18RgRqgozSwhE3yZtdza7eeLKmbDbJM The signed (trusted) comment reads `timestamp: version: file:`, so a signature only vouches for that one file of that one release. -What checks it: +### What signs + +**The release workflow** (`.github/workflows/release.yml`). The platform jobs build the +binaries and the `extras` job packs the two archives from the tagged tree +(`scripts/pack-release-extras.sh`; run it yourself to see exactly what ships). The `publish` +job signs all of them and refuses to publish a release that is missing any binary, either +archive, or any of their `.minisig` files. The desktop builds are made after the release is +public and are signed by the `desktop-sign` job a few minutes later; until then they have no +signature and the installer will not install them. + +### What verifies + +- **`scripts/install.sh`** (served at `https://c0mpute.com/install.sh`): + - **The binary.** Downloads the artifact and its `.minisig`, verifies the signature with + the embedded key and checks that the signed comment names the artifact (and the release + tag when one was asked for). A missing signature, a failed verification or a mismatched + comment aborts the install. + - **Which release.** Everything else in the run comes from the release named in the + binary's signed comment (or, when the binary is already installed, from that binary's own + release), and each signature must be for that tag. + - **Plugin installers** (coinpay, infernet, transcode) run from the verified + `c0mpute-plugins.tar.gz`, and **the TUI** is installed from the verified + `c0mpute-tui.tar.gz`. If an archive is on the release, a missing or bad signature aborts + the install. + - **The desktop AppImage** (`--desktop`) is verified the same way. No signature means it + is not installed. + - **Releases up to v0.3.64** predate these signed extras. Only for those, and only when + the release really does not have the asset (HTTP 404), the installer falls back to the + old downloads (`https://c0mpute.com/plugins//install.sh`, the GitHub source tarball, + the AppImage's `.sha256`) and prints a warning that they are unverified. On any newer + release a missing extra is an error, so hiding the asset cannot force the fallback. + - **minisign.** If `minisign` is not installed the script downloads the pinned upstream + minisign 0.12 release and checks it against a hard-coded sha256 before running it (Linux + x86_64/aarch64, macOS arm64; on Intel macs install minisign with `brew install minisign`). + - **bun** (the runtime the TUI needs; only installed with the TUI). The script downloads + the pinned upstream bun 1.3.14 release zip for the OS/arch and checks it against a + hard-coded sha256 before unpacking it. It no longer pipes `bun.sh/install` to a shell, and + it no longer installs mise at all (a contributor tool, not needed on a node). A platform + without a pinned hash gets a message and no bun. +- **`c0mpute` self-update** (`node/crates/c0mpute-update`) verifies the binary's signature + with the key compiled into the binary, after the sha256 check and before swapping the + binary. No signature or a bad one means no update. +- **`c0mpute tui`** and **`c0mpute plugin install `** download `c0mpute-tui.tar.gz` / + `c0mpute-plugins.tar.gz` of the running binary's own release and verify them with that same + compiled-in key (file and release tag included) before anything is unpacked or run. There + is no unverified fallback. + +`C0MPUTE_ALLOW_UNSIGNED=1` makes the installer accept a missing signature. It never accepts +a bad one. + +### Rollback: what `latest` does and does not promise + +A signature proves an artifact is a real c0mpute release. It does not prove it is the newest +one: whoever answers for `https://c0mpute.com/releases/latest/` (a redirect to GitHub's +latest release) could serve an older, genuinely signed release with a known bug. -- **`scripts/install.sh`** (served at `https://c0mpute.com/install.sh`) downloads the - artifact and its `.minisig`, verifies the signature with the embedded key and checks the - trusted comment names the artifact (and the release tag when it is known). It aborts on a - missing signature, a failed verification or a mismatched comment. If `minisign` is not - installed it downloads the pinned upstream minisign 0.12 release and checks it against a - hard-coded sha256 before running it (Linux x86_64/aarch64, macOS arm64; on Intel macs - install minisign with `brew install minisign`). -- **`c0mpute` self-update** (`node/crates/c0mpute-update`) verifies the same signature with - the key compiled into the binary, after the sha256 check and before swapping the binary. - No signature or a bad one means no update. -- **The release workflow** (`.github/workflows/release.yml`) signs in its `publish` job and - refuses to publish a release that is missing any `.minisig`. +What the installer does about it, without trusting a second server: it reads the release +version out of the signed comment, prints it (`installing c0mpute v`), and when you +did not ask for a specific version it refuses to install a release older than the `c0mpute` +already on the machine. `C0MPUTE_VERSION=v` installs an older release on purpose. +Self-update likewise only moves to a higher version. -Verify a download by hand: +What that does not prevent: + +- A **first install** has nothing to compare against, so it can still be handed an older + signed release. Check the printed version against the + [releases page](https://github.com/profullstack/c0mpute/releases) if that matters to you. +- Being **held at the current version** (served the release you already have, never the + newer one). + +### What is still not verified + +- **`install.sh` itself** is fetched over HTTPS from c0mpute.com; read it before piping it + to `sh` if you prefer (`curl -fsSL https://c0mpute.com/install.sh | less`). +- **What the plugin installers fetch.** The signed `plugins/coinpay/install.sh` and + `plugins/infernet/install.sh` are thin wrappers that download and run the upstream + installers (`https://coinpayportal.com/install.sh`, `https://infernetprotocol.com/install.sh`) + over HTTPS; `plugins/threatcrush/install.sh` does the same for ThreatCrush. Those upstream + scripts and what they install are not covered by the c0mpute signature. The transcode + installer installs ffmpeg through your system package manager. The video installer pulls a + container image pinned by digest, and builds from a clone of the repository's default + branch if that pull fails. +- **The TUI's npm dependencies.** The TUI source is signed, but `bun install` resolves its + dependencies (`@profullstack/hqtui` and what it pulls in) from the npm registry at install + time, without a lockfile. +- **`c0mpute plugin install --unverified `** runs somebody else's script. It + refuses without the flag and warns with it. +- **`C0MPUTE_TUI_REF=`** (a developer override) installs the TUI from a branch + tarball, with a warning. +- **The desktop app's own updater** and the macOS/Windows builds' platform code signing + (there are no certificates yet). + +### PATH, not root + +The installer puts `~/.c0mpute/bin` on your PATH through your shell's rc file and prints the +line. It does not symlink the binary into `/usr/local/bin` with sudo any more: that put a +program your account can rewrite on root's PATH. If an earlier install left such a symlink, +the installer tells you; remove it with `sudo rm /usr/local/bin/c0mpute +/usr/local/bin/git-remote-c0mpute`. (Run as root, the installer still links there, since root +then owns the binary.) It does still write `/etc/profile.d/c0mpute.sh`, using sudo when it is +available: one `export PATH="$HOME/.c0mpute/bin:$HOME/.local/bin:$PATH"` line, which for +every account names that account's own home directory. + +### Verify a download by hand ```sh minisign -Vm c0mpute-linux-x86_64.tar.gz -P RWQqOew3QbWhAWH36TA5roreh18RgRqgozSwhE3yZtdza7eeLKmbDbJM ``` - -The installer script itself is fetched over HTTPS from c0mpute.com; read it before piping it -to `sh` if you prefer (`curl -fsSL https://c0mpute.com/install.sh | less`). diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 7b04d4ee..711f5bff 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -2,7 +2,7 @@ "name": "@c0mpute/desktop", "productName": "c0mpute", "desktopName": "c0mpute-desktop.desktop", - "version": "0.3.39", + "version": "0.3.65", "private": true, "type": "module", "description": "c0mpute desktop: chat, agent with your MCP servers and skills, node manager, hosting and tasks, with the c0mpute CLI bundled.", diff --git a/apps/tui/package.json b/apps/tui/package.json index 2e55b6da..d5c29373 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.3.39", + "version": "0.3.65", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index 3189fd0e..63b5e321 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.3.39", + "version": "0.3.65", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/docs/page.tsx b/apps/web/src/app/docs/page.tsx index 2839dc59..d40854a6 100644 --- a/apps/web/src/app/docs/page.tsx +++ b/apps/web/src/app/docs/page.tsx @@ -411,12 +411,12 @@ c0mpute tui # terminal app: chat, jobs, node, network`}
-{`# Install from a plugin's signed install.sh -c0mpute plugin install https://example.com/my-plugin/install.sh - -# Install by id from the c0mpute marketplace +{`# Install a first-party plugin by id (from the signed release archive) c0mpute plugin install my-plugin +# Install a third-party plugin from its install.sh (not verified by c0mpute) +c0mpute plugin install --unverified https://example.com/my-plugin/install.sh + # List installed c0mpute plugin list diff --git a/apps/web/src/app/plugins/page.tsx b/apps/web/src/app/plugins/page.tsx index cdc56825..1cc81241 100644 --- a/apps/web/src/app/plugins/page.tsx +++ b/apps/web/src/app/plugins/page.tsx @@ -43,7 +43,7 @@ export default function PluginsPage() {

Third-party plugins install with{" "} - c0mpute plugin install <url> where{" "} + c0mpute plugin install --unverified <url> where{" "} <url> points at the plugin's{" "} install.sh. See{" "} diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index 9a2241eb..ada47fcc 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.3.39"; +const FALLBACK_VERSION = "0.3.65"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/docs/apps.md b/docs/apps.md index 8c721aee..3d206590 100644 --- a/docs/apps.md +++ b/docs/apps.md @@ -62,6 +62,8 @@ NODE_ENV = "production" | Secrets | Secret env plus the app key travel sealed to each holder's X25519 key (`Seal` gossip): from the owner's CLI at deploy, then from any holder to newcomers, so the owner can go offline. The record carries only their blake3. | | Serving | Gateways forward to the primary over the `apps.http` libp2p RPC, so NAT'd primaries work. Requests up to 700 KiB, responses up to 5 MiB, 15 s upstream timeout, no websockets. The app sees `X-Forwarded-For/Host/Proto` and `X-Forwarded-Prefix: /a/`. | | Sandbox | `--cap-drop ALL` (plus CHOWN, SETUID, SETGID, DAC_OVERRIDE, FOWNER, NET_BIND_SERVICE), `no-new-privileges`, memory/CPU/pid limits, 64 MiB tmpfs `/tmp`, published on loopback only, no restart policy (the supervisor decides what runs). | +| Network | On a root worker the container joins the `c0mpute-apps` network behind an egress guard: it reaches the public internet but not private, loopback, link-local (cloud metadata) or CGNAT addresses, and not the host. On a non-root worker there is no guard (see below). | +| Disk | State up to 1 GiB, and by default 2 GiB for state plus everything written inside the container. An app over the limit is stopped and restarts from its last snapshot. | ## Custom domains @@ -86,10 +88,38 @@ c0mpute worker start --roles ...,apps [apps] max_apps = 4 # held at once (primary or standby) max_memory_mb = 2048 # reserved by held apps together +owners = [] # owner keys whose apps this node runs; empty = anyone's +egress_guard = true # keep apps off your LAN, the host and metadata addresses +require_egress_guard = false # true: run no apps unless that guard is in place +max_disk_mb = 2048 # per app: state volume plus writes inside the container ``` Every gateway routes to apps whether or not it runs any. +**Whose apps.** With `owners` empty the node runs containers for anyone on +the network, and says so in its log at start. + +**Network.** The egress guard is a set of iptables rules on the apps bridge +(`c0apps0`), the same ones `container.run` uses plus one that lets the app +answer the node's own proxy. Only a root worker can add them. A non-root +worker keeps apps on docker's default bridge, where a container **can reach +your LAN, the host and cloud metadata addresses**; the node logs that once at +start. Set `require_egress_guard = true` to run no apps in that case. An app +that must reach a private address (a database on your LAN) needs +`egress_guard = false`. The guard is IPv4 only. + +**Disk.** Every minute the node asks docker how much a running app has written +inside its container and adds the size of its last state snapshot. Over +`max_disk_mb` (0 = no per-app limit), or with state over 1 GiB, the app is +stopped, the reason is recorded (`error` in `GET /apps/v1/apps/` on that +node, and its log), and another holder takes over from the last snapshot. The +stopped app's state volume is left on the node: what it holds past the last +snapshot exists nowhere else, so removing it is the owner's decision. Apps are also stopped, and +none started, while the docker disk has less free than the smaller of 20 GiB +and 10% (the floor `container.run` keeps). Where docker can enforce a quota +itself (overlay2 on xfs with project quotas) the container is also created +with `--storage-opt size=`. + ## Limits, honestly - **One writer.** A stateful app has a single primary. A healed partition can diff --git a/docs/hosting.md b/docs/hosting.md index dcb7f666..0de97d6b 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -42,6 +42,41 @@ Git repositories ride on the same machinery: see [git hosting](git-hosting.md). | Healing | A site with fewer than 3 live copies: each hosting node ranks the candidates (rendezvous hashing on site id + peer id). The top-ranked ones fetch a copy. A node that stops announcing stops counting after ~200 s and another node takes over. Above 5 copies the lowest-ranked holder lets go. A publisher seeding its own site is a source, not a copy. | | Serving | Any gateway: local chunks when it holds them, otherwise fetched over libp2p from the fastest live holder (measured round-trip, round-robin among equally fast ones), failing over to the next and then to the DHT. Every chunk is checked against its hash. | +### An announced copy counts once it is proven + +Announcing a copy is free: any key can gossip "I hold this site". So an announcement +alone does not fill one of the three copies, and does not let anyone drop theirs. + +- **Filling.** A holder counts toward the three once the node deciding has seen it pass + a challenge (a random chunk, fetched straight from that peer and checked against its + hash) within the last 6 hours, or the site's owner or payer signed a passing receipt + for it in that time. Only the nodes next in line for the site ask (the same top three + on every node, by the rendezvous ranking), in the background, at most 32 challenges + per repair round, 4 per peer and 5 per site. While they ask, the announced holders + keep their slots for one holder TTL (200 s), so slow or briefly unreachable honest + holders are asked again each round and a healthy site gains no extra copy. After + that, holders that still have not proven anything stop counting and the next-ranked + nodes take real copies. Keys that only announce can delay a site's copies by those + 200 s, once; new keys do not restart the clock. A node that has just started knows no + proofs: it works through its sites at the rate above and takes announced holders at + their word until a site's turn comes. +- **Dropping.** A node lets a surplus copy go only when at least three other holders + prove their copy to it at that moment, and each of them has passed its challenges at + 3 or more separate times over at least 6 hours. A surplus copy therefore stays for at + least 6 hours after it appears. +- **Relays.** A peer with no copy can pass a challenge by fetching the chunk from a real + holder while it is being asked. If the holder it fetches from is the challenger, the + challenger sees the chunk being read from its own store during the challenge and does + not count the answer. A holder also has a rule ready that refuses chunk fetches from + a peer that announces the same site whole (such a peer has no reason to fetch it); + it takes effect once the node's chunk server passes the requesting peer to hosting. + What remains: a peer can relay from a holder other than the challenger, under a + second key. That costs it the bandwidth of every challenge, for as long as it wants + to be counted. It can thin a site's honest copies over hours, down to one honest + holder (which cannot be talked into dropping by peers that hold nothing), but not to + none. A peer that really stores the site for 6 hours and then deletes it is not a + relay and is not caught by any of this. + ## Limits | | | @@ -152,6 +187,65 @@ the rate (`--sequential` fetches the old way, one chunk at a time, to compare). `--roles` without `hosting` means the node holds no copies; a gateway still serves sites by proxying. +## Site origins + +A browser treats everything under one scheme, host and port as one origin. Served at +`/s//`, every site would share the gateway's origin with every other +site and with the gateway's own APIs: one site's script could read another site's +localStorage and call those APIs as the same origin. Two settings keep sites apart: + +```toml +# ~/.config/c0mpute/config.toml +[gateway] +sandbox_shared_sites = true # the default +site_host_suffix = "sites.example.com" # unset by default +``` + +**`sandbox_shared_sites`** (default `true`, `C0MPUTE_SANDBOX_SHARED_SITES=0` to turn off). +Pages (HTML, SVG, XML) under `/s//` are sent with + +``` +Content-Security-Policy: sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads allow-pointer-lock allow-presentation +Referrer-Policy: no-referrer +``` + +so each page runs in an origin of its own. Scripts, forms and downloads work. +**localStorage, sessionStorage, IndexedDB, cookies and service workers do not** under +`/s/`, and a page's `fetch()` of its own files is a cross-origin request, which is why +every `/s/` response also carries `Access-Control-Allow-Origin: *` and +`Cross-Origin-Resource-Policy: cross-origin` (everything under `/s/` is public and +served without credentials). A site that needs storage or a service worker works on a +custom domain or a per-site host, where none of these headers are sent. Turning the +setting off restores the old behaviour, shared origin included. + +**`site_host_suffix`** (unset by default, `C0MPUTE_SITE_HOST_SUFFIX` overrides). With +`site_host_suffix = "sites.example.com"`: + +- `https://.sites.example.com/` serves that site at `/`, as an ordinary page + with its own origin. +- On such a host nothing else is served: every other gateway route (storage API, git, + swarm, `/p/`, `/hosting/v1/...`, another site's `/s/...`) is not reachable there, and + anything but GET or HEAD answers 404. +- A browser opening `https:///s//?` is redirected (307, + cacheable for 5 minutes) to `https://.sites.example.com/?`. + Requests that are not page navigations (podcast apps, feed readers, `curl`, the CLI, + the `/p/` loader reading its ciphertext) are answered under `/s/` as before, with the + sandbox headers above. + +The operator has to provide, before setting it: + +1. **Wildcard DNS**: `*.sites.example.com` (A/AAAA or CNAME) pointing at the gateway. + Site ids are 20 hex characters, one DNS label. +2. **Wildcard TLS**: a certificate for `*.sites.example.com` on the proxy in front of + `:7777` (a DNS-01 ACME challenge; HTTP-01 cannot issue wildcards). The redirect + always goes to `https://`. +3. **The Host header** passed through by that proxy unchanged (the same requirement as + custom domains). + +Use a suffix that serves nothing else, ideally a separate registrable domain from the +gateway's own name (as with user content elsewhere), so that cookies set for the +gateway's domain are never sent to site hosts. + ## Custom domains 1. Publish with `--domain blog.example.org` (repeatable; part of the signed record). @@ -254,7 +348,9 @@ cannot demote anybody. The site's copies are kept up by the normal fill meanwhil This proves the peer returned the right bytes for a chunk it could not predict, within the timeout. It does not prove the peer keeps its own copy rather than relaying from another -holder. +holder; see "An announced copy counts once it is proven" above for what is done about +relays and what is not. A challenger that sees the challenged chunk read from its own +store while it waits for the answer signs no receipt for that challenge. ### Pricing and settlement diff --git a/docs/node-safety.md b/docs/node-safety.md index 4f206cff..fce1ed7d 100644 --- a/docs/node-safety.md +++ b/docs/node-safety.md @@ -34,10 +34,10 @@ open an issue; this page is meant to be checkable against the code. Container workloads (video, game servers) start only pinned, well-known images, and only when Docker or Podman already works on the box. - Jobs run as child processes of the worker with CPU-thread and GPU limits. - **This is not a sandbox.** A job runs with your user's permissions, so a - bug in a workload runtime could read files your user can read. If that is - not acceptable for your main machine, run the node in a VM, a container or - on a spare box. It earns the same there. + **This is not a sandbox** (Blender renders are the exception, below). A job + runs with your user's permissions, so a bug in a workload runtime could read + files your user can read. If that is not acceptable for your main machine, + run the node in a VM, a container or on a spare box. It earns the same there. - The node keeps itself up to date (`C0MPUTE_NO_AUTO_UPDATE=1` turns that off). From v0.2.91 on, updates are verified against the same minisign key before the binary is swapped; a node on an older version checks only the sha256 until it @@ -45,6 +45,69 @@ open an issue; this page is meant to be checkable against the code. - `c0mpute plugin install` still runs the plugin's own install script without a signature check. Only install plugins you trust. +## Blender renders + +`render` is a default role on any machine with `blender` installed. A buyer +sends a `.blend` scene, and a scene is more than pixels: it names files by +path (textures, movie clips, fonts, linked libraries, caches) and can ask +Blender to write files. The node does two things about that +(source: [`render.rs`](../node/crates/c0mpute-core/src/render.rs)). + +**Scene sanitisation, always, on every OS.** Blender starts with +`--disable-autoexec`, so no Python in the scene runs. After the file loads and +before the render, the node runs its own fixed script (`SANITIZE_EXPR`; nothing +from the job is written into it) that: + +- empties every file path that is absolute or leaves the job directory: images, + movie clips, sounds, fonts, volumes, Alembic/USD cache files, sequencer + strips and their proxies, point caches, shader nodes that read a file; +- deletes everything that came from a linked library outside the job directory; +- removes compositor File Output nodes, turns Open Shading Language off (a + compiled shader can open any image by name), and points fluid caches and + every scene's output at the job directory; +- turns off the render stamps that would write this machine's hostname or the + scene's path on your disk into the image. + +Data **packed** into the `.blend` is untouched, and an honest scene renders the +same pixels as before. A scene that needed a file it did not pack renders with +Blender's pink "missing" texture (or the built-in font, or without that strip): +buyers must pack their assets. If the script cannot finish, the frame is not +rendered at all. + +**A sandbox, on Linux, where bubblewrap works.** At start the node tries to run +Blender inside [bubblewrap](https://github.com/containers/bubblewrap) (`bwrap`) +and, if that works, renders that way: no network, its own process and hostname +namespaces, a read-only view of the system directories and of Blender's own +install directory, an empty private `/tmp` and home directory, and the job +directory as the only thing it can write. GPU device nodes (`/dev/dri`, +`/dev/nvidia*`) and driver libraries stay visible so GPU renders keep working; +a GPU job the sandbox cannot serve (the probe checks) runs outside it. +`c0mpute` logs which one it is at start. Install it with +`sudo apt install bubblewrap`. + +| | sanitisation only | with bubblewrap | +|---|---|---| +| Read a file the scene names by path into a frame | blocked by the script, for every kind of path listed above | blocked twice (the file is not there) | +| Write outside the job directory when rendering | blocked by the script, for the ways listed above (output path, File Output nodes, fluid caches) | not possible | +| What Blender does while **opening** the file, before the script runs: open a linked library by path (its data is deleted before rendering), create an empty cache directory where the scene says | **not confined** | confined | +| A bug in Blender's file readers turned into code execution | **not confined**: runs as your user | no network, no home directory, read-only system; it can still read world-readable system files and use the GPU | +| CPU, RAM and disk the render uses | thread limit and timeout only | the same | +| macOS, Windows | this column | not available | + +The sandbox is not used when `bwrap` is missing, when unprivileged user +namespaces are switched off (Ubuntu 24.04 does this with AppArmor unless +bubblewrap's profile is installed), or when Blender is a snap or sits behind a +wrapper script the sandbox cannot follow. Choose with: + +```toml +[render] +sandbox = "auto" # default: bubblewrap when it works, else sanitisation only +# sandbox = "require" # serve no render (and advertise none) without the sandbox +# sandbox = "off" # never use bubblewrap +``` + +`C0MPUTE_RENDER_SANDBOX` overrides the file. + ## Models it serves - **House models** (on by default): the models Infernet lists at @@ -53,6 +116,24 @@ open an issue; this page is meant to be checkable against the code. `infernet house-models off`, or `--no-house-models` at install. - **Uncensored house models are off unless you opt in** (`--uncensored-models` at install, or `infernet house-models uncensored on`). +- **Text fine-tunes** (`c0mpute infernet text serve`, opt-in) download the + buyer's base model from Hugging Face and load it. A worker does that only + for the models you list; a job naming another gets no bid: + + ```toml + [text] + # The default: the two small models c0mpute's own commands and docs use. + base_models = ["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"] + # "org/*" allows every model of one organisation, "*" any model. + min_free_gb = 0 # free disk kept; 0 = the smaller of 20 GiB and 10% of the disk + ``` + + A job config may set only the keys c0mpute's clients send (`name`, `kind`, + `base_model`, `system_prompt`, `limits.timeout_s` and the hyperparameters + `lr`, `batch_size`, `epochs`, `seq_len`, `lora_r`, `max_steps`, `seed`, each + within a fixed range); anything else is refused before the trainer sees it. + The downloaded models live in Infernet's Hugging Face cache, which c0mpute + does not size or clean. ## Content policy (images and video) diff --git a/docs/private-storage.md b/docs/private-storage.md index 9a2fef75..12fa3063 100644 --- a/docs/private-storage.md +++ b/docs/private-storage.md @@ -107,10 +107,15 @@ open (any gateway serves `/s//o`), exactly like public objects. and refuses older ones, so a node cannot quietly undo a revocation or a volume write on a machine that saw the newer version. A machine that never saw it cannot tell, and the browser loader does not keep this memory. -- **The browser loader shares an origin** with the public sites that gateway - serves. The key stays in the fragment and the decrypted page is sandboxed, - but a gateway you do not trust can serve a different loader page; for - anything sensitive, `c0mpute storage get` is the stronger path. +- **The browser loader is served from the gateway's own origin**, the one + public sites under `/s/` are served from too. With the default + `[gateway] sandbox_shared_sites = true` those public pages are sandboxed + into origins of their own and cannot script the loader's origin; an + operator who turns that off puts them back on the loader's origin (see + [hosting](hosting.md#site-origins)). The key stays in the fragment and + the decrypted page is sandboxed either way, but a gateway you do not + trust can serve a different loader page; for anything sensitive, + `c0mpute storage get` is the stronger path. - **Post-quantum.** Key wrapping is X25519. Data with a decades-long secrecy horizon should wait for a hybrid ML-KEM wrap (the primitives are already in the tree for `c0mpute dm`). diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index fc0dc787..61988d43 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -646,10 +646,19 @@ enum PluginCmd { List, /// Install a plugin by id (from the c0mpute marketplace) or by URL. /// + /// An id runs the installer shipped in this release's signed plugins + /// archive. A URL is somebody else's script that nothing verifies, so it + /// needs --unverified. + /// /// Examples: /// c0mpute plugin install transcode - /// c0mpute plugin install https://example.com/my-plugin/install.sh - Install { target: String }, + /// c0mpute plugin install --unverified https://example.com/my-plugin/install.sh + Install { + target: String, + /// Run an installer from a URL even though nothing verifies it. + #[arg(long)] + unverified: bool, + }, /// Enable a previously disabled plugin. Enable { id: String }, /// Disable a plugin without uninstalling. @@ -3016,13 +3025,107 @@ mod worker_stop_tests { #[test] fn plugin_targets_cannot_carry_a_command() { - assert_eq!(resolve_plugin_target("tts").unwrap(), "https://c0mpute.com/plugins/tts/install.sh"); - assert_eq!(resolve_plugin_target("https://example.com/p/install.sh").unwrap(), "https://example.com/p/install.sh"); - for bad in ["x; rm -rf ~", "../x", "-o/tmp/x", "a b", "", "https://", "https://e.com/x; id", "https://e.com/$(id)", "https://e.com/x | sh"] { + assert_eq!(resolve_plugin_target("tts").unwrap(), PluginTarget::Id("tts".into())); + assert_eq!(resolve_plugin_target(" ad-generator-agent ").unwrap(), PluginTarget::Id("ad-generator-agent".into())); + assert_eq!( + resolve_plugin_target("https://example.com/p/install.sh").unwrap(), + PluginTarget::Url("https://example.com/p/install.sh".into()) + ); + for bad in ["x; rm -rf ~", "../x", "-o/tmp/x", "a b", "", "https://", "https://e.com/x; id", "https://e.com/$(id)", "https://e.com/x | sh", "http://e.com/install.sh", "a/b", "Tts"] { assert!(resolve_plugin_target(bad).is_err(), "{bad}"); } } + #[test] + fn a_plugin_url_needs_the_unverified_flag() { + let url = "https://example.com/p/install.sh"; + // An id is always the signed installer, flag or not. + assert_eq!(plugin_install_plan("infernet", false).unwrap(), PluginPlan::Signed("infernet".into())); + assert_eq!(plugin_install_plan("infernet", true).unwrap(), PluginPlan::Signed("infernet".into())); + // A URL is refused unless the operator said --unverified. + let refused = plugin_install_plan(url, false).unwrap_err().to_string(); + assert!(refused.contains("--unverified"), "{refused}"); + assert_eq!(plugin_install_plan(url, true).unwrap(), PluginPlan::Unverified(url.into())); + // The flag does not make a bad target acceptable. + assert!(plugin_install_plan("http://example.com/p/install.sh", true).is_err()); + assert!(plugin_install_plan("x; id", true).is_err()); + } + + #[test] + fn plugin_install_parses_the_unverified_flag() { + let parse = |args: &[&str]| Cli::try_parse_from(args.iter().copied()).map(|c| c.command); + match parse(&["c0mpute", "plugin", "install", "transcode"]).unwrap() { + Some(Cmd::Plugin { cmd: PluginCmd::Install { target, unverified } }) => { + assert_eq!(target, "transcode"); + assert!(!unverified); + } + other => panic!("parsed as {other:?}"), + } + match parse(&["c0mpute", "plugin", "install", "--unverified", "https://example.com/i.sh"]).unwrap() { + Some(Cmd::Plugin { cmd: PluginCmd::Install { target, unverified } }) => { + assert_eq!(target, "https://example.com/i.sh"); + assert!(unverified); + } + other => panic!("parsed as {other:?}"), + } + } + + #[test] + fn signed_assets_come_from_this_binarys_own_release() { + let tag = own_release_tag(); + assert_eq!(tag, format!("v{}", env!("CARGO_PKG_VERSION"))); + assert_eq!( + release_asset_url("v0.3.65", PLUGINS_ARCHIVE), + "https://github.com/profullstack/c0mpute/releases/download/v0.3.65/c0mpute-plugins.tar.gz" + ); + assert_eq!( + release_asset_url(&tag, TUI_ARCHIVE), + format!("https://github.com/profullstack/c0mpute/releases/download/{tag}/c0mpute-tui.tar.gz") + ); + } + + #[test] + fn a_tui_branch_override_is_a_plain_branch_name() { + assert_eq!( + tui_branch_url("master").unwrap(), + "https://github.com/profullstack/c0mpute/archive/refs/heads/master.tar.gz" + ); + assert!(tui_branch_url("feat/tui-2.0_x").is_ok()); + for bad in ["", "-x", "a b", "a'b", "../x", "x/../y", "a;b", "$(id)", "a?b", "a#b"] { + assert!(tui_branch_url(bad).is_err(), "{bad}"); + } + } + + #[test] + fn a_scratch_dir_is_removed_on_drop() { + let base = tempfile::tempdir().unwrap(); + let dir = base.path().join("scratch"); + std::fs::create_dir_all(dir.join("stale")).unwrap(); + { + let s = ScratchDir::create(&dir).unwrap(); + assert!(s.path().is_dir()); + assert!(!s.path().join("stale").exists(), "an existing directory is emptied first"); + } + assert!(!dir.exists()); + } + + #[test] + fn a_signed_asset_is_refused_unless_it_is_this_file_of_this_release() { + // The verifier `fetch_signed_release_asset` calls. A real published + // signature (v0.3.64, linux x86_64) over bytes that are not the + // artifact must not verify, whatever name and tag are asked for. + let sig = "untrusted comment: signature from minisign secret key\n\ + RUQqOew3QbWhATNMJDo0v4saAncRTUdP557zY9X/DggMKQNkZ/id8b+vpXGezphgXnVM2+X7cb1TCB8U0gGJHTejkJBcWiFD+gw=\n\ + trusted comment: timestamp:1794555478 version:v0.3.64 file:c0mpute-linux-x86_64.tar.gz\n\ + Lh5cSGzX/jrC/n/bobJZWpYxtHeq9unNog/A/UC6Wonf2RWPCUqrWTH12osx3NiUqbHoIo6oWyuDG1g3EklxDg==\n"; + let verify = |bytes: &[u8], asset: &str, tag: &str| { + c0mpute_update::verify_release_signature(c0mpute_update::RELEASE_PUBKEY, bytes, sig, asset, Some(tag)) + }; + assert!(verify(b"not the artifact", "c0mpute-linux-x86_64.tar.gz", "v0.3.64").is_err()); + assert!(verify(b"not the artifact", PLUGINS_ARCHIVE, &own_release_tag()).is_err()); + assert!(verify(b"", TUI_ARCHIVE, "v0.3.64").is_err()); + } + #[test] fn rpc_sharding_is_off_unless_asked_for() { assert!(!rpc_sharding_opted_in(None)); @@ -5221,7 +5324,7 @@ fn run_plugin(cmd: PluginCmd) -> Result<()> { } Ok(()) } - PluginCmd::Install { target } => install_plugin(&target), + PluginCmd::Install { target, unverified } => install_plugin(&target, unverified), PluginCmd::Enable { id } => { println!("[stub] enable {id}"); Ok(()) @@ -5280,20 +5383,67 @@ fn preset_to_spec(preset: &str) -> Result { /// Resolve a `c0mpute plugin install ` argument and dispatch. /// /// Accepts either: -/// - a plugin **id** registered in the c0mpute marketplace, in which -/// case we chain to `https://c0mpute.com/plugins//install.sh` -/// - or any **install.sh URL** (third-party plugins published at -/// their own URL — e.g. a GitHub raw URL) -/// -/// Both flows pipe `curl -fsSL ` into `sh`. Future: minisign -/// signature verification before execution (per DIP-0006). -fn install_plugin(target: &str) -> Result<()> { - let url = resolve_plugin_target(target)?; +/// - a plugin **id**: its `plugins//install.sh` is taken from the +/// `c0mpute-plugins.tar.gz` asset of this binary's own release, after +/// the archive's minisign signature verified against the key compiled +/// into this binary +/// - or any **https install.sh URL** (third-party plugins published at +/// their own URL). Nothing verifies those, so they need `--unverified`. +fn install_plugin(target: &str, unverified: bool) -> Result<()> { + match plugin_install_plan(target, unverified)? { + PluginPlan::Signed(id) => install_signed_plugin(&id), + PluginPlan::Unverified(url) => { + eprintln!("! {url} is not part of a c0mpute release: nothing verifies this script before it runs (--unverified)"); + run_unverified_installer(&url) + } + } +} + +/// What `plugin install` will do for a target, before anything is fetched. +#[derive(Debug, PartialEq)] +enum PluginPlan { + /// Run this id's installer out of the signed release archive. + Signed(String), + /// Run the script at this URL; only with `--unverified`. + Unverified(String), +} + +fn plugin_install_plan(target: &str, unverified: bool) -> Result { + match resolve_plugin_target(target)? { + PluginTarget::Id(id) => Ok(PluginPlan::Signed(id)), + PluginTarget::Url(url) if unverified => Ok(PluginPlan::Unverified(url)), + PluginTarget::Url(url) => anyhow::bail!( + "{url} is not part of a c0mpute release, so nothing verifies it before it runs. \ + To run it anyway: c0mpute plugin install --unverified {url}" + ), + } +} + +/// Run plugin `id`'s installer from the signed plugins archive of this release. +fn install_signed_plugin(id: &str) -> Result<()> { + let tag = own_release_tag(); + let scratch = ScratchDir::create(&c0mpute_home()?.join(format!("plugin-install-{}", std::process::id())))?; + let archive = fetch_signed_release_asset(PLUGINS_ARCHIVE, scratch.path())?; + untar(&archive, scratch.path())?; + let script = scratch.path().join("plugins").join(id).join("install.sh"); + if !script.is_file() { + anyhow::bail!("plugin '{id}' has no installer in the signed plugins archive of c0mpute {tag}"); + } + println!("→ installing plugin {id} (signed installer from c0mpute {tag})"); + let status = Command::new("sh").arg(&script).status()?; + if !status.success() { + anyhow::bail!("plugin install failed (exit {status})"); + } + Ok(()) +} + +/// The old flow, for `--unverified` URLs only: pipe `curl -fsSL ` into `sh`. +fn run_unverified_installer(url: &str) -> Result<()> { println!("→ installing plugin via {url}"); // No shell string: the URL is one curl argument, and curl's output is // sh's stdin, so nothing in the target can add a command. let mut curl = std::process::Command::new("curl") - .args(["-fsSL", "--", url.as_str()]) + .args(["-fsSL", "--", url]) .stdout(std::process::Stdio::piped()) .spawn()?; let script = curl.stdout.take().context("curl has no stdout")?; @@ -5308,22 +5458,129 @@ fn install_plugin(target: &str) -> Result<()> { Ok(()) } +// Release assets the installer (scripts/install.sh) and this CLI install from. +// Packed by scripts/pack-release-extras.sh and signed with the release key in +// .github/workflows/release.yml, like the binaries. +const PLUGINS_ARCHIVE: &str = "c0mpute-plugins.tar.gz"; +const TUI_ARCHIVE: &str = "c0mpute-tui.tar.gz"; + +/// The release tag this binary was built from (`v`). +fn own_release_tag() -> String { + format!("v{}", env!("CARGO_PKG_VERSION")) +} + +/// Where `asset` of release `tag` is downloaded from. +fn release_asset_url(tag: &str, asset: &str) -> String { + format!("https://github.com/profullstack/c0mpute/releases/download/{tag}/{asset}") +} + +/// `~/.c0mpute`: scratch for installs lives here, not in the shared /tmp where +/// another local user could pre-create the directory and plant files in it. +fn c0mpute_home() -> Result { + let home = std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .ok_or_else(|| anyhow::anyhow!("HOME not set"))?; + Ok(PathBuf::from(home).join(".c0mpute")) +} + +/// A directory that is emptied and created now and removed on drop. +struct ScratchDir(PathBuf); + +impl ScratchDir { + fn create(dir: &std::path::Path) -> Result { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + Ok(Self(dir.to_path_buf())) + } + + fn path(&self) -> &std::path::Path { + &self.0 + } +} + +impl Drop for ScratchDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Download `url` to `dest` over https only (redirects included). +fn curl_to_file(url: &str, dest: &std::path::Path) -> Result<()> { + let status = Command::new("curl") + .args(["-fsSL", "--proto", "=https", "--proto-redir", "=https", "-o"]) + .arg(dest) + .args(["--", url]) + .status() + .context("run curl")?; + if !status.success() { + anyhow::bail!("download failed ({status}): {url}"); + } + Ok(()) +} + +/// Download `asset` of this binary's own release into `dir` and verify it the +/// way self-update verifies a binary: the minisign signature must check out +/// against the key compiled into this binary and its signed comment must name +/// this asset and this release. Anything else is an error; there is no +/// unverified fallback. Returns the verified file. +fn fetch_signed_release_asset(asset: &str, dir: &std::path::Path) -> Result { + let tag = own_release_tag(); + let url = release_asset_url(&tag, asset); + let file = dir.join(asset); + let sig = dir.join(format!("{asset}.minisig")); + curl_to_file(&url, &file) + .with_context(|| format!("{asset} is not available for c0mpute {tag} (a build that is not a published release has none)"))?; + curl_to_file(&format!("{url}.minisig"), &sig) + .with_context(|| format!("no signature for {asset} on c0mpute {tag}; refusing to use it"))?; + let bytes = std::fs::read(&file).with_context(|| format!("read {}", file.display()))?; + let sig_text = std::fs::read_to_string(&sig).with_context(|| format!("read {}", sig.display()))?; + c0mpute_update::verify_release_signature(c0mpute_update::RELEASE_PUBKEY, &bytes, &sig_text, asset, Some(&tag))?; + Ok(file) +} + +/// Unpack a (verified) `.tar.gz` into `dir`. +fn untar(archive: &std::path::Path, dir: &std::path::Path) -> Result<()> { + let status = Command::new("tar") + .arg("-xzf") + .arg(archive) + .arg("-C") + .arg(dir) + .status() + .context("run tar")?; + if !status.success() { + anyhow::bail!("could not unpack {} ({status})", archive.display()); + } + Ok(()) +} + /// Whether `C0MPUTE_RPC_SHARDING` asks for the llama.cpp RPC slice. fn rpc_sharding_opted_in(value: Option<&str>) -> bool { matches!(value.map(str::trim), Some("1" | "true" | "yes" | "on")) } +/// What a `plugin install` argument names. +#[derive(Debug, PartialEq)] +enum PluginTarget { + /// A plugin id from this repo's `plugins/` directory. + Id(String), + /// Somebody's own installer script. + Url(String), +} + /// A plugin id is lowercase letters, digits and dashes; anything else must be -/// an http(s) URL with no whitespace, quotes or shell metacharacters. -fn resolve_plugin_target(target: &str) -> Result { +/// an https URL with no whitespace, quotes or shell metacharacters. +fn resolve_plugin_target(target: &str) -> Result { let t = target.trim(); - if t.starts_with("http://") || t.starts_with("https://") { + if t.starts_with("http://") { + anyhow::bail!("plugin URLs must be https: {t}"); + } + if t.starts_with("https://") { let rest = t.split_once("://").map(|(_, r)| r).unwrap_or(""); let bad = |c: char| c.is_whitespace() || c.is_control() || matches!(c, '"' | '\'' | '`' | '$' | ';' | '|' | '<' | '>' | '\\'); if rest.is_empty() || rest.starts_with('/') || t.chars().any(bad) { anyhow::bail!("not a plugin URL: {t}"); } - return Ok(t.to_string()); + return Ok(PluginTarget::Url(t.to_string())); } let id_ok = !t.is_empty() && t.len() <= 64 @@ -5332,10 +5589,9 @@ fn resolve_plugin_target(target: &str) -> Result { if !id_ok { anyhow::bail!("plugin id must be lowercase letters, digits and dashes (or a full https URL): {t}"); } - // Treat anything else as a plugin id and resolve through the - // c0mpute marketplace. The route at c0mpute.com/plugins//install.sh - // serves the manifest-checked-in install script for that plugin. - Ok(format!("https://c0mpute.com/plugins/{t}/install.sh")) + // Anything else is a plugin id: its installer comes out of the signed + // plugins archive of this release (see install_signed_plugin). + Ok(PluginTarget::Id(t.to_string())) } // ──────────────────────────────────────────────────────────────────────── @@ -5605,10 +5861,12 @@ fn run_menu() -> Result<()> { /// `c0mpute tui` — launch the terminal UI, installing it on demand the first /// time. The TUI is a Bun app (apps/tui, on @profullstack/hqtui), not part of -/// the Rust binary, so `c0mpute tui` fetches the source, runs `bun install`, -/// and writes a launcher — the user never has to know about the underlying -/// `c0mpute-tui` binary. After an update it reinstalls the TUI from this -/// release's tag, so the CLI and the TUI move together. +/// the Rust binary, so `c0mpute tui` fetches the source (the signed +/// `c0mpute-tui.tar.gz` of this release, verified with the release key +/// compiled into this binary), runs `bun install`, and writes a launcher — +/// the user never has to know about the underlying `c0mpute-tui` binary. +/// After an update it reinstalls the TUI from this release, so the CLI and +/// the TUI move together. fn run_tui(args: &[String]) -> Result<()> { let stale = installed_tui_version().is_some_and(|v| tui_is_stale(&v)); if tui_launcher().is_none() { @@ -5685,57 +5943,87 @@ fn ensure_bun() -> Option { mise_which(&mise).or_else(|| which_on_path("bun")) } +/// The GitHub tarball of a branch, for the `C0MPUTE_TUI_REF` developer +/// override. Nothing signs a branch, so this is the unverified path. +fn tui_branch_url(branch: &str) -> Result { + let ok = !branch.is_empty() + && !branch.starts_with('-') + && !branch.contains("..") + && branch + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '/')); + if !ok { + anyhow::bail!("C0MPUTE_TUI_REF is not a branch name: {branch}"); + } + Ok(format!("https://github.com/profullstack/c0mpute/archive/refs/heads/{branch}.tar.gz")) +} + +/// Fetch the TUI source into `scratch` and return its directory (holding +/// `src/`, `package.json`, `tsconfig.json`). Normally that is the signed +/// `c0mpute-tui.tar.gz` of this binary's own release, so the TUI matches the +/// CLI and is verified like it. `C0MPUTE_TUI_REF=` takes a branch +/// instead, unverified, with a warning. +fn fetch_tui_source(scratch: &std::path::Path) -> Result { + if let Some(branch) = std::env::var("C0MPUTE_TUI_REF").ok().filter(|b| !b.is_empty()) { + let url = tui_branch_url(&branch)?; + eprintln!("! C0MPUTE_TUI_REF={branch}: installing the terminal UI UNVERIFIED from that branch"); + let tarball = scratch.join("branch.tar.gz"); + curl_to_file(&url, &tarball)?; + untar(&tarball, scratch)?; + // The tarball has one top-level directory, -. + for entry in std::fs::read_dir(scratch)?.flatten() { + let tui = entry.path().join("apps").join("tui"); + if tui.is_dir() { + return Ok(tui); + } + } + anyhow::bail!("TUI source not found on branch {branch}"); + } + let archive = fetch_signed_release_asset(TUI_ARCHIVE, scratch).with_context(|| { + "could not get the signed terminal UI (developers: C0MPUTE_TUI_REF= installs it unverified from a branch)" + })?; + untar(&archive, scratch)?; + Ok(scratch.join("tui")) +} + fn install_tui() -> Result<()> { println!("c0mpute tui: installing the terminal UI for v{}…", env!("CARGO_PKG_VERSION")); - let home = std::env::var("HOME") - .map(PathBuf::from) - .map_err(|_| anyhow::anyhow!("HOME not set"))?; + let c0mpute_dir = c0mpute_home()?; let bun = ensure_bun().ok_or_else(|| { anyhow::anyhow!( - "bun is required for the TUI and couldn't be installed via mise. \ - Install it, then rerun `c0mpute tui`:\n curl -fsSL https://bun.sh/install | bash" + "bun is required for the TUI. Re-run the c0mpute installer (it installs a pinned, \ + checksum-verified bun), or install bun yourself (https://bun.sh/docs/installation), \ + then rerun `c0mpute tui`" ) })?; - let tui_dir = home.join(".c0mpute/tui"); - let wrapper = home.join(".c0mpute/bin/c0mpute-tui"); + let tui_dir = c0mpute_dir.join("tui"); + let wrapper = c0mpute_dir.join("bin/c0mpute-tui"); // Scratch under the user's own ~/.c0mpute, not a fixed name in the shared // /tmp: there another local user could pre-create the directory and plant - // an `apps/tui` that this script would install and run. - let tmp = home.join(format!(".c0mpute/tui-install-{}", std::process::id())); - // This release's tag, so the TUI matches the CLI; master if the tag is - // not there (a dev build). C0MPUTE_TUI_REF picks a branch instead. - let archive = match std::env::var("C0MPUTE_TUI_REF") { - Ok(branch) if !branch.is_empty() => format!("https://github.com/profullstack/c0mpute/archive/refs/heads/{branch}.tar.gz"), - _ => format!("https://github.com/profullstack/c0mpute/archive/refs/tags/v{}.tar.gz", env!("CARGO_PKG_VERSION")), - }; - let fallback = "https://github.com/profullstack/c0mpute/archive/refs/heads/master.tar.gz"; - - // Fetch the repo, copy apps/tui into ~/.c0mpute/tui, and materialise deps. - let script = format!( - "set -e\n\ - rm -rf '{tmp}' && mkdir -p '{tmp}'\n\ - curl -fsSL '{archive}' | tar -xz -C '{tmp}' || {{ rm -rf '{tmp}' && mkdir -p '{tmp}' && curl -fsSL '{fallback}' | tar -xz -C '{tmp}'; }}\n\ - src=$(find '{tmp}' -type d -path '*/apps/tui' | head -1)\n\ - [ -d \"$src/src\" ] || {{ echo 'TUI source not found' >&2; exit 1; }}\n\ - rm -rf '{tui}' && mkdir -p '{tui}'\n\ - cp -R \"$src/src\" \"$src/package.json\" \"$src/tsconfig.json\" '{tui}/'\n\ - cd '{tui}' && '{bun}' install --production --no-save >/dev/null 2>&1\n\ - rm -rf '{tmp}'\n", - tmp = tmp.display(), - archive = archive, - fallback = fallback, - tui = tui_dir.display(), - bun = bun.display(), - ); - let ok = Command::new("sh") - .arg("-c") - .arg(&script) + // an `apps/tui` that this would install and run. + let scratch = ScratchDir::create(&c0mpute_dir.join(format!("tui-install-{}", std::process::id())))?; + let src = fetch_tui_source(scratch.path())?; + if !src.join("src").is_dir() || !src.join("package.json").is_file() { + anyhow::bail!("TUI source not found in the downloaded archive"); + } + + // Swap the source in (same filesystem: a rename), then materialise deps. + if tui_dir.exists() { + std::fs::remove_dir_all(&tui_dir).with_context(|| format!("remove {}", tui_dir.display()))?; + } + std::fs::rename(&src, &tui_dir).with_context(|| format!("install {}", tui_dir.display()))?; + drop(scratch); + let ok = Command::new(&bun) + .args(["install", "--production", "--no-save"]) + .current_dir(&tui_dir) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .status() .map(|s| s.success()) .unwrap_or(false); if !ok { - anyhow::bail!("failed to fetch/build the TUI (need network + bun)"); + anyhow::bail!("failed to install the TUI's dependencies (need network + bun)"); } // Launcher — runs the Bun source. Kept as source rather than a @@ -5747,7 +6035,7 @@ fn install_tui() -> Result<()> { BUN=\"$(command -v bun 2>/dev/null || true)\"\n\ [ -z \"$BUN\" ] && [ -x \"$HOME/.bun/bin/bun\" ] && BUN=\"$HOME/.bun/bin/bun\"\n\ [ -z \"$BUN\" ] && command -v mise >/dev/null 2>&1 && BUN=\"$(mise which bun 2>/dev/null || true)\"\n\ - [ -n \"$BUN\" ] || {{ echo 'bun not found; run: mise use --global bun@latest' >&2; exit 1; }}\n\ + [ -n \"$BUN\" ] || {{ echo 'bun not found; re-run the c0mpute installer (it installs a pinned bun)' >&2; exit 1; }}\n\ exec \"$BUN\" run \"{}/src/index.ts\" \"$@\"\n", tui_dir.display() ); diff --git a/node/crates/c0mpute-core/src/apps/mod.rs b/node/crates/c0mpute-core/src/apps/mod.rs index ae5bc151..64fb7863 100644 --- a/node/crates/c0mpute-core/src/apps/mod.rs +++ b/node/crates/c0mpute-core/src/apps/mod.rs @@ -64,6 +64,15 @@ pub const RPC_LOGS: &str = "apps.logs"; pub const RPC_RECORD: &str = "apps.record"; /// Apps per announcement, so one message stays well under gossipsub's limit. const APPS_PER_MESSAGE: usize = 100; +/// `[apps] max_disk_mb` when the operator sets none: the 1 GiB of state an +/// app may keep ([`snapshot::MAX_STATE_BYTES`]) plus as much again written +/// inside its container. +pub const DEFAULT_MAX_DISK_MB: u64 = 2048; +/// How often a running app's disk use is measured (docker walks the +/// container's writable layer to answer). +const DISK_CHECK_EVERY: Duration = Duration::from_secs(60); +/// Logged once per run by a node that runs apps without the egress guard. +const OPEN_EGRESS_WARNING: &str = "apps: no egress guard on this node (it needs a root worker, iptables and [apps] egress_guard on): app containers can reach this machine's LAN, the host itself and cloud metadata addresses. Set [apps] require_egress_guard = true to run apps only behind the guard"; /// How often things happen. Tests shrink these. #[derive(Clone, Debug)] @@ -144,6 +153,15 @@ struct Local { last_snap: Option, #[serde(skip)] failures: u32, + /// Size of the state directory at the last snapshot (its tar), or one + /// byte over the cap when a snapshot was refused for size. + #[serde(skip)] + state_bytes: u64, + #[serde(skip)] + disk_checked: Option, + /// Why this node last stopped the app itself (over its disk limit). + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, } impl Local { @@ -162,6 +180,9 @@ impl Local { busy: false, last_snap: None, failures: 0, + state_bytes: 0, + disk_checked: None, + error: None, } } @@ -207,6 +228,12 @@ struct Inner { max_apps: usize, max_memory_mb: u32, owners: Vec, + egress_guard: bool, + require_egress_guard: bool, + /// Most disk one app may use (state plus writable layer); 0 = no limit. + max_disk_bytes: u64, + /// The "apps can reach the LAN" warning has been logged this run. + warned_open_egress: std::sync::atomic::AtomicBool, timing: Timing, x25519: crypto::KeyPair, hosting: Option, @@ -236,6 +263,12 @@ pub struct Options { pub dir: Option, /// Run only apps signed by these owner keys (hex ed25519); empty = anyone's. pub owners: Vec, + /// `[apps] egress_guard`, `require_egress_guard` and `max_disk_mb`. + /// `None` reads the node's config file, so a caller that predates these + /// settings still gets what the operator wrote. + pub egress_guard: Option, + pub require_egress_guard: Option, + pub max_disk_mb: Option, pub timing: Timing, /// To read images published as hosting sites. pub hosting: Option, @@ -254,6 +287,9 @@ impl Default for Options { max_memory_mb: 2048, dir: None, owners: Vec::new(), + egress_guard: None, + require_egress_guard: None, + max_disk_mb: None, timing: Timing::default(), hosting: None, dns: dns::Verifier::default(), @@ -277,6 +313,10 @@ pub struct AppInfo { pub snapshot: Option, pub domains: Vec, pub path: String, + /// Why the answering node last stopped this app itself (over its disk + /// limit), when it holds a copy and did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, } /// The apps service: one per node. Cheap to clone. @@ -359,21 +399,49 @@ impl Apps { .into_iter() .filter(|(_, r)| r.verify()) .collect(); + // Settings the caller did not pass come from the node's config file. + let file = match (opts.egress_guard, opts.require_egress_guard, opts.max_disk_mb) { + (Some(_), Some(_), Some(_)) => crate::config::AppsConfig::default(), + _ if !opts.hosts => crate::config::AppsConfig::default(), + _ => crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default().apps, + }; + let require_egress_guard = opts.require_egress_guard.unwrap_or(file.require_egress_guard); + let egress_guard = opts.egress_guard.unwrap_or(file.egress_guard) || require_egress_guard; + let max_disk_mb = opts.max_disk_mb.unwrap_or(file.max_disk_mb); + let mut hosts = opts.hosts; + if hosts && require_egress_guard && !crate::container_run::is_root() { + // The guard is iptables rules, which only a root worker can add. + warn!("apps: [apps] require_egress_guard is set and this worker is not root, so the egress guard cannot be put in place; running no apps"); + hosts = false; + } + if hosts && opts.owners.is_empty() { + warn!("apps: [apps] owners is empty, so this node runs containers for any owner on the network; list owner keys there to run only theirs"); + } + // Known already at start for a non-root worker or a guard turned + // off; a root worker finds out when docker and iptables are asked. + let open_egress = hosts && (!egress_guard || !crate::container_run::is_root()); + if open_egress { + warn!("{OPEN_EGRESS_WARNING}"); + } let a = Apps(Arc::new(Inner { net: net.clone(), store, me, - hosts: opts.hosts, + hosts, max_apps: opts.max_apps.max(1), max_memory_mb: opts.max_memory_mb, owners: opts.owners, + egress_guard, + require_egress_guard, + max_disk_bytes: max_disk_mb.saturating_mul(1 << 20), + warned_open_egress: std::sync::atomic::AtomicBool::new(open_egress), timing: opts.timing, x25519, hosting: opts.hosting, dns: opts.dns, records: RwLock::new(records), nodes: RwLock::new(HashMap::new()), - local: RwLock::new(if opts.hosts { local } else { HashMap::new() }), + local: RwLock::new(if hosts { local } else { HashMap::new() }), owned: RwLock::new(persisted.owned), sealed_to: RwLock::new(HashMap::new()), released: RwLock::new(HashMap::new()), @@ -911,6 +979,7 @@ impl Apps { } else if let Some(port) = host_port { let ok = runtime::healthy(port, &rec.health_path).await; let alive = ok || runtime::running(&id).await; + let mut disk_due = false; let mut local = self.0.local.write().await; if let Some(l) = local.get_mut(&id) { if l.healthy != ok { @@ -921,9 +990,15 @@ impl Apps { warn!(app = %id, "apps: container stopped; restarting"); l.running_seq = None; l.failures += 1; + } else if l.disk_checked.is_none_or(|t| t.elapsed() >= DISK_CHECK_EVERY) { + l.disk_checked = Some(Instant::now()); + disk_due = true; } } drop(local); + if disk_due { + self.spawn_disk_check(id.clone()); + } if alive && snap_due { self.spawn_snapshot(id); } @@ -1159,6 +1234,15 @@ impl Apps { }; let secret_env = secrets.as_ref().map(|s| s.env.clone()).unwrap_or_default(); let _ = runtime::remove(id).await; + let create = runtime::CreateOpts { + network: self.app_network().await?, + layer_quota_mb: runtime::layer_quota_mb(self.0.max_disk_bytes).await, + }; + // Same floor as the disk watchdog: an app it would stop is not started. + let (free, floor) = runtime::host_free().await; + if free < floor { + anyhow::bail!("this node's docker disk is down to {} GiB free (it keeps {} GiB)", free >> 30, floor >> 30); + } if let Some(state_path) = &rec.state_path { let secrets = secrets.context("no secrets for a stateful app")?; let key = secrets.key()?; @@ -1188,18 +1272,83 @@ impl Apps { runtime::restore_state(id, &tar).await?; info!(app = %id, epoch = s.epoch, n = s.n, bytes = tar.len(), path = %state_path, "apps: state restored"); } - runtime::create(&rec, &secret_env).await?; + runtime::create(&rec, &secret_env, &create).await?; } else { - runtime::create(&rec, &secret_env).await?; + runtime::create(&rec, &secret_env, &create).await?; } } else { - runtime::create(&rec, &secret_env).await?; + runtime::create(&rec, &secret_env, &create).await?; } runtime::start(id).await?; let port = runtime::host_port(id, rec.port).await?; Ok(port) } + /// The docker network this node's app containers join: the guarded apps + /// network when the egress guard is in place. Without it (a non-root + /// worker, or `egress_guard = false`) apps stay on docker's default + /// bridge, said once per run; with `require_egress_guard` nothing starts. + async fn app_network(&self) -> Result> { + let guarded = self.0.egress_guard && runtime::ensure_guarded_network().await; + let network = runtime::network_for(guarded, self.0.require_egress_guard).map_err(anyhow::Error::msg)?; + if network.is_none() && !self.0.warned_open_egress.swap(true, std::sync::atomic::Ordering::Relaxed) { + warn!("{OPEN_EGRESS_WARNING}"); + } + Ok(network) + } + + /// Measure one running app's disk use and stop it when it is over its + /// limit or the host is under its free-space floor. + fn spawn_disk_check(&self, id: String) { + let a = self.clone(); + tokio::spawn(async move { + let Some(state) = a.0.local.read().await.get(&id).map(|l| l.state_bytes) else { + return; + }; + // A size docker cannot report counts as nothing written: an app + // is never stopped on a failed measurement. + let layer = runtime::layer_bytes(&id).await.unwrap_or(0); + let (free, floor) = runtime::host_free().await; + if let Some(stop) = runtime::disk_verdict(layer, state, a.0.max_disk_bytes, free, floor) { + a.stop_for_disk(&id, stop).await; + } + }); + } + + /// Stop an app this node runs for the disk it uses: record why, give the + /// primary role up so another holder can take over from the last + /// snapshot, and remove the container (and the state volume when the + /// state itself is over the cap). + async fn stop_for_disk(&self, id: &str, stop: runtime::DiskStop) { + { + let mut local = self.0.local.write().await; + let Some(l) = local.get_mut(id) else { return }; + // Mid-start or mid-snapshot: the next check decides. + if l.busy || l.role != AppRole::Primary { + return; + } + warn!(app = %id, reason = %stop.reason, "apps: stopping the app"); + l.error = Some(format!("stopped at {}: {}", now_secs(), stop.reason)); + l.role = AppRole::Standby; + l.running_seq = None; + l.host_port = None; + l.healthy = false; + l.ready_seq = None; + l.failures += 1; + l.disk_checked = None; + if stop.drop_state { + l.volume_epoch = None; + l.state_bytes = 0; + } + } + let _ = runtime::remove(id).await; + if stop.drop_state { + let _ = runtime::remove_volume(id).await; + } + self.persist().await; + self.announce().await; + } + async fn snap_manifest(&self, hash_hex: &str) -> Result { let h = Hash::from_hex(hash_hex)?; SnapManifest::from_bytes(&self.0.store.get(&h).await?) @@ -1227,7 +1376,15 @@ impl Apps { a.persist().await; } Ok(None) => {} - Err(e) => warn!(app = %id, err = %e, "apps: snapshot failed"), + Err(e) => { + warn!(app = %id, err = %e, "apps: snapshot failed"); + if runtime::over_state_cap(&e) { + // More state than an app may keep: the disk + // check on the next pass stops it. + l.state_bytes = snapshot::MAX_STATE_BYTES as u64 + 1; + l.disk_checked = None; + } + } } } }); @@ -1248,6 +1405,9 @@ impl Apps { return Ok(None); }; let tar = runtime::snapshot_state(id, path).await?; + if let Some(l) = self.0.local.write().await.get_mut(id) { + l.state_bytes = tar.len() as u64; + } let n = match &prev { Some(p) if p.epoch == epoch => p.n + 1, _ => 1, @@ -1530,6 +1690,7 @@ impl Apps { snapshot: placement::newest_snapshot(&rec.app_id, view).map(|(_, s)| s), domains: rec.domains.clone(), path: format!("/a/{}/", rec.app_id), + error: self.0.local.read().await.get(&rec.app_id).and_then(|l| l.error.clone()), } } diff --git a/node/crates/c0mpute-core/src/apps/runtime.rs b/node/crates/c0mpute-core/src/apps/runtime.rs index d7e5396f..31f05a77 100644 --- a/node/crates/c0mpute-core/src/apps/runtime.rs +++ b/node/crates/c0mpute-core/src/apps/runtime.rs @@ -117,8 +117,64 @@ pub async fn load(tar: &Path) -> Result<()> { .map(|_| ()) } +/// The docker network guarded app containers join, and its Linux interface. +/// Not `container.run`'s jobs network: that one drops everything a container +/// sends to the host, replies included, and an app must answer the node's +/// proxy on its loopback-published port. +pub const APPS_NETWORK: &str = "c0mpute-apps"; +pub const APPS_BRIDGE: &str = "c0apps0"; + +/// The egress guard for the apps bridge: `container.run`'s rules for private, +/// loopback, link-local (metadata), CGNAT and reserved ranges, and for the +/// host itself everything except replies to connections the host opened +/// (docker's loopback port publishing, which is how the proxy and the health +/// probe reach the app). +pub fn guard_rules(bridge: &str) -> Vec> { + let mut rules = crate::container_run::guard_rules(bridge); + rules.retain(|r| r.first().map(String::as_str) != Some("INPUT")); + rules.push(s(&["INPUT", "-i", bridge, "-m", "conntrack", "!", "--ctstate", "ESTABLISHED", "-j", "DROP"])); + rules +} + +/// Make sure the apps network exists with the egress guard on it. True when +/// app containers can be put behind the guard; false for a non-root worker +/// (it cannot change the firewall) or when docker or iptables refuse. +pub async fn ensure_guarded_network() -> bool { + if !crate::container_run::is_root() { + return false; + } + if let Err(e) = crate::container_run::ensure_bridge_network(APPS_NETWORK, APPS_BRIDGE, "c0mpute.apps=1").await { + tracing::warn!(err = %format!("{e:#}"), "apps: could not create the apps network"); + return false; + } + crate::container_run::ensure_rules(guard_rules(APPS_BRIDGE)).await +} + +/// Which network an app container joins: the guarded apps network when the +/// guard is in, docker's default bridge (`None`) when it is not, or an error +/// when the operator runs apps only behind the guard. +pub fn network_for(guarded: bool, required: bool) -> std::result::Result, String> { + match (guarded, required) { + (true, _) => Ok(Some(APPS_NETWORK)), + (false, true) => Err( + "[apps] require_egress_guard is set and the egress guard is not in place (it needs a root worker, docker and iptables)".into(), + ), + (false, false) => Ok(None), + } +} + +/// Where and how tightly an app's container is created. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CreateOpts { + /// The docker network to join; `None` is docker's default bridge. + pub network: Option<&'static str>, + /// A hard quota on the container's writable layer, MiB, where the + /// storage driver has one ([`quota_supported`]). + pub layer_quota_mb: Option, +} + /// The `docker create` argv for an app. Pure, so it is tested without docker. -pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Vec { +pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap, opts: &CreateOpts) -> Vec { let name = container_name(&rec.app_id); let mut a: Vec = vec![ "create".into(), @@ -166,6 +222,14 @@ pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Ve "--publish", ])); a.push(format!("127.0.0.1::{}", rec.port)); + if let Some(network) = opts.network { + a.push("--network".into()); + a.push(network.into()); + } + if let Some(mb) = opts.layer_quota_mb { + a.push("--storage-opt".into()); + a.push(format!("size={mb}m")); + } if let Some(p) = &rec.state_path { a.push("--volume".into()); a.push(format!("{}:{}", volume_name(&rec.app_id), p)); @@ -182,11 +246,154 @@ pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Ve a } -pub async fn create(rec: &AppRecord, secret_env: &BTreeMap) -> Result<()> { +pub async fn create(rec: &AppRecord, secret_env: &BTreeMap, opts: &CreateOpts) -> Result<()> { let _ = remove(&rec.app_id).await; - docker(&create_args(rec, secret_env), None, Duration::from_secs(60)) + let made = docker(&create_args(rec, secret_env, opts), None, Duration::from_secs(60)).await; + if made.is_err() && opts.layer_quota_mb.is_some() { + // The quota was detected, not promised: a daemon that refuses it + // still runs the app, and the disk watchdog stays the limit. + let _ = remove(&rec.app_id).await; + let without = CreateOpts { layer_quota_mb: None, ..*opts }; + return docker(&create_args(rec, secret_env, &without), None, Duration::from_secs(60)).await.map(|_| ()); + } + made.map(|_| ()) +} + +/// Does `docker create --storage-opt size=` work here? Only overlay2 on an +/// xfs filesystem mounted with project quotas enforces it; on ext4 (most +/// hosts) docker refuses the flag. `mount_options` are those of the +/// filesystem that holds docker's root directory. +pub fn quota_supported(driver: &str, backing_fs: &str, mount_options: &str) -> bool { + driver == "overlay2" + && backing_fs == "xfs" + && mount_options.split(',').any(|o| matches!(o.trim(), "pquota" | "prjquota")) +} + +/// The mount options of the filesystem holding `path`, from a +/// `/proc/self/mounts` listing (the longest mount point that is a prefix). +pub fn mount_options<'a>(mounts: &'a str, path: &Path) -> Option<&'a str> { + mounts + .lines() + .filter_map(|l| { + let mut f = l.split_whitespace(); + let (_dev, point, _fs, options) = (f.next()?, f.next()?, f.next()?, f.next()?); + path.starts_with(point).then_some((point.len(), options)) + }) + .max_by_key(|(len, _)| *len) + .map(|(_, options)| options) +} + +/// What `docker info` says about where and how containers are stored. +#[derive(Clone, Debug, Default)] +struct Storage { + root: Option, + quota: bool, +} + +/// Asked once per process: the storage driver does not change under a +/// running node. +async fn storage() -> &'static Storage { + static STORAGE: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); + STORAGE + .get_or_init(|| async { + let format = "{{.DockerRootDir}}\n{{.Driver}}\n{{range .DriverStatus}}{{index . 0}}={{index . 1}}\n{{end}}"; + let Ok(out) = docker(&s(&["info", "--format", format]), None, Duration::from_secs(20)).await else { + return Storage::default(); + }; + let text = String::from_utf8_lossy(&out).into_owned(); + let mut lines = text.lines().map(str::trim); + let root = lines.next().filter(|r| r.starts_with('/')).map(std::path::PathBuf::from); + let driver = lines.next().unwrap_or("").to_string(); + let backing = lines.find_map(|l| l.strip_prefix("Backing Filesystem=")).unwrap_or("").to_string(); + let mounts = std::fs::read_to_string("/proc/self/mounts").unwrap_or_default(); + let quota = root + .as_deref() + .and_then(|r| mount_options(&mounts, r)) + .is_some_and(|o| quota_supported(&driver, &backing, o)); + Storage { root, quota } + }) .await - .map(|_| ()) +} + +/// The writable-layer quota to create an app with, MiB: its disk limit where +/// the storage driver enforces one, else none (the watchdog is the limit). +pub async fn layer_quota_mb(limit_bytes: u64) -> Option { + (limit_bytes > 0 && storage().await.quota).then_some((limit_bytes >> 20).max(1)) +} + +/// Bytes the app has written inside its container (docker's own count of +/// the writable layer; the state volume is not in it). +pub async fn layer_bytes(app_id: &str) -> Option { + let out = docker( + &["inspect".into(), "--size".into(), "--format".into(), "{{.SizeRw}}".into(), container_name(app_id)], + None, + Duration::from_secs(60), + ) + .await + .ok()?; + String::from_utf8_lossy(&out).trim().parse().ok() +} + +/// Free bytes on the disk docker stores containers and volumes on, and the +/// free space `container.run` keeps on such a disk (its default floor). +pub async fn host_free() -> (u64, u64) { + let Some(root) = storage().await.root.as_deref() else { + return (u64::MAX, 0); + }; + let (free, total) = c0mpute_store::free_space_at(root); + (free, crate::container_run::min_free_bytes(&crate::container_run::ContainersConfig::default(), total)) +} + +/// Why an app is being stopped for the disk it uses. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiskStop { + pub reason: String, + /// Remove the state volume as well. No rule asks for it today: an app + /// over its cap is stopped with its data left on this node, because the + /// part written since the last snapshot exists nowhere else and deleting + /// it is the owner's call, not the node's. + pub drop_state: bool, +} + +/// The disk rule for one running app. `layer` is what it wrote inside its +/// container, `state` the size of its state directory (the last snapshot's +/// tar; anything over [`super::snapshot::MAX_STATE_BYTES`] when a snapshot +/// was refused for size). `limit` 0 means no per-app limit. `None`: fine. +pub fn disk_verdict(layer: u64, state: u64, limit: u64, free: u64, floor: u64) -> Option { + let mib = |b: u64| b >> 20; + let state_cap = super::snapshot::MAX_STATE_BYTES as u64; + let state_limit = if limit > 0 { limit.min(state_cap) } else { state_cap }; + if state > state_limit { + return Some(DiskStop { + reason: format!("its state directory is over {} MiB, the most an app may keep here; the data is kept on this node", mib(state_limit)), + drop_state: false, + }); + } + if limit > 0 && layer.saturating_add(state) > limit { + return Some(DiskStop { + reason: format!( + "it uses {} MiB of disk, over its {} MiB limit ({} MiB written inside the container, {} MiB of state)", + mib(layer.saturating_add(state)), + mib(limit), + mib(layer), + mib(state) + ), + drop_state: false, + }); + } + if free < floor { + return Some(DiskStop { + reason: format!("this node's docker disk is down to {} GiB free (it keeps {} GiB)", free >> 30, floor >> 30), + drop_state: false, + }); + } + None +} + +/// Did a snapshot fail because the state is over the size cap (and not +/// because docker did)? +pub fn over_state_cap(err: &anyhow::Error) -> bool { + format!("{err:#}").contains("byte limit") } pub async fn start(app_id: &str) -> Result<()> { @@ -460,9 +667,11 @@ mod tests { let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); let rec = AppRecord::sign(&kp, spec("demo"), 1).unwrap(); let secret = BTreeMap::from([("DB_PASSWORD".to_string(), "hunter2".to_string())]); - let a = create_args(&rec, &secret); + let a = create_args(&rec, &secret, &CreateOpts::default()); let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v); assert_eq!(a[0], "create"); + // Without the guard: docker's default bridge, as before, and no quota flag. + assert!(!a.iter().any(|x| x == "--network" || x == "--storage-opt")); assert!(pair("--memory", "256m") && pair("--memory-swap", "256m")); assert!(pair("--cpus", "0.500")); assert!(pair("--cap-drop", "ALL")); @@ -485,6 +694,104 @@ mod tests { assert!(!a.iter().any(|x| x.starts_with("SYS_") || x == "NET_ADMIN")); } + #[test] + fn guarded_apps_join_the_apps_network_and_stay_published_on_loopback() { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let rec = AppRecord::sign(&kp, spec("demo"), 1).unwrap(); + let opts = CreateOpts { network: Some(APPS_NETWORK), layer_quota_mb: Some(2048) }; + let a = create_args(&rec, &BTreeMap::new(), &opts); + let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v); + assert!(pair("--network", "c0mpute-apps")); + assert!(pair("--storage-opt", "size=2048m")); + // The proxy still reaches it the same way, and the image is still last. + assert!(pair("--publish", "127.0.0.1::8080")); + assert_eq!(a.last().unwrap(), rec.image_ref()); + assert_eq!(a.iter().filter(|x| *x == "--network").count(), 1); + } + + #[test] + fn the_guard_decides_the_network_and_require_refuses_without_it() { + assert_eq!(network_for(true, false), Ok(Some(APPS_NETWORK))); + assert_eq!(network_for(true, true), Ok(Some(APPS_NETWORK))); + // No guard (a non-root worker): the default bridge, as before. + assert_eq!(network_for(false, false), Ok(None)); + let err = network_for(false, true).unwrap_err(); + assert!(err.contains("require_egress_guard"), "{err}"); + } + + #[test] + fn apps_guard_blocks_private_ranges_and_the_host_but_lets_replies_back() { + let rules = guard_rules(APPS_BRIDGE); + let has = |needle: &[&str]| rules.iter().any(|r| r.iter().map(String::as_str).eq(needle.iter().copied())); + // The same forward rules container.run puts on its bridge. + for range in crate::container_run::GUARDED_RANGES { + assert!(has(&["DOCKER-USER", "-i", "c0apps0", "-d", range, "-j", "DROP"]), "{range}"); + } + assert!(crate::container_run::GUARDED_RANGES.contains(&"169.254.0.0/16"), "cloud metadata"); + // To the host: only replies to connections the host opened (the + // published loopback port); a blanket drop would cut the proxy off. + assert!(has(&["INPUT", "-i", "c0apps0", "-m", "conntrack", "!", "--ctstate", "ESTABLISHED", "-j", "DROP"])); + assert!(!has(&["INPUT", "-i", "c0apps0", "-j", "DROP"])); + assert_eq!(rules.iter().filter(|r| r[0] == "INPUT").count(), 1); + assert!(rules.iter().all(|r| r.contains(&"c0apps0".to_string()) && r.last().unwrap() == "DROP")); + } + + #[test] + fn layer_quota_only_on_overlay2_over_xfs_with_project_quotas() { + assert!(quota_supported("overlay2", "xfs", "rw,relatime,attr2,inode64,prjquota")); + assert!(quota_supported("overlay2", "xfs", "rw,pquota")); + assert!(!quota_supported("overlay2", "xfs", "rw,relatime,noquota")); + assert!(!quota_supported("overlay2", "extfs", "rw,relatime,prjquota"), "ext4, like most hosts"); + assert!(!quota_supported("overlayfs", "xfs", "rw,prjquota"), "the containerd image store"); + assert!(!quota_supported("", "", "")); + let mounts = "/dev/sda1 / ext4 rw,relatime 0 0\n/dev/sdb1 /var/lib/docker xfs rw,noatime,prjquota 0 0\nproc /proc proc rw 0 0\n"; + assert_eq!(mount_options(mounts, Path::new("/var/lib/docker")), Some("rw,noatime,prjquota")); + assert_eq!(mount_options(mounts, Path::new("/var/lib/docker/overlay2")), Some("rw,noatime,prjquota")); + assert_eq!(mount_options(mounts, Path::new("/var/lib/dockerx")), Some("rw,relatime"), "a prefix of the name is not the mount"); + assert_eq!(mount_options("", Path::new("/var/lib/docker")), None); + } + + #[test] + fn disk_limit_arithmetic() { + const MIB: u64 = 1 << 20; + const GIB: u64 = 1 << 30; + let limit = 2048 * MIB; + let (free, floor) = (100 * GIB, 20 * GIB); + // Under the limit with room on the host: nothing to do. + assert_eq!(disk_verdict(500 * MIB, 900 * MIB, limit, free, floor), None); + assert_eq!(disk_verdict(1024 * MIB, 1024 * MIB, limit, free, floor), None, "exactly the limit is fine"); + // Layer plus state over the limit: stopped, state kept. + let stop = disk_verdict(1500 * MIB, 600 * MIB, limit, free, floor).unwrap(); + assert!(!stop.drop_state); + assert!(stop.reason.contains("2100 MiB") && stop.reason.contains("2048 MiB limit"), "{}", stop.reason); + // A state directory over what an app may keep (a snapshot refused + // for size reports one byte more than the cap): stopped, and the + // volume is kept, since what it holds past the last snapshot is the + // only copy. + let cap = crate::apps::snapshot::MAX_STATE_BYTES as u64; + let stop = disk_verdict(0, cap + 1, limit, free, floor).unwrap(); + assert!(!stop.drop_state && stop.reason.contains("1024 MiB"), "{}", stop.reason); + // An operator limit below the state cap caps the state too. + assert!(!disk_verdict(0, 300 * MIB, 256 * MIB, free, floor).unwrap().drop_state); + // No per-app limit (0): the state cap and the host floor still hold. + assert_eq!(disk_verdict(50 * GIB, 0, 0, free, floor), None); + assert!(disk_verdict(0, cap + 1, 0, free, floor).is_some()); + // The host under the floor container.run keeps: stopped, state kept. + let stop = disk_verdict(MIB, MIB, limit, 19 * GIB, floor).unwrap(); + assert!(!stop.drop_state && stop.reason.contains("19 GiB free"), "{}", stop.reason); + // Sizes that would overflow do not wrap into "fine". + assert!(disk_verdict(u64::MAX, 1, limit, free, floor).is_some()); + // An unknown docker disk (free MAX, floor 0) never stops an app by itself. + assert_eq!(disk_verdict(0, 0, limit, u64::MAX, 0), None); + } + + #[test] + fn a_snapshot_over_the_cap_is_told_from_a_docker_failure() { + let over = anyhow::anyhow!("output is over the 1073741824 byte limit").context("docker cp"); + assert!(over_state_cap(&over)); + assert!(!over_state_cap(&anyhow::anyhow!("docker cp exited with exit status: 1"))); + } + #[test] fn restore_unpacks_into_the_volume_offline() { let a = restore_args("abc"); diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index 8e8e43fd..6e3424b2 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -107,6 +107,14 @@ pub struct Config { /// and pays bounties this node owes. #[serde(default)] pub swarm: crate::agent_swarm::SwarmConfig, + /// Blender renders (`render` role): whether Blender runs inside a + /// bubblewrap sandbox (`sandbox = "auto" | "off" | "require"`). + #[serde(default)] + pub render: crate::render::RenderConfig, + /// Text fine-tunes (`c0mpute infernet text serve`): the base models this + /// node downloads and runs for buyers, and the free disk it keeps. + #[serde(default)] + pub text: crate::text::TextConfig, } /// Adult mode on the buyer side. Every worker serves adult jobs and @@ -208,6 +216,16 @@ pub struct AppsConfig { /// Run only apps signed by these owner keys (hex ed25519, as `c0mpute /// apps status` shows them). Empty = anyone's apps. pub owners: Vec, + /// Keep app containers off private, loopback, link-local (metadata) and + /// CGNAT addresses and off this host, with the iptables guard + /// `container.run` uses (needs root; without it apps stay on docker's + /// default bridge and the node says so at start). + pub egress_guard: bool, + /// Run no apps at all unless that guard is in place. + pub require_egress_guard: bool, + /// Most disk one app may use, MiB: its state volume plus what it writes + /// inside its container. An app over it is stopped. + pub max_disk_mb: u64, } impl Default for AppsConfig { @@ -216,6 +234,9 @@ impl Default for AppsConfig { max_apps: 4, max_memory_mb: 2048, owners: Vec::new(), + egress_guard: true, + require_egress_guard: false, + max_disk_mb: crate::apps::DEFAULT_MAX_DISK_MB, } } } @@ -840,6 +861,8 @@ impl Default for Config { threatcrush: Default::default(), adult: Default::default(), swarm: Default::default(), + render: Default::default(), + text: Default::default(), } } } @@ -941,6 +964,22 @@ pub struct GatewayConfig { /// (`gateway_announce`). Opt out: false, or `C0MPUTE_GATEWAY_AUTO=0`. #[serde(default = "default_true")] pub auto_announce: bool, + /// Give every hosted site its own web origin: with + /// `site_host_suffix = "sites.example.com"`, `.sites.example.com` + /// serves that site at `/` and nothing else, and a browser opening + /// `/s//...` is sent there. Needs wildcard DNS and TLS for + /// `*.` (docs/hosting.md). Unset (the default): sites are served + /// under `/s/` only. `C0MPUTE_SITE_HOST_SUFFIX` overrides it. + #[serde(default)] + pub site_host_suffix: Option, + /// Pages served under the shared `/s//` path are sandboxed + /// (`Content-Security-Policy: sandbox ...`), so one site's scripts cannot + /// read another site's storage or call this gateway's APIs as the same + /// origin. Pages that need localStorage, cookies or a service worker + /// work on a custom domain or a per-site host instead. On by default; + /// `C0MPUTE_SANDBOX_SHARED_SITES=0` also turns it off. + #[serde(default = "default_true")] + pub sandbox_shared_sites: bool, } impl Default for GatewayConfig { @@ -949,6 +988,8 @@ impl Default for GatewayConfig { bind: "0.0.0.0:7777".into(), announce: None, auto_announce: true, + site_host_suffix: None, + sandbox_shared_sites: true, } } } diff --git a/node/crates/c0mpute-core/src/container_run.rs b/node/crates/c0mpute-core/src/container_run.rs index cf32af1f..6d41d3c7 100644 --- a/node/crates/c0mpute-core/src/container_run.rs +++ b/node/crates/c0mpute-core/src/container_run.rs @@ -1320,7 +1320,7 @@ pub fn guard_rules(bridge: &str) -> Vec> { rules } -fn is_root() -> bool { +pub(crate) fn is_root() -> bool { #[cfg(unix)] { // SAFETY: geteuid has no preconditions. @@ -1333,11 +1333,17 @@ fn is_root() -> bool { } /// Insert the guard rules for `bridge` (idempotent). True when they are in. -async fn ensure_guard(bridge: &str) -> bool { +pub(crate) async fn ensure_guard(bridge: &str) -> bool { + ensure_rules(guard_rules(bridge)).await +} + +/// Insert iptables `rules` (each as [`guard_rules`] gives them) that are not +/// in yet. True when all of them are in; always false for a non-root worker. +pub(crate) async fn ensure_rules(rules: Vec>) -> bool { if !is_root() { return false; } - for rule in guard_rules(bridge) { + for rule in rules { let (chain, rest) = rule.split_first().expect("non-empty rule"); let mut check = vec!["-C".to_string(), chain.clone()]; check.extend(rest.iter().cloned()); @@ -1370,8 +1376,17 @@ async fn ensure_guard(bridge: &str) -> bool { /// Make sure the jobs network exists (containers on it cannot reach each /// other) and, when configured, guard it. Returns whether egress is guarded. async fn ensure_jobs_network(cfg: &ContainersConfig) -> Result { + ensure_bridge_network(JOBS_NETWORK, JOBS_BRIDGE, "c0mpute.jobs=1") + .await + .context("create the jobs network")?; + Ok(cfg.egress_guard && ensure_guard(JOBS_BRIDGE).await) +} + +/// Make sure the docker bridge network `name` exists on the Linux interface +/// `bridge`, with its containers unable to reach each other. +pub(crate) async fn ensure_bridge_network(name: &str, bridge: &str, label: &str) -> Result<()> { let t = Duration::from_secs(30); - if docker_out(&s(&["network", "inspect", JOBS_NETWORK]), t) + if docker_out(&s(&["network", "inspect", name]), t) .await .is_err() { @@ -1379,20 +1394,20 @@ async fn ensure_jobs_network(cfg: &ContainersConfig) -> Result { "network", "create", "--driver=bridge", - &format!("--opt=com.docker.network.bridge.name={JOBS_BRIDGE}"), + &format!("--opt=com.docker.network.bridge.name={bridge}"), "--opt=com.docker.network.bridge.enable_icc=false", - "--label=c0mpute.jobs=1", - JOBS_NETWORK, + &format!("--label={label}"), + name, ]); if let Err(e) = docker_out(&create, t).await - && docker_out(&s(&["network", "inspect", JOBS_NETWORK]), t) + && docker_out(&s(&["network", "inspect", name]), t) .await .is_err() { - return Err(e).context("create the jobs network"); + return Err(e); } } - Ok(cfg.egress_guard && ensure_guard(JOBS_BRIDGE).await) + Ok(()) } /// The N-of-M hook: a buyer that ran the same build on several builders can diff --git a/node/crates/c0mpute-core/src/hosting/edge_api.rs b/node/crates/c0mpute-core/src/hosting/edge_api.rs index ceb92d47..821a78e3 100644 --- a/node/crates/c0mpute-core/src/hosting/edge_api.rs +++ b/node/crates/c0mpute-core/src/hosting/edge_api.rs @@ -178,6 +178,9 @@ pub(super) async fn chunk( let mut resp = (StatusCode::OK, bytes).into_response(); let hdr = resp.headers_mut(); hdr.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + // A chunk of a hosted page is that page's HTML: never let a + // browser guess a type for it on this origin. + hdr.insert("x-content-type-options", HeaderValue::from_static("nosniff")); hdr.insert( header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=31536000, immutable"), diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs index 14215e14..91510a08 100644 --- a/node/crates/c0mpute-core/src/hosting/mod.rs +++ b/node/crates/c0mpute-core/src/hosting/mod.rs @@ -26,7 +26,9 @@ pub mod git; mod git_browse; pub mod market; pub mod meta; +pub mod origin; pub mod proof; +pub mod relay; pub mod replicas; pub mod seed; pub mod settle; @@ -85,6 +87,32 @@ pub const MAX_BYTES_PER_OWNER: u64 = 16 * GIB_BYTES; const SITES_PER_MESSAGE: usize = 200; /// Other holders asked for a chunk before this node lets a surplus copy go. const DROP_PROBES: usize = 12; +/// How long a passed challenge keeps a holder counted toward the copies a +/// site needs, in holder TTLs: 108 x 200 s = 6 h. Long enough that the nodes +/// next in line for a site re-ask each holder a few times a day, not every +/// repair round; short enough that a holder that deleted its copy and kept +/// announcing stops counting the same day. +const PROOF_TTL_HOLDER_TTLS: u32 = 108; +/// Background challenges a repair round may start: in all, of one peer, and +/// for one site. A node that just started knows no proofs and may be next in +/// line for thousands of sites; it works through them at this rate, and until +/// a site's turn comes its announced holders are taken at their word. +const PROOF_ASKS_PER_ROUND: usize = 32; +const PROOF_ASKS_PER_PEER: usize = 4; +const PROOF_ASKS_PER_SITE: usize = MAX_REPLICAS; +/// Background challenges in flight at once. +const PROOF_SLOTS: usize = 8; +/// Before this node gives up a copy because other holders have the site, it +/// must have seen at least [`TARGET_REPLICAS`] of them pass its challenges at +/// this many separate times, over at least this long. Filling asks for one +/// recent pass only, so a new site still settles in minutes; letting go of +/// data is the step that cannot be undone. Keys that only announce and relay +/// then have to stay up and relay for hours before any honest holder trims, +/// and since a relay through the challenger itself is seen (see +/// [`Hosting::holds_now`]), the last honest holder is never talked into +/// dropping by peers that hold nothing. +pub const DROP_STANDING_PASSES: u32 = 3; +pub const DROP_STANDING_SPAN: Duration = Duration::from_secs(6 * 3600); /// Unfinished multi-part holding lists kept at once, and for how long. Each /// is a peer's word, so neither the number nor the size is open-ended. const MAX_PARTIAL_HOLDINGS: usize = 1024; @@ -252,7 +280,23 @@ struct Inner { /// When each site ([`WHOLE`]) or group was first seen over the target, /// for trimming (see [`Hosting::trim_due`]). over_since: std::sync::Mutex>, + /// When this node first asked the announced holders of a site to prove + /// their copies because it would otherwise take one itself (see + /// [`Hosting::fill_due`]). + unproven_since: std::sync::Mutex>, + /// Background challenges in flight, as (site id, holder). + proving: std::sync::Mutex>, + prove_slots: Semaphore, + /// Which held sites use a chunk (see [`relay`]). Filled only once + /// [`Hosting::refuses_fetch`] is in use. + chunk_index: std::sync::Mutex, + guards_fetches: std::sync::atomic::AtomicBool, dns: dns::Verifier, + /// `[gateway] site_host_suffix`, normalized: `.` is + /// that site's own origin (see [`origin`]). + site_host_suffix: Option, + /// `[gateway] sandbox_shared_sites`: pages under `/s/` are sandboxed. + sandbox_shared: bool, /// Optional handler for requests with a query string (podcast ad /// insertion installs one); it may decline and let the plain serve run. hook: std::sync::OnceLock>, @@ -343,6 +387,13 @@ pub struct Options { /// Pretend the disk has (free, total) bytes instead of asking the OS. /// For tests on machines whose disk is below the free-disk floor. pub disk_override: Option<(u64, u64)>, + /// `[gateway] site_host_suffix`: serve each site on its own host under + /// this suffix. `None`: as this machine is configured + /// ([`origin::configured`]); `Some("")`: off whatever is configured. + pub site_host_suffix: Option, + /// `[gateway] sandbox_shared_sites`. `None`: as this machine is + /// configured (on unless turned off). + pub sandbox_shared_sites: Option, } impl Default for Options { @@ -360,6 +411,8 @@ impl Default for Options { payout: None, receipts_path: None, disk_override: None, + site_host_suffix: None, + sandbox_shared_sites: None, } } } @@ -392,6 +445,25 @@ impl Hosting { .unwrap_or_default(); let challenger_hex = opts.challenger.as_ref().and_then(site::owner_hex); let records_path = opts.catalog_path.as_ref().map(|p| p.with_file_name("records.json")); + // Read the machine's configuration only for what the caller left open. + let (site_host_suffix, sandbox_shared) = match (opts.site_host_suffix, opts.sandbox_shared_sites) { + (Some(suffix), Some(sandbox)) => origin::resolve(Some(suffix), sandbox, None, None), + (suffix, sandbox) => { + let (cfg_suffix, cfg_sandbox) = origin::configured(); + ( + match suffix { + Some(s) => origin::resolve(Some(s), true, None, None).0, + None => cfg_suffix, + }, + sandbox.unwrap_or(cfg_sandbox), + ) + } + }; + if let Some(suffix) = &site_host_suffix { + // The gateway's listener keeps every other route off those hosts. + c0mpute_gateway::site_host::configure(suffix); + info!(%suffix, "hosting: each site is served on its own host, ."); + } let known: Vec = records_path .as_ref() .and_then(|p| std::fs::read(p).ok()) @@ -421,7 +493,14 @@ impl Hosting { layouts: RwLock::new(HashMap::new()), short_since: std::sync::Mutex::new(HashMap::new()), over_since: std::sync::Mutex::new(HashMap::new()), + unproven_since: std::sync::Mutex::new(HashMap::new()), + proving: std::sync::Mutex::new(HashSet::new()), + prove_slots: Semaphore::new(PROOF_SLOTS), + chunk_index: std::sync::Mutex::new(relay::ChunkIndex::default()), + guards_fetches: std::sync::atomic::AtomicBool::new(false), dns: opts.dns, + site_host_suffix, + sandbox_shared, hook: std::sync::OnceLock::new(), host_hook: std::sync::OnceLock::new(), challenger: opts.challenger, @@ -913,6 +992,7 @@ impl Hosting { return; } h.announce_holding().await; + h.refresh_chunk_index().await; h.connect_hosts().await; if last_records.elapsed() >= h.0.timing.records_every { h.announce_records().await; @@ -945,24 +1025,42 @@ impl Hosting { // A full node does not put itself forward for new copies (it would be // chosen, fail, and hold the slot from the next ranked node). let can_take = self.0.hosts && !self.is_full().await; + let mut asks = Asks::new(self.0.proving.lock().expect("proving").clone()); let (whole, striped): (Vec<(SiteRecord, Action)>, Vec) = { let mut map = self.0.map.write().await; let now = Instant::now(); + let now_secs = crate::payments::now_secs(); + let ttl = self.proof_ttl(); + let gap = self.standing_gap(); map.prune(now, self.0.timing.holder_ttl); + map.prune_proofs(now, ttl.max(self.standing_span())); self.0 .over_since .lock() .expect("over_since") .retain(|(s, _), _| map.records.contains_key(s)); - let mut whole = Vec::new(); + self.0 + .unproven_since + .lock() + .expect("unproven_since") + .retain(|s, _| map.records.contains_key(s)); + let mut whole: Vec<(SiteRecord, Action)> = Vec::new(); let mut striped = Vec::new(); + // Sites (by place in `whole`) whose unproven holders are to be + // asked, and sites this node is building standing for. + let mut doubted: Vec<(usize, Vec)> = Vec::new(); + let mut standing: Vec<(SiteRecord, Vec)> = Vec::new(); for r in map.records.values() { if stripe::is_striped(r.size) { striped.push(r.clone()); continue; } - let trim = self.trim_due(&r.site_id, WHOLE, map.holders(&r.site_id).len(), now); - let a = replicas::decide( + let holders = map.holders(&r.site_id); + let trim = self.trim_due(&r.site_id, WHOLE, holders.len(), now); + // An announced holder fills a slot once it has proven its + // copy; announcing alone is free. + let proven = |p: &str| p == self.0.me || map.proven(&r.site_id, p, now, now_secs, ttl); + let a = replicas::decide_counting( &map, &r.site_id, &self.0.me, @@ -970,11 +1068,63 @@ impl Hosting { TARGET_REPLICAS, MAX_REPLICAS, trim, + &proven, ); + let short_on_claims = || { + replicas::decide(&map, &r.site_id, &self.0.me, can_take, TARGET_REPLICAS, MAX_REPLICAS, trim) + == Action::Replicate + }; + if a == Action::Replicate && !short_on_claims() { + // Only unproven announcements stand between this node + // and a copy: they are asked below, and the copy waits + // out the grace. + let unproven: Vec = holders.iter().filter(|p| !proven(p)).cloned().collect(); + doubted.push((whole.len(), unproven)); + } else { + self.0.unproven_since.lock().expect("unproven_since").remove(&r.site_id); + } + // A free copy of a site that is over the target may be let + // go later, and that takes standing (see + // [`DROP_STANDING_SPAN`]): start counting the others' passes + // now, whichever holder ends up being the one to let go. + if a == Action::Nothing + && holders.len() > TARGET_REPLICAS + && holders.iter().any(|p| *p == self.0.me) + && !map.effective_preferred(&r.site_id).contains(&self.0.me) + { + let due: Vec = holders + .iter() + .filter(|p| **p != self.0.me && map.pass_due(&r.site_id, p, now, gap)) + .cloned() + .collect(); + standing.push((r.clone(), due)); + } whole.push((r.clone(), a)); } + // The round's challenges go first to sites already waiting on + // their holders (longest first), so a holder that was slow once + // is asked again before its site's grace runs out; then to + // sites not asked yet; and last to building standing. + { + let since = self.0.unproven_since.lock().expect("unproven_since"); + doubted.sort_by_key(|(i, _)| match since.get(&whole[*i].0.site_id) { + Some(t) => (0, Some(*t)), + None => (1, None), + }); + } + for (i, unproven) in doubted { + let most = unproven.len().min(PROOF_ASKS_PER_SITE); + let asked = asks.take(&whole[i].0, unproven, most); + if !self.fill_due(&whole[i].0.site_id, asked >= most, now) { + whole[i].1 = Action::Nothing; + } + } + for (rec, due) in standing { + asks.take(&rec, due, PROOF_ASKS_PER_SITE); + } (whole, striped) }; + self.spawn_proofs(std::mem::take(&mut asks.list)); for (rec, action) in whole { // A site we hold whose record moved on: fetch the new version. let stale = self @@ -1069,17 +1219,41 @@ impl Hosting { .map(|(g, hs)| self.trim_due(&rec.site_id, g as u32, hs.len(), now)) .collect(); let trim = |g: u32| trims.get(g as usize).copied().unwrap_or(false); - stripe::plan( - &map, - &rec.site_id, - &layout, - &self.0.me, - self.0.hosts, - TARGET_REPLICAS, - MAX_REPLICAS, - &widen, - &trim, - ) + // As for a whole site: a holder fills a group's slot once it + // has proven a copy, and while only unproven announcements + // stand between this node and a group they are asked first. + let now_secs = crate::payments::now_secs(); + let ttl = self.proof_ttl(); + let proven = |p: &str| p == self.0.me || map.proven(&rec.site_id, p, now, now_secs, ttl); + let plan_with = |counts: &dyn Fn(&str) -> bool| { + stripe::plan_counting( + &map, + &rec.site_id, + &layout, + &self.0.me, + self.0.hosts, + TARGET_REPLICAS, + MAX_REPLICAS, + &widen, + &trim, + counts, + ) + }; + let strict = plan_with(&proven); + let lenient = plan_with(&|_| true); + if strict.fetch == lenient.fetch { + self.0.unproven_since.lock().expect("unproven_since").remove(&rec.site_id); + strict + } else { + let mut unproven: Vec = raw.iter().flatten().filter(|p| !proven(p)).cloned().collect(); + unproven.sort(); + unproven.dedup(); + let mut asks = Asks::new(self.0.proving.lock().expect("proving").clone()); + let most = unproven.len().min(PROOF_ASKS_PER_SITE); + let asked = asks.take(&rec, unproven, most); + self.spawn_proofs(asks.list); + if self.fill_due(&rec.site_id, asked >= most, now) { strict } else { lenient } + } }; let stale = held.as_ref().is_some_and(|h| h.seq < rec.seq); if !plan.drop.is_empty() && !stale { @@ -1127,6 +1301,79 @@ impl Hosting { now.duration_since(since) >= self.0.timing.holder_ttl } + /// How long a passed challenge keeps a holder counted (see + /// [`PROOF_TTL_HOLDER_TTLS`]). + fn proof_ttl(&self) -> Duration { + self.0.timing.holder_ttl * PROOF_TTL_HOLDER_TTLS + } + + /// [`DROP_STANDING_SPAN`] on a node with the normal timing. Tests shrink + /// [`Timing`] to seconds and cannot wait six hours for a surplus copy to + /// go; there the span shrinks with the holder TTL. + fn standing_span(&self) -> Duration { + if self.0.timing.holder_ttl >= Timing::default().holder_ttl { + DROP_STANDING_SPAN + } else { + self.0.timing.holder_ttl * 2 + } + } + + /// The least time between two passes that both count toward standing. + fn standing_gap(&self) -> Duration { + self.standing_span() / DROP_STANDING_PASSES + } + + /// Whether this node may now take a copy of a site whose announced + /// holders it has not seen prove theirs. `asked`: a challenge of each of + /// them (of [`PROOF_ASKS_PER_SITE`] of them, when there are more) was + /// queued this round or is already running. + /// + /// Not before they have had the grace ([`Self::proof_grace`], one holder + /// TTL) to answer, counted from the first round they were asked: honest holders that are slow or + /// unreachable for a moment are asked again every round meanwhile, so a + /// healthy site does not gain a copy it then has to trim. The clock is + /// the site's, not the holders': announcing under new keys does not + /// restart it, and it only stops when the site has its proven copies (or + /// the unproven announcements go away, which lets the fill run at once). + /// A site whose turn to be asked has not come (the round's challenges + /// were used up) has no clock yet and is left as it is. + fn fill_due(&self, site_id: &str, asked: bool, now: Instant) -> bool { + let mut since = self.0.unproven_since.lock().expect("unproven_since"); + fill_clock(&mut since, site_id, asked, now, self.proof_grace()) + } + + /// How long announced holders have to prove their copies before a node + /// next in line takes one anyway: one holder TTL (what the network + /// already waits before it replaces a holder that went quiet), and + /// never less than two challenges at their slowest, so an answer that + /// is still on its way is not read as no answer (tests set the TTL + /// below the fetch timeout). + fn proof_grace(&self) -> Duration { + self.0.timing.holder_ttl.max(self.0.timing.fetch_timeout * 2) + } + + /// Challenge each (site, holder) in the background, a few at a time. + /// Passes land in the map ([`Self::holds_now`]); the next repair round + /// reads them. + fn spawn_proofs(&self, list: Vec<(SiteRecord, String)>) { + for (rec, holder) in list { + let key = (rec.site_id.clone(), holder.clone()); + if !self.0.proving.lock().expect("proving").insert(key.clone()) { + continue; + } + let h = self.clone(); + tokio::spawn(async move { + if let Ok(_slot) = h.0.prove_slots.acquire().await + && !h.is_stopped() + { + let ok = h.holds_now(&rec, &holder, None).await; + debug!(site = %rec.site_id, %holder, ok, "hosting: asked a holder to prove its copy"); + } + h.0.proving.lock().expect("proving").remove(&key); + }); + } + } + fn spawn_replicate(&self, rec: SiteRecord) { let h = self.clone(); tokio::spawn(async move { @@ -2101,7 +2348,16 @@ impl Hosting { /// Whether `holder` hands over a random chunk of `rec` right now, asked /// directly. A holding announcement is only a claim (any key can gossip - /// one); this is the check a node makes before it lets its own copy go. + /// one); this is the check a node makes before it counts the copy or + /// lets its own go. A pass is remembered ([`ReplicaMap::note_pass`]). + /// + /// A peer with no copy can still pass by fetching the chunk from a real + /// holder while it is being asked. When the real holder it asks is this + /// node, that shows: the chunk is read from this node's store between + /// the question and the answer. Such an answer is not a pass. The read + /// is matched by chunk, not by peer (a relay would ask under another + /// key), so now and then an unrelated read of the same chunk voids an + /// honest answer; the holder is simply asked again later. async fn holds_now(&self, rec: &SiteRecord, holder: &str, only: Option<&[u32]>) -> bool { let Ok(chunk) = self.challenge_chunk(rec, holder, only).await else { return false; @@ -2113,12 +2369,85 @@ impl Hosting { chunk_hash: hash, shard_index: None, }; + let watch = self.0.store.watch_reads(&hash); let got = tokio::time::timeout(self.0.timing.fetch_timeout, self.0.net.fetch_from_peer(pid, &req)).await; - matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash) + let ok = answer_passes(matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash), watch.seen()); + if ok { + self.note_pass(rec, holder).await; + } + ok + } + + /// The announced holders of `rec` whose copy this node has seen proven, + /// after asking those it has not (at most [`DROP_PROBES`] of them, at + /// once). What a publisher counts before it stops seeding: three keys + /// that only announce would otherwise read as three copies, the + /// publisher would leave, and the site would have no source left. + pub async fn verified_holders(&self, rec: &SiteRecord) -> Vec { + let (now_secs, ttl) = (crate::payments::now_secs(), self.proof_ttl()); + let holders = self.0.map.read().await.holders(&rec.site_id); + let unproven: Vec = { + let map = self.0.map.read().await; + let now = Instant::now(); + holders + .iter() + .filter(|p| **p != self.0.me && !map.proven(&rec.site_id, p, now, now_secs, ttl)) + .take(DROP_PROBES) + .cloned() + .collect() + }; + futures::future::join_all(unproven.iter().map(|p| self.holds_now(rec, p, None))).await; + let map = self.0.map.read().await; + let now = Instant::now(); + holders + .into_iter() + .filter(|p| *p == self.0.me || map.proven(&rec.site_id, p, now, now_secs, ttl)) + .collect() + } + + /// Remember that `holder` just passed a challenge for `rec`. + async fn note_pass(&self, rec: &SiteRecord, holder: &str) { + self.0.map.write().await.note_pass( + &rec.site_id, + holder, + &rec.manifest, + Instant::now(), + self.standing_gap(), + self.standing_span(), + ); + } + + /// The holders among `others` worth asking before a copy is let go: + /// those that would have standing with a pass now, and those whose pass + /// would add to their count. Asking the rest proves nothing new. + async fn drop_asks(&self, rec: &SiteRecord, others: Vec) -> Vec { + let (now, gap, span) = (Instant::now(), self.standing_gap(), self.standing_span()); + let map = self.0.map.read().await; + others + .into_iter() + .filter(|p| { + map.has_standing(&rec.site_id, p, now, DROP_STANDING_PASSES, span) + || map.pass_due(&rec.site_id, p, now, gap) + }) + .take(DROP_PROBES) + .collect() + } + + /// Whether `holder` (which just passed) has the standing a drop needs. + async fn has_standing(&self, rec: &SiteRecord, holder: &str) -> bool { + self.0.map.read().await.has_standing( + &rec.site_id, + holder, + Instant::now(), + DROP_STANDING_PASSES, + self.standing_span(), + ) } /// Whether at least `need` other announced holders of `rec` prove their - /// copy (see [`Self::holds_now`]). At most [`DROP_PROBES`] are asked. + /// copy now (see [`Self::holds_now`]) and have proven it before, often + /// and long enough to have standing (see [`DROP_STANDING_SPAN`]). At + /// most [`DROP_PROBES`] are asked. async fn others_prove_whole(&self, rec: &SiteRecord, need: usize) -> bool { let mut others: Vec = self.0.map.read().await.holders(&rec.site_id); others.retain(|p| *p != self.0.me); @@ -2126,9 +2455,10 @@ impl Hosting { use rand::seq::SliceRandom; others.shuffle(&mut rand::thread_rng()); } + let others = self.drop_asks(rec, others).await; let mut proven = 0; - for holder in others.iter().take(DROP_PROBES) { - if self.holds_now(rec, holder, None).await { + for holder in others.iter() { + if self.holds_now(rec, holder, None).await && self.has_standing(rec, holder).await { proven += 1; if proven >= need { return true; @@ -2139,8 +2469,10 @@ impl Hosting { } /// Of `groups` (ones this node is about to let go), those that at least - /// `need` other announced holders prove they hold. Each holder is asked - /// once, for a chunk of one of the groups in question that it claims. + /// `need` other announced holders prove they hold, with the standing a + /// whole site asks for ([`Self::others_prove_whole`]). Each holder is + /// asked once, for a chunk of one of the groups in question that it + /// claims. async fn groups_others_prove(&self, rec: &SiteRecord, groups: &[u32], need: usize) -> Vec { let Ok(layout) = self.layout(rec).await else { return Vec::new(); @@ -2159,9 +2491,10 @@ impl Hosting { use rand::seq::SliceRandom; claimants.shuffle(&mut rand::thread_rng()); } + let claimants = self.drop_asks(rec, claimants).await; let mut proven: HashSet = HashSet::new(); - for holder in claimants.iter().take(DROP_PROBES) { - if self.holds_now(rec, holder, Some(groups)).await { + for holder in claimants.iter() { + if self.holds_now(rec, holder, Some(groups)).await && self.has_standing(rec, holder).await { proven.insert(holder.clone()); } } @@ -2191,8 +2524,19 @@ impl Hosting { shard_index: None, }; let started = Instant::now(); + let watch = self.0.store.watch_reads(&hash); let got = tokio::time::timeout(self.0.timing.fetch_timeout, self.0.net.fetch_from_peer(pid, &req)).await; - let ok = matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash); + let right = matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash); + if right && watch.seen() { + // The chunk was read from this node while the holder was being + // asked for it (see [`Self::holds_now`]): neither a pass nor, + // since the read may have been someone else's, a signed failure. + bail!("{holder} answered while chunk {chunk} was being read from this node"); + } + let ok = right; + if ok { + self.note_pass(rec, holder).await; + } if !ok && !self.0.map.read().await.holders(&rec.site_id).iter().any(|p| p == holder) { // The holder announced it let the copy go while we were asking: // not a failed proof, just a stale target. No receipt. @@ -2351,6 +2695,65 @@ impl Hosting { (dues, outcomes) } + // ── relay guard ────────────────────────────────────────────────────── + + /// Whether a chunk fetch from `peer` should be answered as if the chunk + /// were not here: `peer` announces, whole, every site held here that + /// uses the chunk (see [`relay`]). For whatever serves libp2p chunk + /// fetches; gateway HTTP reads never come this way. + /// + /// The first call turns the chunk index on. Until the next announce + /// round has filled it, and for chunks of sites too big for it, the + /// answer is false (serve). + pub async fn refuses_fetch(&self, peer: &str, chunk: &Hash) -> bool { + self.0.guards_fetches.store(true, Ordering::Relaxed); + let map = self.0.map.read().await; + let index = self.0.chunk_index.lock().expect("chunk_index"); + relay::refuses(&map, &index, peer, chunk) + } + + /// Bring the chunk index in line with the catalog: sites let go are + /// forgotten, new or changed ones (another version, other groups) + /// indexed. Runs with the announcer, so it is at most one round behind; + /// behind means a fetch is served that could have been refused, or (a + /// site just let go) refused to a peer that claims the site anyway. + async fn refresh_chunk_index(&self) { + if !self.0.guards_fetches.load(Ordering::Relaxed) { + return; + } + let (held, stripes): (Vec, HashMap>) = { + let cat = self.0.catalog.read().await; + (cat.sites.values().cloned().collect(), cat.stripes.clone()) + }; + let ids: HashSet<&str> = held.iter().map(|r| r.site_id.as_str()).collect(); + self.0 + .chunk_index + .lock() + .expect("chunk_index") + .retain_sites(&|s| ids.contains(s)); + for r in &held { + let groups = stripes.get(&r.site_id); + let stamp = format!("{}:{}", r.manifest, groups.map(stripe::encode_groups).unwrap_or_default()); + let room = { + let index = self.0.chunk_index.lock().expect("chunk_index"); + if index.stamp(&r.site_id) == Some(stamp.as_str()) { + continue; + } + index.room() + }; + // A site plainly too big is not listed chunk by chunk to find + // that out (a whole one; a striped one is judged by its groups). + if groups.is_none() && r.size / site::FILE_CHUNK_BYTES as u64 > room as u64 { + self.0.chunk_index.lock().expect("chunk_index").skip(&r.site_id, &stamp); + continue; + } + let mut live = HashSet::new(); + self.live_chunks_of(r, groups, &mut live).await; + let chunks: Vec = live.iter().filter_map(|h| Hash::from_hex(h).ok()).collect(); + self.0.chunk_index.lock().expect("chunk_index").set(&r.site_id, &stamp, chunks); + } + } + // ── fetching ───────────────────────────────────────────────────────── /// One chunk: local (store, or a file this process is publishing), else @@ -2897,6 +3300,8 @@ impl Hosting { .route("/p/{site}/", get(private_loader)) .merge(git::router()) .fallback(serve_by_host) + // Last, so it is in front of every route above and the fallback. + .layer(axum::middleware::from_fn_with_state(self.clone(), serve_site_host)) .with_state(self.clone()) } } @@ -2969,6 +3374,79 @@ fn site_root_redirect(site: &str, query: Option<&str>) -> Response { } } +/// Whether an answer to a challenge is a pass: the right bytes came back, +/// and the chunk was not read from this node while the holder was being +/// asked (see [`Hosting::holds_now`]). +fn answer_passes(right_bytes: bool, read_from_us_meanwhile: bool) -> bool { + right_bytes && !read_from_us_meanwhile +} + +/// [`Hosting::fill_due`] on its table of clocks: true once `site_id`'s clock +/// has run for `grace`. The clock starts the first round the site's unproven +/// holders were all asked (`asked`): a site only some of whose holders could +/// be asked has not had its chance yet. +fn fill_clock(since: &mut HashMap, site_id: &str, asked: bool, now: Instant, grace: Duration) -> bool { + match since.get(site_id) { + Some(t) => now.duration_since(*t) >= grace, + None => { + if asked { + since.insert(site_id.to_string(), now); + } + false + } + } +} + +/// The background challenges one repair round starts, within its limits +/// ([`PROOF_ASKS_PER_ROUND`], [`PROOF_ASKS_PER_PEER`]). +struct Asks { + /// Challenges already running, as (site id, holder). + busy: HashSet<(String, String)>, + left: usize, + per_peer: HashMap, + list: Vec<(SiteRecord, String)>, +} + +impl Asks { + fn new(busy: HashSet<(String, String)>) -> Self { + Self { + busy, + left: PROOF_ASKS_PER_ROUND, + per_peer: HashMap::new(), + list: Vec::new(), + } + } + + /// Queue challenges of up to `most` of `holders` of `rec`, picked at + /// random (so which of many announcers are asked cannot be arranged), + /// as far as the round's limits allow. Returns how many are queued or + /// already running. + fn take(&mut self, rec: &SiteRecord, mut holders: Vec, most: usize) -> usize { + use rand::seq::SliceRandom; + holders.shuffle(&mut rand::thread_rng()); + let mut n = 0; + for holder in holders { + if n >= most { + break; + } + if self.busy.contains(&(rec.site_id.clone(), holder.clone())) { + n += 1; + continue; + } + let of_peer = self.per_peer.entry(holder.clone()).or_insert(0); + if self.left == 0 || *of_peer >= PROOF_ASKS_PER_PEER { + continue; + } + self.left -= 1; + *of_peer += 1; + self.busy.insert((rec.site_id.clone(), holder.clone())); + self.list.push((rec.clone(), holder)); + n += 1; + } + n + } +} + const GIB_BYTES: u64 = 1024 * 1024 * 1024; /// Free space a hosting copy must leave on the disk: 10 GiB or 5% of it, @@ -3187,7 +3665,7 @@ async fn serve_root( }; return axum::response::Redirect::permanent(&to).into_response(); } - serve_site(&h, &site, "", uri.query(), &headers).await + serve_site(&h, &site, "", Some(""), uri.query(), &headers).await } async fn serve_path( @@ -3196,18 +3674,92 @@ async fn serve_path( headers: HeaderMap, uri: axum::http::Uri, ) -> Response { - serve_site(&h, &site, &path, uri.query(), &headers).await + // The path as it was sent (still percent-encoded), for a redirect. + let prefix = format!("/s/{site}/"); + serve_site(&h, &site, &path, uri.path().strip_prefix(&prefix), uri.query(), &headers).await } -async fn serve_site(h: &Hosting, site: &str, path: &str, query: Option<&str>, headers: &HeaderMap) -> Response { - match h.record_for(site).await { +/// Serve `path` of a site under the shared `/s//` path. `rest`: the +/// path after `/s//` as the visitor sent it, when it could be told. +async fn serve_site( + h: &Hosting, + site: &str, + path: &str, + rest: Option<&str>, + query: Option<&str>, + headers: &HeaderMap, +) -> Response { + // With per-site hosts on, a browser opening a page here is sent to the + // site's own host, where the page has an origin of its own. Fetches + // that are not navigations (podcast apps, feed readers, the CLI, the + // `/p/` loader reading its ciphertext from this same origin) are + // answered here as before: they run no page on this origin, and a + // redirect to another host would break the ones that cannot follow it. + if let (Some(suffix), Some(rest)) = (h.0.site_host_suffix.as_deref(), rest) + && origin::wants_document(headers) + && let Some(url) = c0mpute_gateway::site_host::site_url(site, suffix, rest, query) + { + return origin::redirect_to_site_host(&url); + } + let mut resp = match h.record_for(site).await { Some(rec) => { let resp = h.serve_with_hook(&rec, path, query, headers).await; let page = meta::public_origin(headers).map(|o| format!("{o}/s/{site}/{path}")); with_absolute_meta(resp, page.as_deref()).await } None => (StatusCode::NOT_FOUND, "no such site on the network (yet)").into_response(), + }; + origin::apply(resp.headers_mut(), origin::Served::Shared, h.0.sandbox_shared); + if h.0.site_host_suffix.is_some() { + origin::vary_on_navigation(resp.headers_mut()); } + resp +} + +/// In front of every hosting route: a request on a per-site host +/// (`.`) is that site and nothing else, so it never reaches +/// the routes behind (`/hosting/v1/...`, `/p/`, git, another site's `/s/`). +/// Any other request passes through. +/// +/// The gateway's listener rewrites such a request to `/s/` and +/// marks it before routing (`c0mpute_gateway::site_host::isolate`), which is +/// what keeps the other services' routes off these hosts; a router used on +/// its own (tests, the podcast listener) gets the request as it was sent. +/// Both are handled here. +async fn serve_site_host(State(h): State, req: axum::extract::Request, next: axum::middleware::Next) -> Response { + use c0mpute_gateway::site_host::{self, SiteHost}; + let Some(suffix) = h.0.site_host_suffix.as_deref() else { + return next.run(req).await; + }; + let (site, path) = match req.extensions().get::() { + Some(mark) => { + let prefix = format!("/s/{}", mark.site); + let path = req.uri().path().strip_prefix(&prefix).unwrap_or("/"); + (mark.site.clone(), path.to_string()) + } + None => { + let Some(site) = + site_host::request_host(req.headers(), req.uri()).and_then(|x| site_host::site_of_host(&x, suffix)) + else { + return next.run(req).await; + }; + if req.method() != axum::http::Method::GET && req.method() != axum::http::Method::HEAD { + return (StatusCode::NOT_FOUND, "not found").into_response(); + } + (site, req.uri().path().to_string()) + } + }; + let (headers, query) = (req.headers(), req.uri().query()); + let mut resp = match h.record_for(&site).await { + Some(rec) => { + let resp = h.serve_with_hook(&rec, &path, query, headers).await; + let page = meta::public_origin(headers).map(|o| format!("{o}{path}")); + with_absolute_meta(resp, page.as_deref()).await + } + None => (StatusCode::NOT_FOUND, "no such site on the network (yet)").into_response(), + }; + origin::apply(resp.headers_mut(), origin::Served::SiteHost, h.0.sandbox_shared); + resp } /// Largest HTML page whose social meta tags get rewritten. @@ -3319,7 +3871,9 @@ async fn serve_by_host(State(h): State, req: axum::extract::Request) -> if h.0.dns.confirms(&host, &rec.site_id).await { let resp = h.serve_with_hook(&rec, uri.path(), uri.query(), &headers).await; let page = meta::public_origin(&headers).map(|o| format!("{o}{}", uri.path())); - return with_absolute_meta(resp, page.as_deref()).await; + let mut resp = with_absolute_meta(resp, page.as_deref()).await; + origin::apply(resp.headers_mut(), origin::Served::Domain, h.0.sandbox_shared); + return resp; } } (StatusCode::NOT_FOUND, "not found").into_response() @@ -3329,6 +3883,83 @@ async fn serve_by_host(State(h): State, req: axum::extract::Request) -> mod tests { use super::*; + #[test] + fn an_answer_relayed_through_the_challenger_is_not_a_pass() { + assert!(answer_passes(true, false)); + assert!(!answer_passes(true, true), "the chunk was read from us while we asked for it"); + assert!(!answer_passes(false, false)); + assert!(!answer_passes(false, true)); + } + + #[tokio::test] + async fn a_challenge_sees_the_chunk_read_from_this_node_meanwhile() { + // What Hosting::holds_now does around its fetch: a relay that has + // no copy can only answer by reading the chunk from a holder, and + // when that holder is the challenger the read is seen. + let dir = std::env::temp_dir().join(format!("c0mpute-relay-test-{}", rand::random::())); + let store = ChunkStore::open(&dir).await.unwrap().with_min_free_bytes(0); + let hash = store.put(b"the challenged chunk").await.unwrap(); + let serving = store.clone(); + // An honest holder answers from its own disk: nothing read here. + let watch = store.watch_reads(&hash); + assert!(answer_passes(true, watch.seen())); + // A relay asks this node (through the chunk server's clone of the + // store) for the very chunk it was challenged on. + let relayed = serving.get(&hash).await.unwrap(); + assert!(!answer_passes(Hash::of(&relayed) == hash, watch.seen())); + drop(watch); + // The next challenge starts clean. + assert!(answer_passes(true, store.watch_reads(&hash).seen())); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn unproven_holders_hold_their_slots_for_one_grace_and_no_longer() { + let grace = Duration::from_secs(200); + let t0 = Instant::now(); + let mut since = HashMap::new(); + // Not asked yet (the round's challenges were used up): no clock, no copy. + assert!(!fill_clock(&mut since, "s", false, t0, grace)); + assert!(since.is_empty()); + // Asked: the clock starts, and the holders have the grace to answer. + assert!(!fill_clock(&mut since, "s", true, t0, grace)); + assert!(!fill_clock(&mut since, "s", true, t0 + Duration::from_secs(199), grace)); + // A round that could not ask again (or new keys still to be asked) + // does not restart it. + assert!(!fill_clock(&mut since, "s", false, t0 + Duration::from_secs(150), grace)); + assert!(fill_clock(&mut since, "s", true, t0 + grace, grace)); + // And it stays due while the site is short, asked this round or not. + assert!(fill_clock(&mut since, "s", false, t0 + grace * 2, grace)); + // Another site has its own clock. + assert!(!fill_clock(&mut since, "t", true, t0 + grace * 2, grace)); + } + + #[test] + fn a_round_of_proof_asks_is_bounded_per_round_per_peer_and_per_site() { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let rec = |name: &str| SiteRecord::sign(&kp, name, &Hash::of(name.as_bytes()), 1, 10, vec![]).unwrap(); + let peers = |n: usize| -> Vec { (0..n).map(|i| format!("p{i}")).collect() }; + // Per site: at most `most`, however many announce it. + let mut asks = Asks::new(HashSet::new()); + assert_eq!(asks.take(&rec("a"), peers(40), PROOF_ASKS_PER_SITE), PROOF_ASKS_PER_SITE); + assert_eq!(asks.list.len(), PROOF_ASKS_PER_SITE); + // Per peer: one peer announcing many sites is asked about a few. + let mut asks = Asks::new(HashSet::new()); + let asked: usize = (0..10).map(|i| asks.take(&rec(&format!("s{i}")), peers(1), 5)).sum(); + assert_eq!(asked, PROOF_ASKS_PER_PEER); + // Per round. + let mut asks = Asks::new(HashSet::new()); + for i in 0..100 { + asks.take(&rec(&format!("s{i}")), vec![format!("q{i}")], 5); + } + assert_eq!(asks.list.len(), PROOF_ASKS_PER_ROUND); + // A challenge still running counts as asked and is not started twice. + let a = rec("a"); + let mut asks = Asks::new([(a.site_id.clone(), "p0".to_string())].into()); + assert_eq!(asks.take(&a, peers(1), 5), 1); + assert!(asks.list.is_empty()); + } + #[test] fn a_holding_part_past_its_own_count_does_not_grow_the_list() { assert!(holding_part_fits(0, 1, 0)); @@ -4372,7 +5003,9 @@ async fn wait_whole(hosting: &Hosting, record: &SiteRecord, wait: Duration) -> u while started.elapsed() < wait { let left = wait.saturating_sub(started.elapsed()); tokio::time::sleep(COPY_POLL.min(left)).await; - let holders = hosting.holders(&record.site_id).await; + // Copies that proved themselves, not announcements (anyone can + // announce): this is what decides when the publisher may leave. + let holders = hosting.verified_holders(record).await; let others = holders.len(); if others != copies { if others > copies { diff --git a/node/crates/c0mpute-core/src/hosting/origin.rs b/node/crates/c0mpute-core/src/hosting/origin.rs new file mode 100644 index 00000000..fb8ced51 --- /dev/null +++ b/node/crates/c0mpute-core/src/hosting/origin.rs @@ -0,0 +1,318 @@ +//! Which web origin a hosted site is served on, and the headers that follow +//! from it. +//! +//! A site is reached three ways: +//! +//! * under the shared path `/s//` ([`Served::Shared`]). +//! Every site there, and the gateway's own APIs, are one origin to a +//! browser, so pages are sandboxed: each document gets an origin of its +//! own that no other document shares; +//! * on a per-site host `.` ([`Served::SiteHost`]), when +//! the operator set `[gateway] site_host_suffix`. The host is the site's +//! origin, and nothing else is served on it; +//! * on a custom domain the owner signed into the record and DNS confirms +//! ([`Served::Domain`]). The domain is the site's origin. +//! +//! On the last two a page is an ordinary page: localStorage, cookies and +//! service workers are its own. + +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; + +/// How a response reaches the visitor. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Served { + /// `/s//...` + Shared, + /// `./...` + SiteHost, + /// A custom domain of the site. + Domain, +} + +/// The policy on pages under the shared path. `sandbox` without +/// `allow-same-origin` gives the document an opaque origin: it runs its +/// scripts and forms, but has no localStorage, cookies or service worker, +/// and its requests to this gateway are cross-origin ones. +pub const SHARED_SANDBOX: &str = "sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads \ +allow-pointer-lock allow-presentation"; + +/// Env overrides of `[gateway] site_host_suffix` / `sandbox_shared_sites`. +pub const SITE_HOST_SUFFIX_ENV: &str = "C0MPUTE_SITE_HOST_SUFFIX"; +pub const SANDBOX_SHARED_SITES_ENV: &str = "C0MPUTE_SANDBOX_SHARED_SITES"; + +/// Whether a response of this type is a document a browser runs scripts +/// from when it is opened: HTML, and the XML types (an SVG opened on its own +/// is a document too). +pub fn active_document(content_type: &str) -> bool { + let ct = content_type.split(';').next().unwrap_or("").trim().to_ascii_lowercase(); + matches!( + ct.as_str(), + "text/html" | "application/xhtml+xml" | "image/svg+xml" | "text/xml" | "application/xml" + ) || ct.ends_with("+xml") +} + +/// Add the headers a hosted response needs for where it is served. +/// `sandbox`: `[gateway] sandbox_shared_sites`. +/// +/// Everywhere: `X-Content-Type-Options: nosniff`, when absent. Under the +/// shared path, documents also get `Referrer-Policy: no-referrer` (the URL +/// names the gateway and the site) and, with `sandbox`, the sandbox policy; +/// and every response then allows cross-origin reads, since a sandboxed +/// page is cross-origin to its own assets and could not otherwise `fetch` +/// them, load its fonts or import its modules. Everything under `/s/` is +/// public and served without credentials, so that gives nothing away. +pub fn apply(headers: &mut HeaderMap, served: Served, sandbox: bool) { + let absent = |headers: &mut HeaderMap, name: &'static str, value: &'static str| { + if !headers.contains_key(name) { + headers.insert(name, HeaderValue::from_static(value)); + } + }; + absent(headers, "x-content-type-options", "nosniff"); + if served != Served::Shared { + return; + } + let document = headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .is_some_and(active_document); + if document { + absent(headers, "referrer-policy", "no-referrer"); + } + if !sandbox { + return; + } + if document { + // Appended, not replaced: a policy already on the response stays + // in force next to this one. + headers.append(header::CONTENT_SECURITY_POLICY, HeaderValue::from_static(SHARED_SANDBOX)); + } + absent(headers, "access-control-allow-origin", "*"); + absent(headers, "cross-origin-resource-policy", "cross-origin"); +} + +/// Whether a request is a browser opening a page (a navigation, in a tab or +/// a frame) rather than a fetch of a file: `Sec-Fetch-Mode: navigate`, or +/// for a browser too old to send that, an `Accept` that asks for HTML. +/// Scripts, feed readers, podcast apps, `curl` and the `/p/` loader's fetch +/// of its ciphertext are not navigations. +pub fn wants_document(headers: &HeaderMap) -> bool { + let get = |name: &str| headers.get(name).and_then(|v| v.to_str().ok()).map(|v| v.trim().to_ascii_lowercase()); + match get("sec-fetch-mode") { + Some(mode) => mode == "navigate", + None => get("accept").is_some_and(|a| a.contains("text/html")), + } +} + +/// Send a page opened under `/s//` to the site's own host. +/// +/// 307: the method is kept, and unlike a 301 or 308 a browser does not +/// remember it for good, so an operator who takes the suffix away again is +/// not left with visitors stuck on a dead name. Caches may keep it for five +/// minutes, per kind of request (only navigations are redirected). +pub fn redirect_to_site_host(url: &str) -> Response { + let Ok(location) = HeaderValue::from_str(url) else { + return (StatusCode::NOT_FOUND, "not found").into_response(); + }; + let mut resp = StatusCode::TEMPORARY_REDIRECT.into_response(); + let h = resp.headers_mut(); + h.insert(header::LOCATION, location); + h.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=300")); + vary_on_navigation(h); + h.insert("x-content-type-options", HeaderValue::from_static("nosniff")); + resp +} + +/// With per-site hosts on, what `/s/...` answers depends on whether the +/// request is a navigation: say so to caches. +pub fn vary_on_navigation(headers: &mut HeaderMap) { + headers.append(header::VARY, HeaderValue::from_static("Sec-Fetch-Mode, Accept")); +} + +/// `(site_host_suffix, sandbox_shared_sites)` as this machine is configured: +/// the env overrides, else the `[gateway]` section of the config file in its +/// usual place, else the defaults (no suffix, sandbox on). +pub fn configured() -> (Option, bool) { + let file = crate::config::Config::load_raw(&crate::config::default_config_path()) + .map(|c| c.gateway) + .unwrap_or_default(); + let env = |name: &str| std::env::var(name).ok().filter(|v| !v.trim().is_empty()); + resolve( + file.site_host_suffix, + file.sandbox_shared_sites, + env(SITE_HOST_SUFFIX_ENV), + env(SANDBOX_SHARED_SITES_ENV), + ) +} + +/// [`configured`] from its inputs. A suffix that is not a DNS name is +/// dropped (sites stay under `/s/`) rather than half-applied. +pub fn resolve( + file_suffix: Option, + file_sandbox: bool, + env_suffix: Option, + env_sandbox: Option, +) -> (Option, bool) { + let suffix = env_suffix.or(file_suffix).filter(|s| !s.trim().is_empty()).and_then(|raw| { + let ok = c0mpute_gateway::site_host::normalize_suffix(&raw); + if ok.is_none() { + tracing::warn!(suffix = %raw, "hosting: site_host_suffix is not a domain name, ignored"); + } + ok + }); + (suffix, crate::config::auto_install_enabled_with(file_sandbox, env_sandbox)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn headers_for(content_type: Option<&str>, served: Served, sandbox: bool) -> HeaderMap { + let mut h = HeaderMap::new(); + if let Some(ct) = content_type { + h.insert(header::CONTENT_TYPE, HeaderValue::from_str(ct).unwrap()); + } + apply(&mut h, served, sandbox); + h + } + + fn get<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> { + h.get(name).and_then(|v| v.to_str().ok()) + } + + #[test] + fn pages_under_the_shared_path_are_sandboxed_and_their_assets_readable() { + let h = headers_for(Some("text/html; charset=utf-8"), Served::Shared, true); + assert_eq!(get(&h, "content-security-policy"), Some(SHARED_SANDBOX)); + assert_eq!(get(&h, "referrer-policy"), Some("no-referrer")); + assert_eq!(get(&h, "access-control-allow-origin"), Some("*")); + assert_eq!(get(&h, "cross-origin-resource-policy"), Some("cross-origin")); + assert_eq!(get(&h, "x-content-type-options"), Some("nosniff")); + // The policy gives the page an opaque origin: never allow-same-origin. + assert!(SHARED_SANDBOX.starts_with("sandbox ") && !SHARED_SANDBOX.contains("allow-same-origin")); + assert!(!SHARED_SANDBOX.contains("allow-top-navigation")); + // An asset: readable by the sandboxed page, and not itself sandboxed + // (the policy would break a PDF opened in the browser's viewer). + for ct in ["text/css", "application/javascript", "image/png", "application/pdf", "audio/mpeg"] { + let h = headers_for(Some(ct), Served::Shared, true); + assert_eq!(get(&h, "content-security-policy"), None, "{ct}"); + assert_eq!(get(&h, "referrer-policy"), None, "{ct}"); + assert_eq!(get(&h, "access-control-allow-origin"), Some("*"), "{ct}"); + assert_eq!(get(&h, "cross-origin-resource-policy"), Some("cross-origin"), "{ct}"); + } + // A response with no type (a 304, a plain 404) still allows the read. + let h = headers_for(None, Served::Shared, true); + assert_eq!(get(&h, "content-security-policy"), None); + assert_eq!(get(&h, "access-control-allow-origin"), Some("*")); + assert_eq!(get(&h, "x-content-type-options"), Some("nosniff")); + } + + #[test] + fn an_svg_or_xml_opened_as_a_page_is_a_document_too() { + for ct in ["image/svg+xml", "application/xhtml+xml", "text/xml", "application/xml", "TEXT/HTML", "application/rss+xml"] { + assert!(active_document(ct), "{ct}"); + let h = headers_for(Some(ct), Served::Shared, true); + assert_eq!(get(&h, "content-security-policy"), Some(SHARED_SANDBOX), "{ct}"); + } + for ct in ["text/plain", "application/json", "text/htmlx", "video/mp4", ""] { + assert!(!active_document(ct), "{ct}"); + } + } + + #[test] + fn the_operator_can_turn_the_sandbox_off() { + let h = headers_for(Some("text/html"), Served::Shared, false); + assert_eq!(get(&h, "content-security-policy"), None); + assert_eq!(get(&h, "access-control-allow-origin"), None); + assert_eq!(get(&h, "cross-origin-resource-policy"), None); + // The baseline stays. + assert_eq!(get(&h, "referrer-policy"), Some("no-referrer")); + assert_eq!(get(&h, "x-content-type-options"), Some("nosniff")); + } + + #[test] + fn a_site_on_its_own_origin_is_an_ordinary_page() { + for served in [Served::SiteHost, Served::Domain] { + for sandbox in [true, false] { + let h = headers_for(Some("text/html; charset=utf-8"), served, sandbox); + assert_eq!(get(&h, "content-security-policy"), None, "{served:?}"); + assert_eq!(get(&h, "access-control-allow-origin"), None, "{served:?}"); + assert_eq!(get(&h, "cross-origin-resource-policy"), None, "{served:?}"); + assert_eq!(get(&h, "referrer-policy"), None, "{served:?}"); + assert_eq!(get(&h, "x-content-type-options"), Some("nosniff"), "{served:?}"); + } + } + } + + #[test] + fn headers_already_set_are_kept_and_a_policy_is_added_to_not_replaced() { + let mut h = HeaderMap::new(); + h.insert(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + h.insert("x-content-type-options", HeaderValue::from_static("nosniff")); + h.insert("referrer-policy", HeaderValue::from_static("same-origin")); + h.insert("access-control-allow-origin", HeaderValue::from_static("https://ads.example")); + h.insert(header::CONTENT_SECURITY_POLICY, HeaderValue::from_static("default-src 'self'")); + apply(&mut h, Served::Shared, true); + assert_eq!(get(&h, "referrer-policy"), Some("same-origin")); + assert_eq!(get(&h, "access-control-allow-origin"), Some("https://ads.example")); + let policies: Vec<&str> = h + .get_all(header::CONTENT_SECURITY_POLICY) + .iter() + .filter_map(|v| v.to_str().ok()) + .collect(); + assert_eq!(policies, vec!["default-src 'self'", SHARED_SANDBOX]); + assert_eq!(h.get_all("x-content-type-options").iter().count(), 1); + } + + #[test] + fn only_a_browser_opening_a_page_is_a_navigation() { + let with = |pairs: &[(&'static str, &'static str)]| { + let mut h = HeaderMap::new(); + for (k, v) in pairs { + h.insert(*k, HeaderValue::from_static(*v)); + } + h + }; + assert!(wants_document(&with(&[("sec-fetch-mode", "navigate"), ("accept", "text/html,*/*")]))); + assert!(wants_document(&with(&[("sec-fetch-mode", "Navigate")]))); + // fetch() from a page (the /p/ loader reading its ciphertext), an + // , a script: served where they asked. + assert!(!wants_document(&with(&[("sec-fetch-mode", "cors"), ("accept", "text/html")]))); + assert!(!wants_document(&with(&[("sec-fetch-mode", "same-origin")]))); + assert!(!wants_document(&with(&[("sec-fetch-mode", "no-cors"), ("accept", "image/avif,*/*")]))); + // An old browser sends no Sec-Fetch-Mode but asks for HTML. + assert!(wants_document(&with(&[("accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8")]))); + // curl, reqwest, a feed reader. + assert!(!wants_document(&with(&[("accept", "*/*")]))); + assert!(!wants_document(&with(&[("accept", "application/rss+xml")]))); + assert!(!wants_document(&HeaderMap::new())); + } + + #[test] + fn the_redirect_keeps_the_method_and_is_cached_briefly() { + let r = redirect_to_site_host("https://0123456789abcdef0123.sites.example.com/a%20b?x=1"); + assert_eq!(r.status(), StatusCode::TEMPORARY_REDIRECT); + assert_eq!( + get(r.headers(), "location"), + Some("https://0123456789abcdef0123.sites.example.com/a%20b?x=1") + ); + assert_eq!(get(r.headers(), "cache-control"), Some("public, max-age=300")); + assert_eq!(get(r.headers(), "vary"), Some("Sec-Fetch-Mode, Accept")); + assert_eq!(redirect_to_site_host("bad\r\nset-cookie: x").status(), StatusCode::NOT_FOUND); + } + + #[test] + fn the_env_overrides_the_file_and_a_bad_suffix_is_dropped() { + let s = |v: &str| Some(v.to_string()); + assert_eq!(resolve(None, true, None, None), (None, true), "the default: /s/ only, sandboxed"); + assert_eq!(resolve(s("*.Sites.Example.com"), true, None, None), (s("sites.example.com"), true)); + assert_eq!(resolve(s("a.example.com"), true, s("b.example.com"), None).0, s("b.example.com")); + assert_eq!(resolve(s("not a domain"), true, None, None).0, None); + assert_eq!(resolve(s("localhost"), true, None, None).0, None); + assert_eq!(resolve(s(" "), true, None, None).0, None); + assert!(!resolve(None, false, None, None).1); + assert!(!resolve(None, true, None, s("0")).1); + assert!(!resolve(None, true, None, s("off")).1); + assert!(resolve(None, false, None, s("1")).1); + } +} diff --git a/node/crates/c0mpute-core/src/hosting/proof.rs b/node/crates/c0mpute-core/src/hosting/proof.rs index 2fabaa28..5ddb817f 100644 --- a/node/crates/c0mpute-core/src/hosting/proof.rs +++ b/node/crates/c0mpute-core/src/hosting/proof.rs @@ -16,6 +16,25 @@ //! own copy rather than relaying from another holder; relaying costs the //! relayer bandwidth and latency, and the rest of the network still has the //! data, so the site stays up either way. +//! +//! What narrows the relay, without a new message: +//! +//! * a relay through the challenger itself is seen (the challenged chunk is +//! read from the challenger's store while it waits) and is not a pass +//! (`Hosting::holds_now`); +//! * a holder refuses chunk fetches from peers that announce the same site +//! whole ([`super::relay`]); +//! * a copy is let go only on the strength of holders that passed at +//! several separate times over hours (`DROP_STANDING_SPAN`). +//! +//! What was weighed and left out: a response deadline tight enough to miss a +//! relay's extra round trip also fails honest holders behind a relayed or +//! slow link, which is the churn the fill must not cause; asking for several +//! chunks, or a keyed digest over ranges of them, multiplies an honest +//! holder's cost by the same factor as the relay's (the relay still passes, +//! it only pays more) and the digest needs a message old nodes do not have. +//! A relay from a holder other than the challenger, under a second key, +//! still passes. use anyhow::Result; use c0mpute_proto::Hash; @@ -31,6 +50,9 @@ pub const KEEP_PER_HOLDER: usize = 16; pub const RECENT: usize = 5; /// …and loses preference when at least this many of them failed. pub const FAIL_LIMIT: usize = 3; +/// How far ahead of this node's clock a receipt may be dated and still be +/// read as a recent proof (see `ReplicaMap::proven`). +pub const CLOCK_SKEW_SECS: u64 = 300; /// "I asked `holder` for chunk `chunk` of `site_id` at `at`; it passed or not." #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/node/crates/c0mpute-core/src/hosting/relay.rs b/node/crates/c0mpute-core/src/hosting/relay.rs new file mode 100644 index 00000000..e35dfabc --- /dev/null +++ b/node/crates/c0mpute-core/src/hosting/relay.rs @@ -0,0 +1,243 @@ +//! A holder does not serve the peers that claim to hold the same site. +//! +//! A storage challenge ([`super::proof`]) cannot tell a peer that keeps its +//! copy from one that fetches the challenged chunk from a real holder and +//! passes it on. The cheapest place to fetch it is the real holders +//! themselves, so they refuse: a peer that announces the current version of a +//! site whole has every chunk of it and no reason to ask for one, and a node +//! that holds the site answers such a fetch as if the chunk were absent +//! ([`refuses`]). A relay is left with holders that do not run this rule and +//! with identities that announce nothing. +//! +//! Only whole claims are refused. A peer that announces another version, or +//! only some groups of a striped site, or nothing, is served as before: that +//! is every node still fetching (a copy counts, and is announced, only once +//! it is complete), so two nodes that each hold some groups always serve +//! each other the rest. + +use std::collections::HashMap; +use std::sync::Arc; + +use c0mpute_proto::Hash; + +use super::replicas::ReplicaMap; + +/// Chunks the index covers. Sites that do not fit are left out, and fetches +/// of their chunks are served to everyone as before. +pub const MAX_INDEXED_CHUNKS: usize = 131_072; + +/// Which held sites use a chunk, for the chunks of the sites this node +/// holds. Kept up from the catalog, bounded by [`MAX_INDEXED_CHUNKS`]. +#[derive(Default)] +pub struct ChunkIndex { + chunks: HashMap>>, + /// Per indexed site: what was indexed (version and groups) and its + /// chunks. A site too big for the index keeps its stamp and no chunks, + /// so it is not listed again until it changes. + sites: HashMap)>, + indexed: usize, +} + +impl ChunkIndex { + /// What `site` was last indexed as (see [`Self::set`]). + pub fn stamp(&self, site: &str) -> Option<&str> { + self.sites.get(site).map(|(s, _)| s.as_str()) + } + + /// Chunks the index can still take. + pub fn room(&self) -> usize { + MAX_INDEXED_CHUNKS.saturating_sub(self.indexed) + } + + /// Index `chunks` as the chunks of `site` held here, replacing what was + /// indexed for it. `stamp` names the version and groups, so the caller + /// can tell when to index again. Returns false (and indexes none of + /// them) when they do not fit. + pub fn set(&mut self, site: &str, stamp: &str, chunks: Vec) -> bool { + self.set_capped(site, stamp, chunks, MAX_INDEXED_CHUNKS) + } + + /// [`Self::set`] with the cap given (tests use a small one). + pub(crate) fn set_capped(&mut self, site: &str, stamp: &str, mut chunks: Vec, max: usize) -> bool { + self.remove(site); + chunks.sort_by(|a, b| a.0.cmp(&b.0)); + chunks.dedup(); + if self.indexed + chunks.len() > max { + self.sites.insert(site.to_string(), (stamp.to_string(), Vec::new())); + return false; + } + let name: Arc = Arc::from(site); + for h in &chunks { + self.chunks.entry(*h).or_default().push(name.clone()); + } + self.indexed += chunks.len(); + self.sites.insert(site.to_string(), (stamp.to_string(), chunks)); + true + } + + /// Remember `site` by its stamp without indexing any of its chunks (it + /// is too big for the index). + pub fn skip(&mut self, site: &str, stamp: &str) { + self.remove(site); + self.sites.insert(site.to_string(), (stamp.to_string(), Vec::new())); + } + + /// Forget `site`. + pub fn remove(&mut self, site: &str) { + let Some((_, chunks)) = self.sites.remove(site) else { + return; + }; + self.indexed -= chunks.len(); + for h in chunks { + if let Some(v) = self.chunks.get_mut(&h) { + v.retain(|s| &**s != site); + if v.is_empty() { + self.chunks.remove(&h); + } + } + } + } + + /// Forget every site `keep` does not name. + pub fn retain_sites(&mut self, keep: &dyn Fn(&str) -> bool) { + let gone: Vec = self.sites.keys().filter(|s| !keep(s)).cloned().collect(); + for s in gone { + self.remove(&s); + } + } + + /// The held sites that use `chunk`. + pub fn sites_of(&self, chunk: &Hash) -> &[Arc] { + self.chunks.get(chunk).map(|v| v.as_slice()).unwrap_or(&[]) + } +} + +/// Whether `peer` announces the current version of `site_id` whole: the +/// record's manifest, with no list of groups. +pub fn claims_whole(map: &ReplicaMap, site_id: &str, peer: &str) -> bool { + let (Some(rec), Some(h)) = (map.records.get(site_id), map.peers.get(peer)) else { + return false; + }; + h.sites.get(site_id).is_some_and(|(m, _)| *m == rec.manifest) && !h.groups.contains_key(site_id) +} + +/// Whether a fetch of `chunk` by `peer` is refused: the chunk belongs to +/// sites held here, and `peer` claims every one of them whole. A chunk two +/// sites share is served unless the peer claims both, since it may be +/// fetching the one it does not have. +pub fn refuses(map: &ReplicaMap, index: &ChunkIndex, peer: &str, chunk: &Hash) -> bool { + let sites = index.sites_of(chunk); + !sites.is_empty() && sites.iter().all(|s| claims_whole(map, s, peer)) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeSet, HashMap}; + use std::time::Instant; + + use super::*; + use crate::hosting::site::SiteRecord; + + fn rec(name: &str) -> SiteRecord { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + SiteRecord::sign(&kp, name, &Hash::of(name.as_bytes()), 1, 10, vec![]).unwrap() + } + + fn announce(m: &mut ReplicaMap, peer: &str, held: &[(&SiteRecord, Option<&str>, Option<&[u32]>)]) { + let mut sites = HashMap::new(); + let mut groups = HashMap::new(); + for (r, manifest, gs) in held { + sites.insert(r.site_id.clone(), (manifest.unwrap_or(&r.manifest).to_string(), r.seq)); + if let Some(gs) = gs { + groups.insert(r.site_id.clone(), gs.iter().copied().collect::>()); + } + } + m.apply_holding_full(peer, sites, groups, Instant::now(), true, None, None, None, vec![]); + } + + #[test] + fn a_peer_claiming_the_site_whole_is_refused_and_every_other_peer_is_served() { + let a = rec("a"); + let mut m = ReplicaMap::default(); + m.apply_record(a.clone()); + let chunk = Hash::of(b"chunk of a"); + let mut idx = ChunkIndex::default(); + assert!(idx.set(&a.site_id, "v1", vec![chunk])); + + announce(&mut m, "whole", &[(&a, None, None)]); + announce(&mut m, "nothing", &[]); + announce(&mut m, "old-version", &[(&a, Some("an-older-manifest"), None)]); + announce(&mut m, "some-groups", &[(&a, None, Some(&[0u32, 2][..]))]); + + assert!(refuses(&m, &idx, "whole", &chunk), "claims whole: has no reason to fetch"); + assert!(!refuses(&m, &idx, "nothing", &chunk), "a node taking its first copy"); + assert!(!refuses(&m, &idx, "old-version", &chunk), "a holder moving to the new version"); + assert!(!refuses(&m, &idx, "some-groups", &chunk), "a holder of other groups"); + assert!(!refuses(&m, &idx, "never-heard-of", &chunk)); + // A chunk of no site held here is nobody's business. + assert!(!refuses(&m, &idx, "whole", &Hash::of(b"something else"))); + // It stops claiming (a repairing node must): served again. + announce(&mut m, "whole", &[]); + assert!(!refuses(&m, &idx, "whole", &chunk)); + } + + #[test] + fn a_chunk_two_sites_share_is_refused_only_to_a_peer_claiming_both() { + let (a, b) = (rec("a"), rec("b")); + let mut m = ReplicaMap::default(); + m.apply_record(a.clone()); + m.apply_record(b.clone()); + let shared = Hash::of(b"shared"); + let only_a = Hash::of(b"only a"); + let mut idx = ChunkIndex::default(); + idx.set(&a.site_id, "v1", vec![shared, only_a]); + idx.set(&b.site_id, "v1", vec![shared]); + announce(&mut m, "has-a", &[(&a, None, None)]); + announce(&mut m, "has-both", &[(&a, None, None), (&b, None, None)]); + assert!(refuses(&m, &idx, "has-a", &only_a)); + assert!(!refuses(&m, &idx, "has-a", &shared), "it may be fetching b"); + assert!(refuses(&m, &idx, "has-both", &shared)); + // Letting b go leaves the chunk as a's alone. + idx.remove(&b.site_id); + assert!(refuses(&m, &idx, "has-a", &shared)); + } + + #[test] + fn two_nodes_holding_different_groups_serve_each_other() { + // Each announces the groups it has; neither claims the site whole, + // so each can fetch from the other the groups it lacks. + let a = rec("a"); + let mut m = ReplicaMap::default(); + m.apply_record(a.clone()); + announce(&mut m, "left", &[(&a, None, Some(&[0u32, 1][..]))]); + announce(&mut m, "right", &[(&a, None, Some(&[2u32, 3][..]))]); + let (g0, g2) = (Hash::of(b"group 0"), Hash::of(b"group 2")); + // What "left" and "right" each index: the chunks of their groups. + let mut left = ChunkIndex::default(); + left.set(&a.site_id, "v1:0,1", vec![g0]); + let mut right = ChunkIndex::default(); + right.set(&a.site_id, "v1:2,3", vec![g2]); + assert!(!refuses(&m, &left, "right", &g0)); + assert!(!refuses(&m, &right, "left", &g2)); + } + + #[test] + fn the_index_is_bounded_and_forgets_sites_let_go() { + let mut idx = ChunkIndex::default(); + let h = |i: u32| Hash::of(&i.to_le_bytes()); + assert!(idx.set_capped("a", "v1", vec![h(1), h(2), h(2)], 3)); + assert_eq!(idx.indexed, 2, "a chunk listed twice is one chunk"); + // Too big for what is left: remembered by its stamp, not indexed. + assert!(!idx.set_capped("b", "v1", vec![h(3), h(4)], 3)); + assert_eq!(idx.stamp("b"), Some("v1")); + assert!(idx.sites_of(&h(3)).is_empty()); + idx.skip("c", "v1"); + assert_eq!((idx.stamp("c"), idx.indexed), (Some("v1"), 2)); + // A new version of a replaces the old one's chunks. + assert!(idx.set_capped("a", "v2", vec![h(5)], 3)); + assert!(idx.sites_of(&h(1)).is_empty()); + assert_eq!(idx.sites_of(&h(5)).len(), 1); + idx.retain_sites(&|s| s == "b"); + assert_eq!((idx.indexed, idx.chunks.len(), idx.stamp("a")), (0, 0, None)); + } +} diff --git a/node/crates/c0mpute-core/src/hosting/replicas.rs b/node/crates/c0mpute-core/src/hosting/replicas.rs index 92d326c3..87019723 100644 --- a/node/crates/c0mpute-core/src/hosting/replicas.rs +++ b/node/crates/c0mpute-core/src/hosting/replicas.rs @@ -63,6 +63,27 @@ pub struct Holding { /// nobody announces go first, and then new ones are refused. pub const MAX_RECORDS: usize = 200_000; pub const MAX_RECEIPT_LISTS: usize = 200_000; +/// Proofs (see [`Proof`]) a node remembers. Past the cap the one passed +/// longest ago makes room. +pub const MAX_PROOFS: usize = 200_000; + +/// What this node has itself seen one holder of one site prove: it asked the +/// holder for chunks it picked at random and got the right bytes back. A +/// holding announcement is only a claim (any key can gossip one); this is +/// what makes a claimed copy count. +#[derive(Clone, Debug)] +pub struct Proof { + /// The version the passes were for. A new version starts over. + pub manifest: String, + /// The first pass of this run of passes, and the latest. + pub first: Instant, + pub last: Instant, + /// Passes counted in the run. At most one is counted per gap (see + /// [`ReplicaMap::note_pass`]), so three passes are three separate times. + pub passes: u32, + /// When a pass was last counted. + counted: Instant, +} #[derive(Debug, Default)] pub struct ReplicaMap { @@ -74,6 +95,8 @@ pub struct ReplicaMap { pub rtt_ms: HashMap, /// Recent availability receipts per (site id, holder), newest last. pub receipts: HashMap<(String, String), Vec>, + /// Challenges this node saw pass, per (site id, holder). + pub proofs: HashMap<(String, String), Proof>, } impl ReplicaMap { @@ -336,6 +359,122 @@ impl ReplicaMap { .collect() } + /// `holder` just passed a challenge by this node for `manifest` of + /// `site_id`. A pass less than `gap` after the last counted one renews + /// the proof without adding to the count; a run with no pass for `lapse` + /// (or for another version) starts over. + pub fn note_pass(&mut self, site_id: &str, holder: &str, manifest: &str, now: Instant, gap: Duration, lapse: Duration) { + self.note_pass_capped(site_id, holder, manifest, now, gap, lapse, MAX_PROOFS); + } + + /// [`Self::note_pass`] with the table's cap given (tests use a small one). + #[allow(clippy::too_many_arguments)] + pub(crate) fn note_pass_capped( + &mut self, + site_id: &str, + holder: &str, + manifest: &str, + now: Instant, + gap: Duration, + lapse: Duration, + max: usize, + ) { + let key = (site_id.to_string(), holder.to_string()); + if let Some(p) = self.proofs.get_mut(&key) + && p.manifest == manifest + && now.duration_since(p.last) <= lapse + { + p.last = now; + if now.duration_since(p.counted) >= gap { + p.passes += 1; + p.counted = now; + } + return; + } + if !self.proofs.contains_key(&key) && self.proofs.len() >= max { + let oldest = self.proofs.iter().min_by_key(|(_, p)| p.last).map(|(k, _)| k.clone()); + if let Some(k) = oldest { + self.proofs.remove(&k); + } + } + self.proofs.insert( + key, + Proof { + manifest: manifest.to_string(), + first: now, + last: now, + passes: 1, + counted: now, + }, + ); + } + + /// Forget proofs with no pass for `lapse`, and those of sites no longer + /// known. + pub fn prune_proofs(&mut self, now: Instant, lapse: Duration) { + let records = &self.records; + self.proofs + .retain(|(site, _), p| now.duration_since(p.last) <= lapse && records.contains_key(site)); + } + + /// Whether `holder`'s copy of the current version of `site_id` counts + /// toward the copies the site needs: this node saw it pass a challenge + /// within `ttl`, or the latest receipt for it signed by the site's owner + /// or payer (the keys [`proof::demoted`] trusts) is a pass no older than + /// `ttl`. `now_secs` is the wall clock receipts are dated with. + pub fn proven(&self, site_id: &str, holder: &str, now: Instant, now_secs: u64, ttl: Duration) -> bool { + let Some(rec) = self.records.get(site_id) else { + return false; + }; + if self + .proofs + .get(&(site_id.to_string(), holder.to_string())) + .is_some_and(|p| p.manifest == rec.manifest && now.duration_since(p.last) <= ttl) + { + return true; + } + let trusted = [rec.owner.as_str(), rec.payer_key()]; + self.receipts_for(site_id, holder) + .iter() + .filter(|r| r.manifest == rec.manifest && trusted.contains(&r.challenger.as_str())) + // A receipt dated ahead of this node's clock is not fresher + // than one dated now. + .filter(|r| r.at <= now_secs.saturating_add(proof::CLOCK_SKEW_SECS)) + .max_by(|a, b| a.at.cmp(&b.at).then(a.sig.cmp(&b.sig))) + .is_some_and(|r| r.ok && now_secs.saturating_sub(r.at) <= ttl.as_secs()) + } + + /// Whether `holder` has the standing a copy needs before this node gives + /// up its own on the strength of it: this node saw it pass at least + /// `passes` challenges at separate times, the first at least `span` ago, + /// with no gap of `span` between passes since. Receipts from other keys + /// do not count here: only what this node asked for itself. + pub fn has_standing(&self, site_id: &str, holder: &str, now: Instant, passes: u32, span: Duration) -> bool { + let Some(rec) = self.records.get(site_id) else { + return false; + }; + self.proofs + .get(&(site_id.to_string(), holder.to_string())) + .is_some_and(|p| { + p.manifest == rec.manifest + && p.passes >= passes + && now.duration_since(p.first) >= span + && now.duration_since(p.last) <= span + }) + } + + /// Whether a pass by `holder` now would be counted (see + /// [`Self::note_pass`]): nothing counted for the current version within + /// `gap`. Asking sooner proves nothing new. + pub fn pass_due(&self, site_id: &str, holder: &str, now: Instant, gap: Duration) -> bool { + let Some(rec) = self.records.get(site_id) else { + return false; + }; + self.proofs + .get(&(site_id.to_string(), holder.to_string())) + .is_none_or(|p| p.manifest != rec.manifest || now.duration_since(p.counted) >= gap) + } + /// Forget peers not heard from within `ttl`. pub fn prune(&mut self, now: Instant, ttl: Duration) { self.peers @@ -636,8 +775,9 @@ pub enum Action { Drop, } -/// Decide for `me` (which may or may not host). `trim`: the site has had more -/// than `target` copies for the whole grace period (see [`releases`]). +/// Decide for `me` (which may or may not host), taking every announced +/// holder at its word. `trim`: the site has had more than `target` copies for +/// the whole grace period (see [`releases`]). pub fn decide( map: &ReplicaMap, site_id: &str, @@ -646,6 +786,29 @@ pub fn decide( target: usize, max: usize, trim: bool, +) -> Action { + decide_counting(map, site_id, me, i_host, target, max, trim, &|_| true) +} + +/// [`decide`], where an announced holder fills one of the `target` copies +/// only when `counts` says so (it proved its copy, see +/// [`ReplicaMap::proven`]). Announcing is free, so three keys that only +/// announce would otherwise keep a site from ever being copied. +/// +/// A holder that does not count is still left out of the candidates and is +/// still a source to fetch from. The fill takes the `target` minus counted +/// top-ranked candidates, so however many nodes doubt the announced holders, +/// the same few (never more than `target`) are the ones to act. +#[allow(clippy::too_many_arguments)] +pub fn decide_counting( + map: &ReplicaMap, + site_id: &str, + me: &str, + i_host: bool, + target: usize, + max: usize, + trim: bool, + counts: &dyn Fn(&str) -> bool, ) -> Action { let holders = map.holders(site_id); let i_hold = holders.iter().any(|p| p == me); @@ -669,7 +832,8 @@ pub fn decide( if preferred.iter().any(|p| p == me) { return Action::Replicate; } - if holders.len() >= target { + let counted = holders.iter().filter(|p| counts(p)).count(); + if counted >= target { return Action::Nothing; } // The rendezvous fill. Preferred hosts that do not hold yet are left out: @@ -693,7 +857,7 @@ pub fn decide( } candidates.push(me.to_string()); } - if chosen_by_space(map, site_id, &candidates, target - holders.len()) + if chosen_by_space(map, site_id, &candidates, target - counted) .iter() .any(|p| p == me) { @@ -1027,6 +1191,204 @@ mod tests { assert!(m.live_peers().is_empty()); } + /// A site with `claimants` announcing it and `spare` hosting nodes that + /// do not hold it. Returns (map, site id, manifest, claimants, spares). + fn claimed(claimants: usize, spare: usize) -> (ReplicaMap, String, String, Vec, Vec) { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let r = rec(&kp, 1); + let mut m = ReplicaMap::default(); + m.apply_record(r.clone()); + let fake: Vec = (0..claimants).map(|i| format!("claim{i}")).collect(); + holding(&mut m, &r.site_id, &r.manifest, &fake); + let spares: Vec = (0..spare).map(|i| format!("spare{i:02}")).collect(); + hosting_peers(&mut m, &spares); + (m, r.site_id, r.manifest, fake, spares) + } + + const TTL: Duration = Duration::from_secs(1800); + const GAP: Duration = Duration::from_secs(7200); + const SPAN: Duration = Duration::from_secs(6 * 3600); + + /// The `peers` that would take a copy when only proven holders count. + fn takers(m: &ReplicaMap, site: &str, peers: &[String], now: Instant) -> Vec { + peers + .iter() + .filter(|p| { + let counts = |h: &str| h == p.as_str() || m.proven(site, h, now, 1_000_000, TTL); + decide_counting(m, site, p, true, 3, 5, false, &counts) == Action::Replicate + }) + .cloned() + .collect() + } + + #[test] + fn holders_that_only_announce_do_not_fill_the_target() { + let (m, site, _, _, spares) = claimed(3, 6); + let now = Instant::now(); + // Taken at their word, three announcements are three copies. + assert!(spares.iter().all(|p| decide(&m, &site, p, true, 3, 5, false) == Action::Nothing)); + // Counted only once proven, the site still needs all three, and + // exactly the three top-ranked spare nodes take them: not everyone + // who doubts the claims. + let t = takers(&m, &site, &spares, now); + assert_eq!(t.len(), 3, "{t:?}"); + let mut want = chosen(&site, &spares, 3); + want.sort(); + assert_eq!(t, want); + } + + #[test] + fn proven_holders_fill_the_target_and_a_stale_proof_expires() { + let (mut m, site, manifest, fake, spares) = claimed(3, 6); + let t0 = Instant::now(); + for p in &fake[..2] { + m.note_pass(&site, p, &manifest, t0, GAP, SPAN); + } + // Two proven: one more copy, from the top-ranked spare node alone. + assert_eq!(takers(&m, &site, &spares, t0), chosen(&site, &spares, 1)); + m.note_pass(&site, &fake[2], &manifest, t0, GAP, SPAN); + assert!(takers(&m, &site, &spares, t0).is_empty(), "three proven copies: nothing to do"); + assert!(takers(&m, &site, &spares, t0 + TTL).is_empty()); + // Past the TTL the proofs no longer count. + let late = t0 + TTL + Duration::from_secs(1); + assert!(!m.proven(&site, &fake[0], late, 1_000_000, TTL)); + assert_eq!(takers(&m, &site, &spares, late).len(), 3); + // A proof is for one version of the site. + m.note_pass(&site, &fake[0], "another-manifest", t0, GAP, SPAN); + assert!(!m.proven(&site, &fake[0], t0, 1_000_000, TTL)); + } + + #[test] + fn the_ceiling_still_holds_when_claims_do_not_count() { + // Eight keys announce a site none of them holds. The fill is still + // three copies from the three top-ranked nodes, however many nodes + // look, and once those three are proven nobody else takes one. + let (mut m, site, manifest, fake, spares) = claimed(8, 12); + let now = Instant::now(); + let first = takers(&m, &site, &spares, now); + assert_eq!(first.len(), 3, "{first:?}"); + holding(&mut m, &site, &manifest, &first); + for p in &first { + m.note_pass(&site, p, &manifest, now, GAP, SPAN); + } + let rest: Vec = spares.iter().filter(|p| !first.contains(p)).cloned().collect(); + assert!(takers(&m, &site, &rest, now).is_empty(), "three proven copies stop the fill"); + // With eleven announced copies the site is far over the ceiling, and + // the rule above it is unchanged: one holder a round is named to let + // go (whether it does is decided on proofs of the others, see + // `Hosting::others_prove_whole`). + let all: Vec = fake.iter().chain(first.iter()).cloned().collect(); + assert_eq!(droppers(&m, &site, &all, false).len(), 1); + } + + #[test] + fn a_trusted_receipt_proves_a_holder_until_it_is_stale() { + use super::super::proof::{Challenge, Receipt}; + let owner = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let stranger = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let r = rec(&owner, 1); + let mut m = ReplicaMap::default(); + m.apply_record(r.clone()); + holding(&mut m, &r.site_id, &r.manifest, &["h".to_string()]); + let chunk = Hash::of(b"c").to_hex(); + let receipt = |kp: &c0mpute_net::libp2p::identity::Keypair, ok: bool, at: u64| { + Receipt::sign( + kp, + Challenge { + site_id: &r.site_id, + manifest: &r.manifest, + holder: "h", + chunk: &chunk, + ok, + rtt_ms: 1, + at, + }, + ) + .unwrap() + }; + let now = Instant::now(); + let proven = |m: &ReplicaMap, secs: u64| m.proven(&r.site_id, "h", now, secs, TTL); + // Anyone can sign a receipt: a stranger's proves nothing. + m.apply_receipt(receipt(&stranger, true, 1_000)); + assert!(!proven(&m, 1_000)); + m.apply_receipt(receipt(&owner, true, 1_000)); + assert!(proven(&m, 1_000) && proven(&m, 1_000 + TTL.as_secs())); + assert!(!proven(&m, 1_001 + TTL.as_secs()), "an old pass is not a copy today"); + // The owner's latest word wins, and a date from the future is not one. + m.apply_receipt(receipt(&owner, false, 1_100)); + assert!(!proven(&m, 1_200)); + m.apply_receipt(receipt(&owner, true, 900_000)); + assert!(!proven(&m, 1_200)); + } + + #[test] + fn one_fresh_pass_counts_for_the_fill_but_dropping_needs_standing() { + let (mut m, site, manifest, fake, _) = claimed(1, 0); + let h = &fake[0]; + let t0 = Instant::now(); + let standing = |m: &ReplicaMap, at: Instant| m.has_standing(&site, h, at, 3, SPAN); + assert!(m.pass_due(&site, h, t0, GAP), "never asked: a pass would count"); + m.note_pass(&site, h, &manifest, t0, GAP, SPAN); + assert!(m.proven(&site, h, t0, 0, TTL), "one pass fills a slot"); + assert!(!standing(&m, t0)); + // Passes close together are one time, however many there are. + for s in 1..50 { + m.note_pass(&site, h, &manifest, t0 + Duration::from_secs(s), GAP, SPAN); + } + assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 1); + assert!(!m.pass_due(&site, h, t0 + Duration::from_secs(60), GAP)); + assert!(!standing(&m, t0 + SPAN), "six hours since one pass is not three passes"); + // Three passes two hours apart, but only four hours in: not yet. + m.note_pass(&site, h, &manifest, t0 + GAP, GAP, SPAN); + m.note_pass(&site, h, &manifest, t0 + GAP * 2, GAP, SPAN); + assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 3); + assert!(!standing(&m, t0 + GAP * 2)); + // A pass at six hours: three or more separate times spanning six hours. + m.note_pass(&site, h, &manifest, t0 + SPAN, GAP, SPAN); + assert!(standing(&m, t0 + SPAN)); + } + + #[test] + fn standing_lapses_and_the_count_starts_over() { + let (mut m, site, manifest, fake, _) = claimed(1, 0); + let h = &fake[0]; + let t0 = Instant::now(); + for i in 0..4 { + m.note_pass(&site, h, &manifest, t0 + GAP * i, GAP, SPAN); + } + let end = t0 + GAP * 3; + assert!(m.has_standing(&site, h, end, 3, SPAN)); + // No pass for a whole span: the standing is gone... + let late = end + SPAN + Duration::from_secs(1); + assert!(!m.has_standing(&site, h, late, 3, SPAN)); + // ...and the next pass is the first of a new run, not the fifth. + m.note_pass(&site, h, &manifest, late, GAP, SPAN); + let p = &m.proofs[&(site.clone(), h.clone())]; + assert_eq!((p.passes, p.first), (1, late)); + // A new version of the site starts over too. + m.note_pass(&site, h, "next-version", late + GAP, GAP, SPAN); + assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 1); + // Pruning drops what lapsed, and proofs of sites no longer known. + m.note_pass("gone-site", h, &manifest, late + GAP, GAP, SPAN); + m.prune_proofs(late + GAP, SPAN); + assert_eq!(m.proofs.len(), 1); + m.prune_proofs(late + GAP + SPAN + Duration::from_secs(1), SPAN); + assert!(m.proofs.is_empty()); + } + + #[test] + fn the_proof_table_is_capped_and_the_oldest_pass_makes_room() { + let mut m = ReplicaMap::default(); + let t0 = Instant::now(); + for i in 0..3u64 { + m.note_pass_capped("s", &format!("h{i}"), "m", t0 + Duration::from_secs(i), GAP, SPAN, 3); + } + m.note_pass_capped("s", "h9", "m", t0 + Duration::from_secs(9), GAP, SPAN, 3); + assert_eq!(m.proofs.len(), 3); + assert!(!m.proofs.contains_key(&("s".to_string(), "h0".to_string()))); + assert!(m.proofs.contains_key(&("s".to_string(), "h9".to_string()))); + } + fn pid() -> String { c0mpute_net::libp2p::identity::Keypair::generate_ed25519() .public() diff --git a/node/crates/c0mpute-core/src/hosting/stripe.rs b/node/crates/c0mpute-core/src/hosting/stripe.rs index 730b7e8f..bcba49b8 100644 --- a/node/crates/c0mpute-core/src/hosting/stripe.rs +++ b/node/crates/c0mpute-core/src/hosting/stripe.rs @@ -239,6 +239,27 @@ pub fn plan( max: usize, widen: &dyn Fn(u32) -> usize, trim: &dyn Fn(u32) -> bool, +) -> Plan { + plan_counting(map, site_id, layout, me, i_host, target, max, widen, trim, &|_| true) +} + +/// [`plan`], where an announced holder of a group fills one of its `target` +/// copies only when `counts` says so (it proved a copy of the site, see +/// `ReplicaMap::proven`), as in `replicas::decide_counting`. Releasing is +/// unchanged: it is decided on every announced holder and then checked +/// against proofs by the caller. +#[allow(clippy::too_many_arguments)] +pub fn plan_counting( + map: &ReplicaMap, + site_id: &str, + layout: &Layout, + me: &str, + i_host: bool, + target: usize, + max: usize, + widen: &dyn Fn(u32) -> usize, + trim: &dyn Fn(u32) -> bool, + counts: &dyn Fn(&str) -> bool, ) -> Plan { let holders = map.group_holders(site_id, layout.len(), true); let preferred = map.effective_preferred(site_id); @@ -278,7 +299,8 @@ pub fn plan( wanted.push((0, score(&key, me), g)); continue; } - if hs.len() >= target { + let counted = hs.iter().filter(|p| counts(p)).count(); + if counted >= target { continue; } if hs.iter().any(|h| same_machine(&machines, h, me)) { @@ -308,9 +330,9 @@ pub fn plan( } candidates.push(me.to_string()); } - let n = target - hs.len() + widen(g); + let n = target - counted + widen(g); if chosen(&key, &candidates, n).iter().any(|p| p == me) { - wanted.push((1 + hs.len(), score(&key, me), g)); + wanted.push((1 + counted, score(&key, me), g)); } } // Paid groups, then groups with fewest copies, then a stable order. @@ -469,6 +491,39 @@ mod tests { map.set_machine(p, machine); } + #[test] + fn groups_whose_holders_only_announce_are_still_filled() { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let m = manifest(64 * 8); + let rec = big_record(&kp, &m); + let layout = Layout::of(&m); + let n = layout.len() as u32; + let peers: Vec = (0..9).map(|i| format!("peer{i}")).collect(); + let mut map = map_with(&peers, &rec); + // Three keys announce every group. + for p in &peers[..3] { + hold_all(&mut map, &rec, p, 0..n); + } + let takers = |counts: &dyn Fn(&str) -> bool| -> Vec { + let mut t = vec![0usize; layout.len()]; + for p in &peers[3..] { + let mine = |h: &str| h == p.as_str() || counts(h); + let plan = plan_counting(&map, &rec.site_id, &layout, p, true, 3, 5, &|_| 0, &|_| false, &mine); + assert!(plan.drop.is_empty()); + for g in plan.fetch { + t[g as usize] += 1; + } + } + t + }; + // Taken at their word (or all proven) every group is full. + assert!(takers(&|_| true).iter().all(|t| *t == 0)); + // Unproven, they fill nothing: each group gets three real takers. + assert!(takers(&|_| false).iter().all(|t| *t == 3)); + // One of them proven: two more takers per group. + assert!(takers(&|h| h == "peer0").iter().all(|t| *t == 2)); + } + #[test] fn two_nodes_on_one_machine_are_one_copy() { let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); diff --git a/node/crates/c0mpute-core/src/render.rs b/node/crates/c0mpute-core/src/render.rs index 75e44edf..bde08c5c 100644 --- a/node/crates/c0mpute-core/src/render.rs +++ b/node/crates/c0mpute-core/src/render.rs @@ -21,7 +21,11 @@ //! Safety: Blender always runs with `--disable-autoexec` before the file is //! loaded, so Python embedded in the `.blend` (drivers, registered text //! blocks, app handlers) never runs. The only Python is the one-line settings -//! expression this module builds from validated integers. +//! expression this module builds from validated integers, and the constant +//! [`SANITIZE_EXPR`], which cuts every file reference that leaves the job +//! directory before the render starts. Where bubblewrap works (Linux), +//! Blender also runs inside it, with no network and nothing of the host but +//! the system directories and the job directory ([`Sandbox`]). use std::collections::HashMap; use std::ffi::OsString; @@ -74,6 +78,65 @@ pub const MAX_SAMPLES: u32 = 16_384; const CACHE_MAX_BYTES: u64 = 4 * 1024 * 1024 * 1024; const PROBE_TIMEOUT: Duration = Duration::from_secs(90); +// ─── config ────────────────────────────────────────────────────────────── + +/// `[render]` in config.toml. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct RenderConfig { + /// `"auto"` (the default): run Blender inside bubblewrap where it works, + /// otherwise with scene sanitisation only. `"off"`: never use + /// bubblewrap. `"require"`: serve no render that cannot be sandboxed. + /// `C0MPUTE_RENDER_SANDBOX` wins over the file. + pub sandbox: String, +} + +impl Default for RenderConfig { + fn default() -> Self { + Self { sandbox: "auto".into() } + } +} + +impl RenderConfig { + /// The mode `sandbox` (or `env`, when set) names. A value that is not + /// one of the three is read as `auto`, so a typo never stops a node. + pub fn mode_with(&self, env: Option) -> SandboxMode { + let value = env.filter(|v| !v.trim().is_empty()).unwrap_or_else(|| self.sandbox.clone()); + SandboxMode::parse(&value).unwrap_or_else(|| { + warn!(value = %value, "render: sandbox must be auto, off or require; using auto"); + SandboxMode::Auto + }) + } +} + +/// Whether Blender runs inside bubblewrap (`[render] sandbox`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SandboxMode { + #[default] + Auto, + Off, + Require, +} + +impl SandboxMode { + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "auto" => Some(Self::Auto), + "off" => Some(Self::Off), + "require" => Some(Self::Require), + _ => None, + } + } +} + +/// The node's `[render] sandbox`, from its config file and the environment. +/// The render role is started without a config handle, so this reads the +/// file itself, as the speech gateway does for `[adult]`. +fn sandbox_mode() -> SandboxMode { + let file = crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default(); + file.render.mode_with(std::env::var("C0MPUTE_RENDER_SANDBOX").ok()) +} + // ─── spec ──────────────────────────────────────────────────────────────── /// Render engine. `EEVEE` is mapped to the identifier the worker's Blender @@ -634,25 +697,36 @@ pub struct Runtime { pub version: (u32, u32, u32), /// Cycles GPU backends with at least one device, best first. pub gpu_backends: Vec, + /// The bubblewrap sandbox Blender runs in, where one works. + pub sandbox: Option, + pub sandbox_mode: SandboxMode, } /// Preference order when several backends see devices. const GPU_BACKENDS: &[&str] = &["OPTIX", "CUDA", "HIP", "METAL", "ONEAPI"]; impl Runtime { + /// Blender answers and, with `sandbox = "require"`, the sandbox works. pub fn ready(&self) -> bool { self.version_line.is_some() + && (self.sandbox_mode != SandboxMode::Require || self.sandbox.is_some()) } pub fn gpu(&self) -> Option<&str> { self.gpu_backends.first().map(String::as_str) } + /// Outside `require` every job may run; with it, only what the sandbox + /// was seen to do (`can`). + fn sandbox_allows(&self, can: impl Fn(&Sandbox) -> bool) -> bool { + self.sandbox_mode != SandboxMode::Require || self.sandbox.as_ref().is_some_and(can) + } pub fn can_serve(&self, parsed: &Parsed) -> bool { self.can_serve_engine(parsed.engine, parsed.device) } pub fn can_serve_engine(&self, engine: Engine, device: Device) -> bool { self.ready() && match (engine.needs_gpu(), device) { - (true, _) | (false, Device::Gpu) => self.gpu().is_some(), + (true, _) => self.gpu().is_some() && self.sandbox_allows(|s| s.raster), + (false, Device::Gpu) => self.gpu().is_some() && self.sandbox_allows(|s| s.cycles_gpu), (false, _) => true, } } @@ -661,22 +735,47 @@ impl Runtime { if parsed.engine != Engine::Cycles || parsed.device == Device::Cpu { return None; } + // `require` with a sandbox that cannot reach the GPU: AUTO means CPU. + if !self.sandbox_allows(|s| s.cycles_gpu) { + return None; + } self.gpu().map(str::to_string) } + /// The sandbox this job's Blender runs in. `None` runs it directly: + /// there is no sandbox, or (outside `require`) the job needs a GPU the + /// sandbox was not seen to reach. + pub fn sandbox_for(&self, parsed: &Parsed) -> Option<&Sandbox> { + let sandbox = self.sandbox.as_ref()?; + let usable = if parsed.engine.needs_gpu() { + sandbox.raster + } else if self.cycles_device(parsed).is_some() { + sandbox.cycles_gpu + } else { + true + }; + usable.then_some(sandbox) + } pub fn tags(&self) -> Vec { if !self.ready() { return Vec::new(); } let mut tags = vec![ROLE_TAG.to_string(), Engine::Cycles.tag()]; if self.gpu().is_some() { - tags.push(GPU_TAG.to_string()); - tags.push(Engine::Eevee.tag()); - tags.push(Engine::Workbench.tag()); - tags.extend( - self.gpu_backends - .iter() - .map(|b| format!("c0mpute:render:device:{}", b.to_ascii_lowercase())), - ); + let cycles = self.sandbox_allows(|s| s.cycles_gpu); + if cycles { + tags.push(GPU_TAG.to_string()); + } + if self.sandbox_allows(|s| s.raster) { + tags.push(Engine::Eevee.tag()); + tags.push(Engine::Workbench.tag()); + } + if cycles { + tags.extend( + self.gpu_backends + .iter() + .map(|b| format!("c0mpute:render:device:{}", b.to_ascii_lowercase())), + ); + } } tags } @@ -762,12 +861,29 @@ fn probe() -> Runtime { .map(|out| parse_gpu_probe(&out)) .unwrap_or_default() }; - let rt = Runtime { + let mut rt = Runtime { version_line: Some(line), version, gpu_backends, + sandbox: None, + sandbox_mode: sandbox_mode(), }; info!(blender = ?rt.version_line, gpu = ?rt.gpu_backends, "render: blender found"); + if rt.sandbox_mode != SandboxMode::Off { + rt.sandbox = probe_sandbox(&rt.gpu_backends); + } + // Said once: the probe runs once per process. + match (&rt.sandbox, rt.sandbox_mode) { + (Some(sb), _) => info!( + bwrap = %sb.bwrap.display(), + cycles_gpu = sb.cycles_gpu, + raster = sb.raster, + "render: blender runs inside bubblewrap (no network, only the job directory writable)" + ), + (None, SandboxMode::Off) => info!("render: [render] sandbox is off; renders run with scene sanitisation only"), + (None, SandboxMode::Auto) => info!("render: bubblewrap does not work on this machine; renders run with scene sanitisation only"), + (None, SandboxMode::Require) => warn!("render: [render] sandbox is \"require\" but bubblewrap does not work on this machine; not serving renders"), + } rt } @@ -803,6 +919,269 @@ pub fn parse_gpu_probe(text: &str) -> Vec { .collect() } +// ─── the sandbox ───────────────────────────────────────────────────────── + +/// One thing of the host the sandbox shows. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Mount { + /// A file or directory, read-only, at its own path. + RoBind(PathBuf), + /// A device node (or a directory of them), usable, at its own path. + DevBind(PathBuf), + /// A symlink the host has (`/bin` to `usr/bin`), made again. + Symlink { target: PathBuf, link: PathBuf }, +} + +/// How Blender is confined on this machine, found once by the probe: a +/// bubblewrap (`bwrap`) command line that was seen to start this Blender. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Sandbox { + pub bwrap: PathBuf, + /// The Blender binary with symlinks resolved; run by this path inside. + pub blender: PathBuf, + pub mounts: Vec, + /// The operator's home directory: an empty tmpfs inside. + pub home: Option, + /// The worker is root, so bubblewrap is told to drop every capability + /// (it keeps them for uid 0 otherwise). + pub as_root: bool, + /// Cycles found the same GPU backends inside the sandbox as outside. + pub cycles_gpu: bool, + /// A Workbench frame rendered inside the sandbox (EEVEE and Workbench + /// need a GPU context, which a missing display socket can take away). + pub raster: bool, +} + +/// The `bwrap` arguments that come before the Blender binary: no network, +/// new pid, ipc and hostname namespaces, an empty root with a private +/// `/tmp` and home, the host's `mounts`, and `job_dir` (absolute) as the +/// only writable part of the host. `blend`, the job's input inside it, is +/// read-only again: it can be a hard link to the node's cached copy. Pure, +/// so it is tested without bwrap. +pub fn bwrap_args(sandbox: &Sandbox, job_dir: &Path, blend: Option<&Path>) -> Vec { + let mut a: Vec = [ + "--unshare-net", + "--unshare-pid", + "--unshare-ipc", + "--unshare-uts", + "--hostname", + "c0mpute-render", + "--die-with-parent", + "--new-session", + ] + .into_iter() + .map(OsString::from) + .collect(); + if sandbox.as_root { + a.extend(["--cap-drop", "ALL"].map(OsString::from)); + } + a.extend(["--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp"].map(OsString::from)); + // The tmpfs comes before the binds: a Blender or a job directory under + // the home (or under /tmp) is then mounted on top of it. + if let Some(home) = &sandbox.home { + a.push("--tmpfs".into()); + a.push(home.into()); + } + for mount in &sandbox.mounts { + let (flag, from, to) = match mount { + Mount::RoBind(p) => ("--ro-bind", p, p), + Mount::DevBind(p) => ("--dev-bind", p, p), + Mount::Symlink { target, link } => ("--symlink", target, link), + }; + a.push(flag.into()); + a.push(from.into()); + a.push(to.into()); + } + a.push("--bind".into()); + a.push(job_dir.into()); + a.push(job_dir.into()); + if let Some(blend) = blend { + a.push("--ro-bind".into()); + a.push(blend.into()); + a.push(blend.into()); + } + a.push("--setenv".into()); + a.push("HOME".into()); + a.push(sandbox.home.clone().unwrap_or_else(|| PathBuf::from("/tmp")).into()); + a.push("--chdir".into()); + a.push(job_dir.into()); + a.push("--".into()); + a +} + +/// Is `path` under something already bound read-only? +#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))] +fn covered(mounts: &[Mount], path: &Path) -> bool { + mounts.iter().any(|m| matches!(m, Mount::RoBind(root) if path.starts_with(root))) +} + +/// May a directory outside the system directories be shown read-only? Not +/// the root, not a top-level directory, and nothing that holds the home. +#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))] +fn shareable(dir: &Path, home: Option<&Path>) -> bool { + dir.is_absolute() + && dir.components().filter(|c| matches!(c, std::path::Component::Normal(_))).count() >= 2 + && !home.is_some_and(|h| h.starts_with(dir)) +} + +/// Sets a 16x16 frame for the sandbox's Workbench test render. +#[cfg(target_os = "linux")] +const RASTER_PROBE_EXPR: &str = "import bpy\n\ +s = bpy.context.scene\n\ +s.render.resolution_x = 16\n\ +s.render.resolution_y = 16\n\ +s.render.resolution_percentage = 100\n"; + +/// What `/etc` holds that a dynamically linked, GPU-using Blender reads. +/// Never the whole directory: a root worker can read all of it. +#[cfg(target_os = "linux")] +const ETC_ENTRIES: &[&str] = &["ld.so.cache", "ld.so.conf", "ld.so.conf.d", "alternatives", "OpenCL", "vulkan", "glvnd", "egl"]; + +#[cfg(target_os = "linux")] +fn find_on_path(name: &str) -> Option { + std::env::split_paths(&std::env::var_os("PATH")?).map(|dir| dir.join(name)).find(|p| p.is_file()) +} + +/// Show `path` read-only where it really lives, keeping the symlink when it +/// is one (`/bin`, `/opt/rocm`). +#[cfg(target_os = "linux")] +fn add_ro(mounts: &mut Vec, path: &Path) { + let (Ok(meta), Ok(real)) = (std::fs::symlink_metadata(path), path.canonicalize()) else { + return; + }; + if meta.file_type().is_symlink() + && let Ok(target) = std::fs::read_link(path) + { + mounts.push(Mount::Symlink { target, link: path.to_path_buf() }); + } + if !covered(mounts, &real) { + mounts.push(Mount::RoBind(real)); + } +} + +/// What of this host Blender needs: the system directories, the loader and +/// GPU vendor files under `/etc`, `/sys` and the GPU device nodes, and the +/// directory Blender is installed in when that is not a system directory (a +/// blender.org tarball under `/opt` or the operator's home). `None` when +/// showing that directory would show the home itself. +#[cfg(target_os = "linux")] +fn host_mounts(blender: &Path, home: Option<&Path>) -> Option> { + let mut mounts = Vec::new(); + for dir in ["/usr", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/libx32", "/sys", "/opt/rocm"] { + add_ro(&mut mounts, Path::new(dir)); + } + for name in ETC_ENTRIES { + add_ro(&mut mounts, &Path::new("/etc").join(name)); + } + // Driver libraries the operator points the loader at (a CUDA or ROCm + // install outside the system directories). + if let Some(paths) = std::env::var_os("LD_LIBRARY_PATH") { + for dir in std::env::split_paths(&paths) { + if let Ok(real) = dir.canonicalize() + && real.is_dir() + && !covered(&mounts, &real) + && shareable(&real, home) + { + mounts.push(Mount::RoBind(real)); + } + } + } + let install = blender.parent()?; + if !covered(&mounts, install) { + if !shareable(install, home) { + return None; + } + mounts.push(Mount::RoBind(install.to_path_buf())); + } + for dev in ["/dev/dri", "/dev/kfd"] { + if Path::new(dev).exists() { + mounts.push(Mount::DevBind(dev.into())); + } + } + let mut nvidia: Vec = std::fs::read_dir("/dev") + .into_iter() + .flatten() + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with("nvidia")) + .map(|e| e.path()) + .collect(); + nvidia.sort(); + mounts.extend(nvidia.into_iter().map(Mount::DevBind)); + Some(mounts) +} + +/// Find out, once, whether bubblewrap can run this machine's Blender: build +/// the sandbox and start Blender inside it. A machine without `bwrap`, with +/// unprivileged user namespaces switched off (Ubuntu 24.04's AppArmor +/// default without a bwrap profile), or with a Blender the sandbox cannot +/// start (a snap, a wrapper script around an install elsewhere) gets `None`. +/// With a GPU, also checks that Cycles and Workbench still reach it inside. +#[cfg(target_os = "linux")] +fn probe_sandbox(gpu_backends: &[String]) -> Option { + let bwrap = find_on_path("bwrap")?; + let blender = find_on_path("blender")?.canonicalize().ok()?; + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .filter(|h| h.is_absolute() && h.parent().is_some()); + let mounts = host_mounts(&blender, home.as_deref())?; + let mut sandbox = Sandbox { + bwrap, + blender, + mounts, + home, + as_root: crate::container_run::is_root(), + cycles_gpu: false, + raster: false, + }; + let dir = std::env::temp_dir().join(format!("c0mpute-render-probe-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).ok()?; + let run = |sandbox: &Sandbox, args: &[&std::ffi::OsStr]| { + let mut cmd = std::process::Command::new(&sandbox.bwrap); + cmd.args(bwrap_args(sandbox, &dir, None)).arg(&sandbox.blender).args(args); + run_with_deadline(cmd, PROBE_TIMEOUT) + }; + let os = |s: &'static str| std::ffi::OsStr::new(s); + let works = run(&sandbox, &[os("--factory-startup"), os("--version")]) + .is_some_and(|out| parse_version(&out).is_some()); + if works && !gpu_backends.is_empty() { + sandbox.cycles_gpu = run( + &sandbox, + &[os("--background"), os("--factory-startup"), os("--disable-autoexec"), os("--python-expr"), os(GPU_PROBE_EXPR)], + ) + .is_some_and(|out| parse_gpu_probe(&out) == gpu_backends); + let frame = dir.join("probe_####"); + sandbox.raster = run( + &sandbox, + &[ + os("--background"), + os("--factory-startup"), + os("--disable-autoexec"), + os("--python-expr"), + os(RASTER_PROBE_EXPR), + os("--engine"), + os("BLENDER_WORKBENCH"), + os("--render-output"), + frame.as_os_str(), + os("--render-format"), + os("PNG"), + os("--render-frame"), + os("1"), + ], + ) + .is_some() + && dir.join("probe_0001.png").is_file(); + } + let _ = std::fs::remove_dir_all(&dir); + works.then_some(sandbox) +} + +/// Bubblewrap is Linux only: everywhere else renders run with scene +/// sanitisation only. +#[cfg(not(target_os = "linux"))] +fn probe_sandbox(_gpu_backends: &[String]) -> Option { + None +} + /// Bid filter: skip offers this box cannot render (a GPU engine or device /// without a GPU, or a spec it cannot read). pub fn offer_filter() -> crate::dispatch::OfferFilter { @@ -863,6 +1242,176 @@ pub fn settings_expr(parsed: &Parsed, cycles_device: Option<&str>) -> String { lines.join("\n") } +/// The job directory and its output directory, handed to [`SANITIZE_EXPR`] +/// in the environment so that no path is ever written into the script. +pub const JOB_DIR_ENV: &str = "C0MPUTE_RENDER_JOB_DIR"; +pub const OUT_DIR_ENV: &str = "C0MPUTE_RENDER_OUT_DIR"; +/// Blender's exit code when [`SANITIZE_EXPR`] could not finish: the scene is +/// not rendered at all rather than rendered as the buyer saved it. +pub const SANITIZE_EXIT: i32 = 97; + +/// Python run after the file loads and before the render, on every OS: it +/// leaves the scene able to read and write inside the job directory only. +/// A constant: nothing from the job is formatted into it (the two +/// directories come from the environment), so a buyer cannot reach it. +/// +/// A `.blend` names files by path: image textures, movie clips, sounds, +/// fonts, linked libraries, caches, sequencer strips. Blender reads them +/// from the worker's disk when it renders, so a scene that names +/// `/home//Pictures/x.png` would return that picture inside a frame. +/// Every such path that is absolute or leaves the job directory is emptied +/// (the scene then renders with Blender's pink "missing" texture, the +/// built-in font, or without the strip or cache); data packed into the +/// `.blend` is untouched. Linked libraries are loaded by Blender while the +/// file opens, before this runs, so what came from them is deleted instead. +/// Compositor File Output nodes are removed, Open Shading Language is turned +/// off (a compiled shader may open any image by name), and every scene's +/// output path is set to the node's own output directory. +/// +/// Written for Blender 3.x to 5.x: attributes a version lacks are skipped +/// (`getattr`), and any other failure exits with [`SANITIZE_EXIT`]. +pub const SANITIZE_EXPR: &str = r#"import bpy, os, sys, traceback + +def _c0_sanitize(): + D = bpy.data + job_dir = os.environ.get('C0MPUTE_RENDER_JOB_DIR') + out_dir = os.environ.get('C0MPUTE_RENDER_OUT_DIR') + if not job_dir or not out_dir: + raise RuntimeError('the job and output directories are not set') + root = os.path.normcase(os.path.realpath(job_dir)) + cut = [0] + + def inside(path, lib=None): + try: + full = os.path.normcase(os.path.realpath(bpy.path.abspath(path, library=lib))) + return os.path.commonpath([root, full]) == root + except Exception: + return False + + def drop(owner, attr, lib=None, to=''): + path = getattr(owner, attr, None) + if not isinstance(path, str) or path in ('', to) or inside(path, lib): + return False + setattr(owner, attr, to) + cut[0] += 1 + return True + + def outside_libs(): + return [lib for lib in D.libraries if not inside(lib.filepath)] + + bad = outside_libs() + if bad: + def from_bad(i): + if i.library in bad: + return True + ref = getattr(getattr(i, 'override_library', None), 'reference', None) + return ref is not None and ref.library in bad + D.batch_remove([i for i in D.user_map() if from_bad(i)]) + cut[0] += len(bad) + for lib in outside_libs(): + D.libraries.remove(lib) + if outside_libs(): + raise RuntimeError('a linked library could not be removed') + + for im in D.images: + packed = im.packed_file is not None or len(getattr(im, 'packed_files', ())) > 0 + if packed and im.source in ('FILE', 'TILED'): + continue + drop(im, 'filepath', im.library) + for name, keep_packed in (('movieclips', False), ('sounds', True), ('volumes', True), ('cache_files', False)): + for item in getattr(D, name, ()): + if keep_packed and getattr(item, 'packed_file', None) is not None: + continue + drop(item, 'filepath', item.library) + for font in D.fonts: + if font.packed_file is None: + drop(font, 'filepath', font.library, '') + + for ob in D.objects: + for mod in ob.modifiers: + domain = getattr(mod, 'domain_settings', None) + if domain is not None: + drop(domain, 'cache_directory', ob.library, os.path.join(job_dir, 'fluid_cache')) + drop(mod, 'filepath', ob.library) + drop(mod, 'bake_directory', ob.library) + for bake in getattr(mod, 'bakes', ()): + drop(bake, 'directory', ob.library) + caches = [getattr(mod, 'point_cache', None)] + for surface in getattr(getattr(mod, 'canvas_settings', None), 'canvas_surfaces', ()): + caches.append(getattr(surface, 'point_cache', None)) + for cache in caches: + if cache is not None: + drop(cache, 'filepath', ob.library) + for psys in ob.particle_systems: + cache = getattr(psys, 'point_cache', None) + if cache is not None: + drop(cache, 'filepath', ob.library) + + trees = list(D.node_groups) + for name in ('materials', 'worlds', 'lights', 'textures', 'linestyles', 'scenes'): + for item in getattr(D, name, ()): + tree = getattr(item, 'node_tree', None) + if tree is not None: + trees.append(tree) + for tree in trees: + for node in list(tree.nodes): + if node.type == 'OUTPUT_FILE': + try: + tree.nodes.remove(node) + except Exception: + node.mute = True + for attr in ('base_path', 'directory'): + if hasattr(node, attr): + setattr(node, attr, out_dir) + cut[0] += 1 + else: + drop(node, 'filepath', tree.library) + + for sc in D.scenes: + r = sc.render + r.filepath = os.path.join(out_dir, 'frame_####') + r.use_file_extension = True + for attr in ('use_render_cache', 'use_stamp_hostname', 'use_stamp_filename'): + if hasattr(r, attr): + setattr(r, attr, False) + cycles = getattr(sc, 'cycles', None) + if cycles is not None and hasattr(cycles, 'shading_system'): + cycles.shading_system = False + world = getattr(sc, 'rigidbody_world', None) + if world is not None and getattr(world, 'point_cache', None) is not None: + drop(world.point_cache, 'filepath', sc.library) + ed = sc.sequence_editor + if ed is None: + continue + if drop(ed, 'proxy_dir', sc.library) and hasattr(ed, 'proxy_storage'): + ed.proxy_storage = 'PER_STRIP' + strips = getattr(ed, 'strips_all', None) + if strips is None: + strips = getattr(ed, 'sequences_all', ()) + for strip in strips: + hit = drop(strip, 'filepath', sc.library) + hit = drop(strip, 'directory', sc.library) or hit + if hit: + strip.mute = True + proxy = getattr(strip, 'proxy', None) + if proxy is not None: + hit = drop(proxy, 'filepath', sc.library) + hit = drop(proxy, 'directory', sc.library) or hit + if hit and hasattr(strip, 'use_proxy'): + strip.use_proxy = False + + left = [p for p in bpy.utils.blend_paths(absolute=True, packed=False, local=False) if not inside(p)] + print('c0mpute: scene sanitised: %d outside references cut, %d path fields a render does not read left as they were' % (cut[0], len(left))) + +try: + _c0_sanitize() +except BaseException: + traceback.print_exc() + sys.stdout.flush() + sys.stderr.flush() + os._exit(97) +"#; + /// The full `blender` argument list for one job. `out_dir` must not contain /// `#` (Blender's frame-number placeholder). The `.blend` path is one /// argument (no shell is involved), and `--disable-autoexec` comes before it @@ -885,6 +1434,8 @@ pub fn blender_args( args.push(blend.as_os_str().to_owned()); args.push("--python-expr".into()); args.push(settings_expr(parsed, cycles_device).into()); + args.push("--python-expr".into()); + args.push(SANITIZE_EXPR.into()); args.push("--engine".into()); args.push(parsed.engine.blender_id(version).into()); args.push("--render-output".into()); @@ -931,9 +1482,12 @@ pub struct Rendered { /// output. Needs a ready [`runtime`]. pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result> { let rt = runtime(); - if !rt.ready() { + if rt.version_line.is_none() { bail!("blender is not installed on this node"); } + if !rt.ready() { + bail!("this node renders only inside a sandbox, and bubblewrap does not work here"); + } if !rt.can_serve(parsed) { bail!( "this node cannot render {} on {}", @@ -941,11 +1495,29 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result tokio::fs::canonicalize(job_dir).await?, + None => std::path::absolute(job_dir)?, + }; let out_dir = job_dir.join("out"); - tokio::fs::create_dir_all(&out_dir).await?; + let blend = &stage_input(blend, job_dir).await?; let cycles_device = rt.cycles_device(parsed); - let mut command = Command::new("blender"); + let mut command = match sandbox { + Some(sb) => { + let mut command = Command::new(&sb.bwrap); + command.args(bwrap_args(sb, job_dir, Some(blend.as_path()))).arg(&sb.blender); + command + } + None => Command::new("blender"), + }; + // Environment only, so the limits reach Blender through bubblewrap too. let threads = crate::resources::apply_job_limits(&mut command); + command.env(JOB_DIR_ENV, job_dir).env(OUT_DIR_ENV, &out_dir); command.args(blender_args( blend, &out_dir, @@ -972,6 +1544,10 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result = HashMap::new(); let mut entries = tokio::fs::read_dir(&out_dir).await?; @@ -1011,6 +1587,27 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result/input.blend`: a hard link +/// to the cached copy (a copy where the filesystem cannot link). Blender +/// resolves `//relative` paths against the file's directory, and the cache +/// directory holds other buyers' scenes; the job directory holds only this +/// job's. +async fn stage_input(blend: &Path, job_dir: &Path) -> Result { + // Already the job's own (an opened sealed input), however it is spelled. + let real = tokio::fs::canonicalize(blend).await.context("find the .blend")?; + if real.parent() == tokio::fs::canonicalize(job_dir).await.ok().as_deref() + && let Some(name) = blend.file_name() + { + return Ok(job_dir.join(name)); + } + let staged = job_dir.join("input.blend"); + let _ = tokio::fs::remove_file(&staged).await; + if tokio::fs::hard_link(blend, &staged).await.is_err() { + tokio::fs::copy(blend, &staged).await.context("copy the .blend into the job directory")?; + } + Ok(staged) +} + async fn log_tail(path: &Path) -> String { let text = tokio::fs::read_to_string(path).await.unwrap_or_default(); let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); @@ -1934,6 +2531,8 @@ mod tests { version_line: Some("Blender 3.4.1".into()), version: (3, 4, 1), gpu_backends: vec![], + sandbox: None, + sandbox_mode: SandboxMode::Auto, }; let gpu = Runtime { gpu_backends: vec!["CUDA".into()], @@ -1957,6 +2556,213 @@ mod tests { assert!(Runtime::default().tags().is_empty()); } + #[test] + fn sandbox_setting_parses_and_a_typo_means_auto() { + use crate::config::Config; + // A config written before the section existed, and the default. + let cfg: Config = toml::from_str("roles = []\n").unwrap(); + assert_eq!(cfg.render.mode_with(None), SandboxMode::Auto); + assert_eq!(Config::default().render.sandbox, "auto"); + let mode = |text: &str| toml::from_str::(text).unwrap().render.mode_with(None); + assert_eq!(mode("[render]\nsandbox = \"off\"\n"), SandboxMode::Off); + assert_eq!(mode("[render]\nsandbox = \"require\"\n"), SandboxMode::Require); + assert_eq!(mode("[render]\nsandbox = \" Require \"\n"), SandboxMode::Require); + // A value that is none of the three never stops the node: it is auto. + assert_eq!(mode("[render]\nsandbox = \"always\"\n"), SandboxMode::Auto); + // The environment wins when it is set to something. + let cfg: Config = toml::from_str("[render]\nsandbox = \"require\"\n").unwrap(); + assert_eq!(cfg.render.mode_with(Some("off".into())), SandboxMode::Off); + assert_eq!(cfg.render.mode_with(Some(" ".into())), SandboxMode::Require); + } + + fn sandbox(cycles_gpu: bool, raster: bool) -> Sandbox { + Sandbox { + bwrap: "/usr/bin/bwrap".into(), + blender: "/home/op/blender-4.2/blender".into(), + mounts: vec![ + Mount::RoBind("/usr".into()), + Mount::Symlink { target: "usr/bin".into(), link: "/bin".into() }, + Mount::RoBind("/etc/ld.so.cache".into()), + Mount::RoBind("/sys".into()), + Mount::RoBind("/home/op/blender-4.2".into()), + Mount::DevBind("/dev/dri".into()), + Mount::DevBind("/dev/nvidia0".into()), + ], + home: Some("/home/op".into()), + as_root: false, + cycles_gpu, + raster, + } + } + + #[test] + fn bwrap_argv_shows_the_system_read_only_and_only_the_job_writable() { + let job = Path::new("/home/op/.local/share/c0mpute/cache/jobs/j1"); + let blend = job.join("input.blend"); + let a: Vec = bwrap_args(&sandbox(true, true), job, Some(&blend)).iter().map(|x| x.to_string_lossy().into_owned()).collect(); + let at = |w: &[&str]| a.windows(w.len()).position(|x| x == w).unwrap_or_else(|| panic!("{w:?} not in {a:?}")); + for flag in ["--unshare-net", "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--die-with-parent", "--new-session"] { + assert!(a.iter().any(|x| x == flag), "{flag}"); + } + at(&["--proc", "/proc"]); + at(&["--ro-bind", "/usr", "/usr"]); + at(&["--symlink", "usr/bin", "/bin"]); + at(&["--ro-bind", "/sys", "/sys"]); + at(&["--dev-bind", "/dev/nvidia0", "/dev/nvidia0"]); + // /dev is made fresh before the GPU nodes are put back in it. + assert!(at(&["--dev", "/dev"]) < at(&["--dev-bind", "/dev/dri", "/dev/dri"])); + // The home is an empty tmpfs; Blender's own directory and the job + // directory, both under it, are mounted on top afterwards. + let home = at(&["--tmpfs", "/home/op"]); + assert!(at(&["--tmpfs", "/tmp"]) < home); + assert!(home < at(&["--ro-bind", "/home/op/blender-4.2", "/home/op/blender-4.2"])); + let job_s = job.to_str().unwrap(); + assert!(home < at(&["--bind", job_s, job_s])); + // The input (a hard link to the cached copy) is read-only inside it. + let blend_s = blend.to_str().unwrap(); + assert!(at(&["--bind", job_s, job_s]) < at(&["--ro-bind", blend_s, blend_s])); + at(&["--setenv", "HOME", "/home/op"]); + at(&["--chdir", job_s]); + assert_eq!(a.last().unwrap(), "--"); + // Exactly one writable bind of the host, and it is the job directory. + assert_eq!(a.iter().filter(|x| *x == "--bind").count(), 1); + assert!(!a.iter().any(|x| x == "--share-net" || x == "--dev-bind-try" || x == "/home" || x == "/root" || x == "/etc")); + // An unprivileged worker gets no capabilities from bwrap; root must drop them. + assert!(!a.iter().any(|x| x == "--cap-drop")); + let root = Sandbox { as_root: true, home: None, ..sandbox(false, false) }; + let r: Vec = bwrap_args(&root, Path::new("/var/lib/c0mpute/jobs/j1"), None).iter().map(|x| x.to_string_lossy().into_owned()).collect(); + assert!(r.windows(2).any(|w| w == ["--cap-drop", "ALL"])); + assert!(r.windows(3).any(|w| w == ["--setenv", "HOME", "/tmp"]), "no home: HOME is the private /tmp"); + } + + #[test] + fn only_install_directories_that_do_not_hold_the_home_are_shown() { + let home = Some(Path::new("/home/op")); + assert!(shareable(Path::new("/opt/blender-4.2"), home)); + assert!(shareable(Path::new("/home/op/blender-4.2"), home)); + assert!(!shareable(Path::new("/home/op"), home), "the home itself"); + assert!(!shareable(Path::new("/home"), home)); + assert!(!shareable(Path::new("/opt"), home), "a top-level directory"); + assert!(!shareable(Path::new("/"), None)); + assert!(!shareable(Path::new("blender"), None)); + let mounts = vec![Mount::RoBind("/usr".into()), Mount::DevBind("/dev/dri".into())]; + assert!(covered(&mounts, Path::new("/usr/bin"))); + assert!(!covered(&mounts, Path::new("/usrlocal"))); + assert!(!covered(&mounts, Path::new("/dev/dri/card0")), "a device bind is not a read-only view"); + } + + #[test] + fn require_serves_only_what_the_sandbox_was_seen_to_do() { + let p = |extra: Value| parse_spec(&spec(extra)).unwrap(); + let base = Runtime { + version_line: Some("Blender 4.2.3 LTS".into()), + version: (4, 2, 3), + gpu_backends: vec!["OPTIX".into()], + sandbox: None, + sandbox_mode: SandboxMode::Require, + }; + // No working sandbox: no role, no tags, no bids. + assert!(!base.ready()); + assert!(base.tags().is_empty()); + assert!(!base.can_serve(&p(json!({"device":"CPU"})))); + // A sandbox that starts Blender but cannot reach the GPU: CPU Cycles only. + let cpu_only = Runtime { sandbox: Some(sandbox(false, false)), ..base.clone() }; + assert!(cpu_only.ready()); + assert!(cpu_only.can_serve(&p(json!({})))); + assert_eq!(cpu_only.cycles_device(&p(json!({}))), None, "AUTO renders on the CPU inside the sandbox"); + assert!(!cpu_only.can_serve(&p(json!({"device":"GPU"})))); + assert!(!cpu_only.can_serve(&p(json!({"engine":"EEVEE"})))); + assert_eq!(cpu_only.tags(), vec![ROLE_TAG, "c0mpute:render:engine:cycles"]); + assert!(cpu_only.sandbox_for(&p(json!({}))).is_some()); + // A sandbox that reaches the GPU serves everything, inside it. + let full = Runtime { sandbox: Some(sandbox(true, true)), ..base.clone() }; + assert!(full.can_serve(&p(json!({"device":"GPU"}))) && full.can_serve(&p(json!({"engine":"EEVEE"})))); + assert_eq!(full.cycles_device(&p(json!({}))).as_deref(), Some("OPTIX")); + assert!(full.sandbox_for(&p(json!({"engine":"EEVEE"}))).is_some()); + assert!(full.tags().contains(&GPU_TAG.to_string())); + + // "auto" keeps today's behaviour: every job runs, and a GPU job the + // sandbox cannot serve runs outside it. + let auto = Runtime { sandbox_mode: SandboxMode::Auto, ..cpu_only.clone() }; + assert!(auto.can_serve(&p(json!({"device":"GPU"}))) && auto.can_serve(&p(json!({"engine":"EEVEE"})))); + assert_eq!(auto.cycles_device(&p(json!({}))).as_deref(), Some("OPTIX")); + assert!(auto.sandbox_for(&p(json!({}))).is_none(), "GPU Cycles, sandbox without the GPU"); + assert!(auto.sandbox_for(&p(json!({"engine":"EEVEE"}))).is_none()); + assert!(auto.sandbox_for(&p(json!({"device":"CPU"}))).is_some()); + let none = Runtime { sandbox: None, ..auto.clone() }; + assert!(none.ready() && none.sandbox_for(&p(json!({"device":"CPU"}))).is_none()); + } + + #[test] + fn no_buyer_text_reaches_the_python_blender_runs() { + // Everything a buyer chooses, at its most hostile: the file's path + // (its hash names it in practice) and every spec field. + let p = parse_spec(&spec(json!({ + "frame_start": MAX_FRAME - 1, "frame_end": MAX_FRAME, "resolution_pct": MAX_RESOLUTION_PCT, + "samples": MAX_SAMPLES, "format": "jpg", "engine": "cycles", "device": "gpu", "frame_timeout_secs": 10, + }))) + .unwrap(); + let blend = Path::new("/cache/'); import os; os.system('id') #.blend"); + let out = Path::new("/jobs/\"]); __import__('os')/out"); + let args: Vec = blender_args(blend, out, &p, (4, 2, 0), Some("CUDA"), 4) + .unwrap() + .iter() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + let exprs: Vec<&String> = args.iter().zip(args.iter().skip(1)).filter(|(flag, _)| *flag == "--python-expr").map(|(_, e)| e).collect(); + assert_eq!(exprs.len(), 2, "the settings expression and the sanitiser"); + // The sanitiser is the constant, byte for byte, whatever the job is. + assert_eq!(exprs[1], SANITIZE_EXPR); + let other = parse_spec(&spec(json!({"engine": "EEVEE", "samples": 1}))).unwrap(); + let other_args = blender_args(Path::new("/x.blend"), Path::new("/o"), &other, (3, 4, 1), None, 1).unwrap(); + assert!(other_args.iter().any(|a| a.to_str() == Some(SANITIZE_EXPR))); + // The settings expression holds the validated numbers and fixed words only. + for needle in ["os.system", "__import__", "/cache", "/jobs", ".blend", "CUDA"] { + assert!(!exprs[0].contains(needle), "{needle} in {}", exprs[0]); + } + assert!(exprs[0].contains(&format!("s.cycles.samples = {MAX_SAMPLES}"))); + // Both run before the render and after the file is loaded. + let at = |needle: &str| args.iter().position(|a| a == needle).unwrap(); + assert!(at(SANITIZE_EXPR) > at(blend.to_str().unwrap()) && at(SANITIZE_EXPR) < at("--render-anim")); + // The directories reach the script through the environment, and a + // script that cannot finish stops Blender instead of rendering. + assert!(SANITIZE_EXPR.contains(&format!("os.environ.get('{JOB_DIR_ENV}')"))); + assert!(SANITIZE_EXPR.contains(&format!("os.environ.get('{OUT_DIR_ENV}')"))); + assert!(SANITIZE_EXPR.contains(&format!("os._exit({SANITIZE_EXIT})"))); + // Python needs its indentation (a `\` line continuation in a Rust + // string would strip it), and the script has no format placeholders. + assert!(SANITIZE_EXPR.contains("\n D = bpy.data\n") && SANITIZE_EXPR.contains("\n try:\n")); + assert!(!SANITIZE_EXPR.contains("{}") && !SANITIZE_EXPR.contains("{0}")); + for covered in ["D.libraries", "D.images", "'movieclips'", "'sounds'", "'volumes'", "'cache_files'", "D.fonts", "OUTPUT_FILE", "sequence_editor", "shading_system", "blend_paths"] { + assert!(SANITIZE_EXPR.contains(covered), "{covered}"); + } + } + + #[tokio::test] + async fn the_blend_is_staged_into_the_job_directory() { + let dir = tempfile::tempdir().unwrap(); + let cache = dir.path().join("render-inputs"); + let job = dir.path().join("jobs").join("j1"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::create_dir_all(&job).unwrap(); + std::fs::write(cache.join("abc.blend"), b"BLENDER").unwrap(); + std::fs::write(cache.join("other-buyer.blend"), b"BLENDER2").unwrap(); + let staged = stage_input(&cache.join("abc.blend"), &job).await.unwrap(); + assert_eq!(staged, job.join("input.blend")); + assert_eq!(std::fs::read(&staged).unwrap(), b"BLENDER"); + // Its directory holds this job's file only, not the other scenes. + assert_eq!(std::fs::read_dir(&job).unwrap().count(), 1); + // The cached copy outlives the job directory. + std::fs::remove_dir_all(&job).unwrap(); + assert!(cache.join("abc.blend").exists()); + // An input already in the job directory (an opened sealed .blend) stays where it is. + std::fs::create_dir_all(&job).unwrap(); + std::fs::write(job.join("input.blend"), b"PLAIN").unwrap(); + let same = stage_input(&job.join("input.blend"), &job).await.unwrap(); + assert_eq!(same, job.join("input.blend")); + assert_eq!(std::fs::read(&same).unwrap(), b"PLAIN"); + } + /// Runs only where `blender` is installed; elsewhere it says so and /// passes. Builds a tiny scene with Blender itself and renders two frames. #[tokio::test] @@ -2000,6 +2806,58 @@ mod tests { ); } + /// Runs only where `blender` is installed. A scene that names an image + /// outside the job directory by absolute path still renders, with that + /// reference cut before the render starts. + #[tokio::test] + async fn a_scene_naming_a_host_file_renders_without_it() { + let rt = tokio::task::spawn_blocking(runtime).await.unwrap(); + if !rt.ready() { + eprintln!("skipping: blender is not installed"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + let (blend, secret) = (dir.path().join("scene.blend"), outside.join("secret.png")); + // Saved once before the image is added: Blender turns absolute paths + // into relative ones on a file's first save, and this scene must + // keep naming the image by its absolute path. + let expr = format!( + "import bpy\ns = bpy.context.scene\ns.render.engine = 'CYCLES'\n\ + s.render.resolution_x = 64\ns.render.resolution_y = 48\n\ + s.render.resolution_percentage = 100\ns.cycles.samples = 2\n\ + bpy.ops.wm.save_as_mainfile(filepath={blend:?})\n\ + img = bpy.data.images.new('secret', 8, 8)\nimg.filepath_raw = {secret:?}\n\ + img.file_format = 'PNG'\nimg.save()\nbpy.data.images.remove(img)\n\ + ext = bpy.data.images.load({secret:?})\next.filepath = {secret:?}\n\ + ext.use_fake_user = True\ns.render.filepath = {out:?}\n\ + bpy.ops.wm.save_mainfile()\n", + secret = secret.to_string_lossy(), + out = outside.join("render_").to_string_lossy(), + blend = blend.to_string_lossy(), + ); + let status = std::process::Command::new("blender") + .args(["--background", "--factory-startup", "--python-expr", &expr]) + .stdout(Stdio::null()) + .status() + .unwrap(); + assert!(status.success() && blend.exists() && secret.exists(), "could not build the test scene"); + let size = std::fs::metadata(&blend).unwrap().len(); + let p = parse_spec(&spec(json!({ + "input_size": size, "input_pieces": vec![hash(); piece_count(size)], + "frame_start": 1, "frame_end": 1, "samples": 2, "device": "CPU", + }))) + .unwrap(); + let job = dir.path().join("job"); + let frames = render(&blend, &p, &job).await.unwrap(); + assert_eq!(frames.len(), 1); + let log = std::fs::read_to_string(job.join("blender.log")).unwrap(); + assert!(log.contains("scene sanitised: 1 outside references cut"), "{log}"); + // Nothing was written beside the file the scene pointed its output at. + assert_eq!(std::fs::read_dir(&outside).unwrap().count(), 1); + } + /// Save the default scene (cube, camera, light) at 64x48 to `path`. pub(crate) fn make_test_scene(path: &Path) { let expr = format!( diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index c79bec10..a4719180 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -915,6 +915,11 @@ impl Supervisor { dir: Some(self.config.storage.root.join("apps")), hosting, registry: Some(self.registry.clone()), + // From the config this node was started with, not the + // default path (a node run with --config elsewhere). + egress_guard: Some(cfg.egress_guard), + require_egress_guard: Some(cfg.require_egress_guard), + max_disk_mb: Some(cfg.max_disk_mb), ..Default::default() }, ) diff --git a/node/crates/c0mpute-core/src/text.rs b/node/crates/c0mpute-core/src/text.rs index 7f2b103a..97b6c1b5 100644 --- a/node/crates/c0mpute-core/src/text.rs +++ b/node/crates/c0mpute-core/src/text.rs @@ -78,6 +78,159 @@ pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30); /// the worker up (three missed beats plus gossip slack) and re-auctions. pub const HEARTBEAT_SILENCE: Duration = Duration::from_secs(120); +/// The base models a worker serves when its operator lists none: the ones +/// this codebase's own clients name (`c0mpute train lora --base`'s example +/// and the job file on the /train page). A base model is downloaded from +/// Hugging Face and loaded by the trainer, so a worker runs only the ones +/// its operator has agreed to. +pub const DEFAULT_BASE_MODELS: [&str; 2] = ["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"]; + +/// `[text]` in config.toml: what `c0mpute infernet text serve` accepts. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct TextConfig { + /// Base models buyers may train on and chat with here: exact Hugging + /// Face ids (`org/name`, case as on the Hub), `org/*` for every model of + /// one organisation, or `*` for any. Jobs naming another get no bid. + pub base_models: Vec, + /// Free space kept on the disk the text cache is on, GiB; a job that + /// starts below it fails at once (0 = min(20 GiB, 10% of the disk), the + /// floor container jobs keep). + pub min_free_gb: u64, +} + +impl Default for TextConfig { + fn default() -> Self { + Self { base_models: DEFAULT_BASE_MODELS.iter().map(|m| m.to_string()).collect(), min_free_gb: 0 } + } +} + +impl TextConfig { + /// The node's `[text]` section. `text serve` is started without a config + /// handle, so this reads the file itself, as the speech gateway does. + pub fn load() -> Self { + crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default().text + } + + /// May a buyer's job use `base_model` on this node? + pub fn admits_model(&self, base_model: &str) -> bool { + model_allowed(&self.base_models, base_model) + } + + /// Fail unless the disk holding `dir` has its free-space floor left. + pub fn check_free(&self, dir: &Path) -> Result<()> { + let (free, total) = c0mpute_store::free_space_at(dir); + let floor = crate::container_run::min_free_bytes( + &crate::container_run::ContainersConfig { min_free_gb: self.min_free_gb, ..Default::default() }, + total, + ); + if free < floor { + bail!("E_DISK: {} has {} GiB free; this node keeps {} GiB", dir.display(), free >> 30, floor >> 30); + } + Ok(()) + } +} + +/// A Hugging Face repo id, strictly: `org/name`, each part 1 to 96 of +/// `A-Z a-z 0-9 . _ -`, not starting or ending with `.` or `-`, no `..` or +/// `--`. Anything else could be read by the trainer as a path on the worker +/// (`models/x`, `./x`, a bare directory name) or as a URL. +pub fn valid_repo_id(id: &str) -> bool { + let part = |p: &str| { + !p.is_empty() + && p.len() <= 96 + && p.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) + && !p.starts_with(['.', '-']) + && !p.ends_with(['.', '-']) + && !p.contains("..") + && !p.contains("--") + }; + matches!(id.split_once('/'), Some((org, name)) if part(org) && part(name)) +} + +/// Is `id` a valid repo id that `allow` names: exactly, by `org/*`, or by +/// `*`? Case matters: the Hub answers a differently-cased id with the same +/// model, and each spelling would be a second copy in the worker's cache. +pub fn model_allowed(allow: &[String], id: &str) -> bool { + let Some((org, _)) = id.split_once('/').filter(|_| valid_repo_id(id)) else { + return false; + }; + allow.iter().map(|e| e.trim()).any(|entry| entry == "*" || entry == id || entry.strip_suffix("/*") == Some(org)) +} + +/// Top-level keys a job config may carry: what `c0mpute train lora` +/// ([`crate::train::LoraOptions::job`]) and a hand-written job file for +/// `c0mpute infernet text train` send, plus `limits.timeout_s`, which the +/// worker reads. The trainer is handed the job as its arguments, so a key it +/// does not know is refused here, not passed on. +pub const JOB_KEYS: [&str; 8] = ["name", "kind", "base_model", "system_prompt", "hyperparams", "limits", "dataset", "output"]; +/// Hyperparameters a job may set, with the ranges `c0mpute train lora` uses. +pub const HYPERPARAM_KEYS: [&str; 7] = ["lr", "batch_size", "epochs", "seq_len", "lora_r", "max_steps", "seed"]; +const MAX_SYSTEM_PROMPT_BYTES: usize = 8 * 1024; + +fn whole_in(v: &Value, range: std::ops::RangeInclusive) -> bool { + v.as_u64().is_some_and(|n| range.contains(&n)) +} + +/// Check a job config before it reaches the trainer: only [`JOB_KEYS`], only +/// [`HYPERPARAM_KEYS`], every value of the right type and in range. +pub fn validate_job(job: &Value) -> Result<()> { + let obj = job.as_object().context("job must be an object")?; + if let Some(k) = obj.keys().find(|k| !JOB_KEYS.contains(&k.as_str())) { + bail!("job.{k} is not accepted; a text training job may set {JOB_KEYS:?}"); + } + let kind = obj.get("kind").and_then(Value::as_str).unwrap_or(""); + if !KINDS.contains(&kind) { + bail!("job.kind must be one of {KINDS:?}, got {kind:?}"); + } + // A Hugging Face repo id (org/name): never a path or a URL on the worker. + if !valid_repo_id(obj.get("base_model").and_then(Value::as_str).unwrap_or("")) { + bail!("job.base_model must be a model repo id like org/name"); + } + match obj.get("name") { + None => {} + Some(Value::String(n)) if !n.is_empty() && n.len() <= 64 && n.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) => {} + Some(_) => bail!("job.name must be 1-64 characters of letters, digits, '-', '_' or '.'"), + } + match obj.get("system_prompt") { + None | Some(Value::Null) => {} + Some(Value::String(p)) if p.len() <= MAX_SYSTEM_PROMPT_BYTES => {} + Some(_) => bail!("job.system_prompt must be a string of at most {MAX_SYSTEM_PROMPT_BYTES} bytes"), + } + // The worker replaces both with its own local paths; they only have to + // be what the clients send, an object. + for k in ["dataset", "output"] { + if obj.get(k).is_some_and(|v| !v.is_object()) { + bail!("job.{k} must be an object"); + } + } + if let Some(limits) = obj.get("limits") { + let l = limits.as_object().context("job.limits must be an object")?; + if l.keys().any(|k| k != "timeout_s") || l.get("timeout_s").is_some_and(|t| !t.is_u64()) { + bail!("job.limits may set only timeout_s, a whole number of seconds"); + } + } + if let Some(hp) = obj.get("hyperparams") { + let hp = hp.as_object().context("job.hyperparams must be an object")?; + for (k, v) in hp { + let ok = match k.as_str() { + "lr" => v.as_f64().is_some_and(|x| x.is_finite() && x > 0.0 && x <= 1e-2), + "batch_size" => whole_in(v, 1..=64), + "epochs" => whole_in(v, 1..=20), + "seq_len" => whole_in(v, 32..=4096), + "lora_r" => whole_in(v, 1..=128), + "max_steps" => whole_in(v, 1..=1_000_000), + "seed" => whole_in(v, 0..=u32::MAX as u64), + _ => bail!("job.hyperparams.{k} is not accepted; a text training job may set {HYPERPARAM_KEYS:?}"), + }; + if !ok { + bail!("job.hyperparams.{k} is out of range: lr in (0, 0.01], batch_size 1-64, epochs 1-20, seq_len 32-4096, lora_r 1-128, max_steps 1-1000000, seed 0-4294967295, whole numbers except lr"); + } + } + } + Ok(()) +} + /// Jobs a text worker is running now, by job id, with each one's run id, so a /// shutdown can fail them out loud instead of leaving the buyer waiting. pub type InFlight = Arc)>>>; @@ -177,15 +330,7 @@ fn is_http(u: &str) -> bool { impl TextTrainSpec { pub fn validate(&self) -> Result<()> { - let kind = self.job.get("kind").and_then(|v| v.as_str()).unwrap_or(""); - if !KINDS.contains(&kind) { - bail!("job.kind must be one of {KINDS:?}, got {kind:?}"); - } - let base = self.job.get("base_model").and_then(|v| v.as_str()).unwrap_or(""); - // A Hugging Face repo id (org/name): never a path or a URL on the worker. - if base.is_empty() || base.len() > 200 || base.starts_with('/') || base.contains("..") || base.contains("://") { - bail!("job.base_model must be a model repo id like org/name"); - } + validate_job(&self.job)?; if !crate::temporal::is_sha256_ref(&self.dataset_hash) { bail!("dataset_hash must be sha256:<64 lowercase hex>"); } @@ -323,11 +468,20 @@ pub async fn fetch_corpus(net: &Arc, buyer: Option, spec: /// Run one accepted job: fetch the corpus, run `infernet model train --job /// --json`, relay every event, return the adapter dir and its weights hash. /// `buyer` is the peer the corpus pieces are fetched from first. -pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec, net: &Arc, buyer: Option) -> Result<(PathBuf, String)> { +pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec, net: &Arc, buyer: Option, policy: &TextConfig) -> Result<(PathBuf, String)> { spec.validate()?; + // The opened spec is judged again: at bid time a sealed job showed only + // its public fields. + let base = spec.job["base_model"].as_str().unwrap_or(""); + if !policy.admits_model(base) { + bail!("base model {base} is not one this node runs ([text] base_models)"); + } tokio::fs::create_dir_all(job_dir).await?; + policy.check_free(job_dir)?; let dataset = job_dir.join("dataset.jsonl"); fetch_corpus(net, buyer, spec, &dataset).await.context("fetch dataset")?; + // The corpus is on disk now; training writes checkpoints and the adapter. + policy.check_free(job_dir)?; let mut job = spec.job.clone(); job["dataset"] = json!({"url": "dataset.jsonl", "sha256": spec.dataset_hash.trim_start_matches("sha256:")}); job["output"] = json!({"destination_url": "adapter"}); @@ -536,7 +690,7 @@ impl Drop for RemoveDirOnDrop { } } -pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc, in_flight: InFlight) -> mpsc::Sender { +pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc, in_flight: InFlight, policy: Arc) -> mpsc::Sender { let (tx, mut rx) = mpsc::channel::(8); tokio::spawn(async move { while let Some(job) = rx.recv().await { @@ -553,7 +707,7 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root let result: Result = crate::job_control::cancellable(&accept.job_id, async { let spec: TextTrainSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?; let buyer = accept.buyer_peer_id.parse::().ok(); - let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer).await?; + let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer, &policy).await?; if spec.adapter_pieces { // Receipt -> manifest -> pieces, all over libp2p. // Sealed to the buyer for e2e jobs. @@ -582,8 +736,10 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root tx } -/// Bid only on valid text-train offers, and only while idle. -pub fn train_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter { +/// Bid only on valid text-train offers for a base model this node runs, and +/// only while idle. A job for another model gets no bid, so its buyer is not +/// charged and a worker that does run the model can take it. +pub fn train_offer_filter(busy: Arc, policy: Arc) -> crate::dispatch::OfferFilter { Arc::new(move |offer| { !busy.load(Ordering::Relaxed) && match crate::job_e2e::bid_spec(offer) { @@ -592,11 +748,11 @@ pub fn train_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter Some(crate::job_e2e::BidSpec::Public(v)) => { let kind = v["job"]["kind"].as_str().unwrap_or(""); let base = v["job"]["base_model"].as_str().unwrap_or(""); - KINDS.contains(&kind) && valid_repo_id(base) - } - Some(crate::job_e2e::BidSpec::Full(v)) => { - serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok()) + KINDS.contains(&kind) && policy.admits_model(base) } + Some(crate::job_e2e::BidSpec::Full(v)) => serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|s| s.validate().is_ok() && policy.admits_model(s.job["base_model"].as_str().unwrap_or(""))), None => false, } }) @@ -652,10 +808,6 @@ pub struct TextInferSpec { pub adapter_key: Option, } -fn valid_repo_id(base: &str) -> bool { - !(base.is_empty() || base.len() > 200 || base.starts_with('/') || base.contains("..") || base.contains("://")) -} - impl TextInferSpec { pub fn validate(&self) -> Result<()> { if !crate::temporal::is_sha256_ref(&self.adapter_hash) { @@ -674,6 +826,10 @@ impl TextInferSpec { bail!("message role must be system, user or assistant"); } bytes += m["content"].as_str().context("message content must be a string")?.len(); + // The messages go to the generator as they are: nothing rides along. + if m.as_object().is_none_or(|o| o.len() != 2) { + bail!("a message carries only role and content"); + } } if bytes > MAX_PROMPT_BYTES { bail!("messages exceed {MAX_PROMPT_BYTES} bytes"); @@ -746,34 +902,109 @@ async fn fetch_adapter(url: &str, want: &str, dir: &Path) -> Result<()> { Ok(()) } +/// The digest an adapter cache entry is filed under: every one of +/// [`ADAPTER_FILES`], in that order, by name and sha256 (or as absent). +/// Returns it with the weights' `sha256:` hash. Fails when the directory +/// holds anything else, or no weights: such a directory is never an entry. +pub fn adapter_cache_key(dir: &Path) -> Result<(String, String)> { + for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry?; + let name = entry.file_name(); + if !entry.file_type()?.is_file() || !ADAPTER_FILES.iter().any(|f| name.to_str() == Some(*f)) { + bail!("{} holds {:?}, which is not an adapter file", dir.display(), name); + } + } + let mut key = Sha256::new(); + let mut weights = None; + for f in ADAPTER_FILES { + let p = dir.join(f); + let hash = if p.exists() { crate::temporal::worker::file_sha256(&p)? } else { "-".to_string() }; + key.update(f.as_bytes()); + key.update([0u8]); + key.update(hash.as_bytes()); + key.update([b'\n']); + if f == "adapter_model.safetensors" { + weights = Some(hash); + } + } + let weights = weights.filter(|w| w != "-").context("no adapter_model.safetensors")?; + Ok((hex::encode(key.finalize()), weights)) +} + +/// The cached adapter for `manifest` under `slot` (`//`), if its one entry still is what it was filed as: the directory's +/// name is the [`adapter_cache_key`] of its contents, and its weights are +/// the ones the request names. +fn cached_adapter(slot: &Path, adapter_hash: &str) -> Option { + std::fs::read_dir(slot).ok()?.flatten().map(|e| e.path()).find(|dir| { + adapter_cache_key(dir).is_ok_and(|(key, weights)| dir.file_name().and_then(|n| n.to_str()) == Some(key.as_str()) && weights == adapter_hash) + }) +} + /// Answer one request: adapter from the cache (or fetched), then `infernet model generate`. /// `buyer` is the peer adapter pieces are fetched from first. -pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option) -> Result> { +pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option, policy: &TextConfig) -> Result> { spec.validate()?; - // A sealed adapter (DIP-0032) is the buyer's private data: it lives only - // in this job's directory, removed when the job ends, never in the cache. + if !policy.admits_model(&spec.base_model) { + bail!("base model {} is not one this node runs ([text] base_models)", spec.base_model); + } + tokio::fs::create_dir_all(cache_root.join("text")).await?; + policy.check_free(&cache_root.join("text"))?; let input_key = spec.adapter_key.as_deref().map(crate::job_e2e::key_from_hex).transpose()?; - let (adapter, _private_adapter) = match &input_key { - Some(_) => { - let d = crate::runner::job_subdir(&cache_root.join("text").join("infer-adapters"), job_id)?; - (d.clone(), Some(RemoveDirOnDrop(d))) + let (adapter, _private_adapter) = match (&input_key, &spec.adapter_manifest) { + // An unsealed manifest names every file of the adapter by content + // hash, so the same manifest is the same adapter: the one case that + // is cached, in a directory filed under the digest of its files and + // checked against it on every hit. (Filing by the weights' hash + // alone let one buyer leave an adapter_config.json that the next + // buyer of the same weights then ran with.) + (None, Some(m)) => { + let slot = cache_root.join("text").join("adapter-cache").join(m); + let cached = cached_adapter(&slot, &spec.adapter_hash); + let dir = match cached { + Some(dir) => dir, + None => { + let _ = tokio::fs::remove_dir_all(&slot).await; + tokio::fs::create_dir_all(&slot).await?; + let tmp = slot.join(format!("fetch-{}", uuid::Uuid::new_v4())); + let filed: Result = async { + let got = fetch_adapter_pieces(net, buyer, m, &tmp, None).await.context("fetch adapter pieces")?; + if got != spec.adapter_hash { + bail!("adapter {got} differs from the offer's {}", spec.adapter_hash); + } + let dir = slot.join(adapter_cache_key(&tmp)?.0); + tokio::fs::rename(&tmp, &dir).await?; + Ok(dir) + } + .await; + if filed.is_err() { + let _ = tokio::fs::remove_dir_all(&slot).await; + } + filed? + } + }; + (dir, None) } - None => (cache_root.join("text").join("adapters").join(spec.adapter_hash.trim_start_matches("sha256:")), None), - }; - let cached = crate::temporal::worker::file_sha256(&adapter.join("adapter_model.safetensors")).ok(); - if cached.as_deref() != Some(spec.adapter_hash.as_str()) { - match &spec.adapter_manifest { - Some(m) => { - let _ = tokio::fs::remove_dir_all(&adapter).await; - let got = fetch_adapter_pieces(net, buyer, m, &adapter, input_key.as_ref()).await.context("fetch adapter pieces")?; - if got != spec.adapter_hash { - let _ = tokio::fs::remove_dir_all(&adapter).await; - bail!("adapter {got} differs from the offer's {}", spec.adapter_hash); + // A sealed adapter (DIP-0032) is the buyer's private data, and an + // HTTP bundle names only its weights' hash, not the files beside + // them: both live only in this job's directory, removed when the + // job ends, never in the cache. + _ => { + let d = crate::runner::job_subdir(&cache_root.join("text").join("infer-adapters"), job_id)?; + let private = RemoveDirOnDrop(d.clone()); + match &spec.adapter_manifest { + Some(m) => { + let _ = tokio::fs::remove_dir_all(&d).await; + let got = fetch_adapter_pieces(net, buyer, m, &d, input_key.as_ref()).await.context("fetch adapter pieces")?; + if got != spec.adapter_hash { + bail!("adapter {got} differs from the offer's {}", spec.adapter_hash); + } } + None => fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &d).await.context("fetch adapter")?, } - None => fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &adapter).await.context("fetch adapter")?, + (d, Some(private)) } - } + }; let dir = crate::runner::job_subdir(&cache_root.join("text").join("infer"), job_id)?; tokio::fs::create_dir_all(&dir).await?; // The request holds the prompt: removed on every exit, timeouts included. @@ -796,7 +1027,7 @@ pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: & Ok(serde_json::to_vec(&ans)?) } -pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc) -> mpsc::Sender { +pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, policy: Arc) -> mpsc::Sender { let (tx, mut rx) = mpsc::channel::(16); tokio::spawn(async move { while let Some(job) = rx.recv().await { @@ -805,7 +1036,7 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root let result: Result = crate::job_control::cancellable(&accept.job_id, async { let spec: TextInferSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?; let buyer = accept.buyer_peer_id.parse::().ok(); - let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer).await?; + let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer, &policy).await?; if spec.answer_chunk { // Sealed to the buyer for e2e jobs (one chunk either way). let answer = crate::job_e2e::seal_result(&accept, &answer)?.unwrap_or(answer); @@ -834,17 +1065,18 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root tx } -pub fn infer_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter { +/// Bid only on valid requests for a base model this node runs. +pub fn infer_offer_filter(busy: Arc, policy: Arc) -> crate::dispatch::OfferFilter { Arc::new(move |offer| { !busy.load(Ordering::Relaxed) && match crate::job_e2e::bid_spec(offer) { Some(crate::job_e2e::BidSpec::Public(v)) => { - valid_repo_id(v["base_model"].as_str().unwrap_or("")) + policy.admits_model(v["base_model"].as_str().unwrap_or("")) && v["max_new_tokens"].as_u64().is_some_and(|n| (1..=1024).contains(&n)) } - Some(crate::job_e2e::BidSpec::Full(v)) => { - serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok()) - } + Some(crate::job_e2e::BidSpec::Full(v)) => serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|s| s.validate().is_ok() && policy.admits_model(&s.base_model)), None => false, } }) @@ -926,17 +1158,21 @@ pub async fn serve(net_opts: &NetOptions, infernet: PathBuf, cache_root: PathBuf let net = spawn_node_with_store(net_opts, store.clone()).await?; let hardware = json!({"version": env!("CARGO_PKG_VERSION"), "text_runtime": "peft", "ram_mb": ram_mb, "cuda": runtime["cuda"]}); tokio::spawn(crate::capabilities::advertise_loop(net.clone(), tags.clone(), hardware, crate::capabilities::local_bench_score(), crate::capabilities::DEFAULT_ADVERTISE_INTERVAL)); + let policy = Arc::new(TextConfig::load()); + // Adapters were once cached by their weights' hash alone; those entries + // are never read again (see `run_infer`), so they only take up disk. + let _ = tokio::fs::remove_dir_all(cache_root.join("text").join("adapters")).await; let busy = Arc::new(AtomicBool::new(false)); let slots = Arc::new(Semaphore::new(1)); let in_flight = InFlight::default(); - let tx = spawn_train_runner(net.clone(), store.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots, in_flight.clone()); - crate::dispatch::run_worker_subscriber_filtered(net.clone(), TRAIN_WORKLOAD.into(), tags.clone(), Some(tx), Some(train_offer_filter(busy))); + let tx = spawn_train_runner(net.clone(), store.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots, in_flight.clone(), policy.clone()); + crate::dispatch::run_worker_subscriber_filtered(net.clone(), TRAIN_WORKLOAD.into(), tags.clone(), Some(tx), Some(train_offer_filter(busy, policy.clone()))); if infer { let ibusy = Arc::new(AtomicBool::new(false)); - let itx = spawn_infer_runner(net.clone(), store.clone(), cache_root, infernet, ibusy.clone()); - crate::dispatch::run_worker_subscriber_filtered(net.clone(), INFER_WORKLOAD.into(), tags.clone(), Some(itx), Some(infer_offer_filter(ibusy))); + let itx = spawn_infer_runner(net.clone(), store.clone(), cache_root, infernet, ibusy.clone(), policy.clone()); + crate::dispatch::run_worker_subscriber_filtered(net.clone(), INFER_WORKLOAD.into(), tags.clone(), Some(itx), Some(infer_offer_filter(ibusy, policy.clone()))); } - info!(peer = %net.peer_id().to_base58(), ?tags, "text: serving"); + info!(peer = %net.peer_id().to_base58(), ?tags, base_models = ?policy.base_models, "text: serving"); tokio::time::sleep(Duration::from_millis(300)).await; Ok(Serving { net, tags, runtime, infer, in_flight }) } @@ -1880,6 +2116,285 @@ mod tests { assert!(bad.validate().is_err()); } + fn allow(entries: &[&str]) -> Vec { + entries.iter().map(|e| e.to_string()).collect() + } + + #[test] + fn base_model_allowlist_matches_exactly_or_by_org() { + let list = allow(&["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/*"]); + // Exact. + assert!(model_allowed(&list, "HuggingFaceTB/SmolLM2-135M-Instruct")); + assert!(!model_allowed(&list, "HuggingFaceTB/SmolLM2-360M-Instruct")); + assert!(!model_allowed(&list, "HuggingFaceTB/SmolLM2-135M-Instruct-evil")); + // A whole organisation, and only that organisation. + assert!(model_allowed(&list, "Qwen/Qwen2.5-0.5B-Instruct")); + assert!(model_allowed(&list, "Qwen/anything-else")); + assert!(!model_allowed(&list, "QwenX/Qwen2.5-0.5B-Instruct")); + assert!(!model_allowed(&list, "Qwe/n")); + assert!(!model_allowed(&list, "evil/Qwen")); + // Case matters: another spelling is another copy in the cache. + assert!(!model_allowed(&list, "huggingfacetb/smollm2-135m-instruct")); + assert!(!model_allowed(&list, "qwen/Qwen2.5-0.5B-Instruct")); + // Paths, traversal and URLs never match, whatever the list says. + let any = allow(&["*", "Qwen/*", "../*", "/*"]); + for bad in [ + "", "gpt2", "/etc/passwd", "../x", "Qwen/../etc", "Qwen/..", "./Qwen/x", "Qwen/x/y", "Qwen//x", "/Qwen/x", "Qwen/x/", + "https://evil/model", "Qwen/x y", "Qwen/x\n", "Qwen\\x", ".Qwen/x", "Qwen/.x", "-Qwen/x", "Qwen/-x", "Qwen/x.", "Qwen/a--b", "~/model", "Qwen/*", + ] { + assert!(!model_allowed(&any, bad), "{bad:?}"); + assert!(!valid_repo_id(bad), "{bad:?}"); + } + assert!(!valid_repo_id(&format!("org/{}", "a".repeat(97)))); + // "*" is the operator's way back to any model, still only by repo id. + assert!(model_allowed(&any, "someone/some-model_v1.5")); + assert!(!model_allowed(&[], "Qwen/Qwen2.5-0.5B-Instruct"), "an empty list runs nothing"); + // Entries are trimmed; an org entry needs its `/*`. + assert!(model_allowed(&allow(&[" Qwen/* "]), "Qwen/x")); + assert!(!model_allowed(&allow(&["Qwen"]), "Qwen/x")); + } + + #[test] + fn the_default_allowlist_is_what_the_clients_name_and_config_replaces_it() { + let cfg = TextConfig::default(); + // `c0mpute train lora --base`'s example and the /train page's job file. + assert_eq!(cfg.base_models, vec!["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"]); + assert!(cfg.base_models.iter().all(|m| cfg.admits_model(m) && valid_repo_id(m))); + assert!(!cfg.admits_model("meta-llama/Llama-3.1-8B-Instruct")); + // A config from before the section existed gets the defaults. + let old: crate::config::Config = toml::from_str("roles = []\n").unwrap(); + assert_eq!(old.text, cfg); + let set: crate::config::Config = toml::from_str("[text]\nbase_models = [\"meta-llama/*\"]\nmin_free_gb = 50\n").unwrap(); + assert!(set.text.admits_model("meta-llama/Llama-3.1-8B-Instruct")); + assert!(!set.text.admits_model("Qwen/Qwen2.5-0.5B-Instruct"), "the list replaces the default"); + assert_eq!(set.text.min_free_gb, 50); + } + + fn offer(workload: &str, spec: Value, sealed: bool) -> c0mpute_net::topics::JobOffer { + let mut o = json!({ + "job_id": "j", "workload_type": workload, "buyer_peer_id": "buyer", "spec_hash": "", + "required_capabilities": [], "max_price_usd": 1.0, "deadline_unix_ms": 2_000_000, "published_at_ms": 1_000_000, + }); + if sealed { + // What a buyer publishes for a sealed job: the public view only. + o["e2e"] = json!({"v": 1, "buyer_key": "00".repeat(32), "public_spec": crate::job_e2e::public_view(workload, &spec)}); + } else { + o["spec_inline"] = spec; + } + serde_json::from_value(o).unwrap() + } + + #[test] + fn a_job_for_a_model_off_the_list_gets_no_bid() { + let policy = Arc::new(TextConfig::default()); + let idle = || Arc::new(AtomicBool::new(false)); + let train = train_offer_filter(idle(), policy.clone()); + let infer = infer_offer_filter(idle(), policy.clone()); + let with_base = |base: &str| { + let mut t = p2p_spec(); + t.job["base_model"] = json!(base); + let i = TextInferSpec { adapter_manifest: Some(c0mpute_proto::Hash::of(b"m").to_hex()), answer_chunk: true, base_model: base.into(), ..infer_spec() }; + (serde_json::to_value(t).unwrap(), serde_json::to_value(i).unwrap()) + }; + for sealed in [false, true] { + let (t, i) = with_base("Qwen/Qwen2.5-0.5B-Instruct"); + assert!(train(&offer(TRAIN_WORKLOAD, t, sealed)), "sealed={sealed}"); + assert!(infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}"); + // A valid repo id, not one this operator runs. + let (t, i) = with_base("meta-llama/Llama-3.1-8B-Instruct"); + assert!(!train(&offer(TRAIN_WORKLOAD, t, sealed)), "sealed={sealed}"); + assert!(!infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}"); + // Shaped like a repo id, but it could as well be a directory + // under the trainer's working directory: the list decides. + let (t, i) = with_base("models/local-dir"); + assert!(!train(&offer(TRAIN_WORKLOAD, t, sealed)) && !infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}"); + } + // An operator who lists the org bids on it. + let wide = Arc::new(TextConfig { base_models: allow(&["meta-llama/*"]), ..TextConfig::default() }); + let (t, _) = with_base("meta-llama/Llama-3.1-8B-Instruct"); + assert!(train_offer_filter(idle(), wide)(&offer(TRAIN_WORKLOAD, t.clone(), true))); + // Busy workers still do not bid. + assert!(!train_offer_filter(Arc::new(AtomicBool::new(true)), Arc::new(TextConfig { base_models: allow(&["*"]), ..TextConfig::default() }))(&offer(TRAIN_WORKLOAD, t, false))); + } + + fn train_request(replicas: u32) -> TrainRequest { + TrainRequest { + job_path: "job.json".into(), + dataset_path: None, + output: "adapter".into(), + workers: vec![], + max_price_usd: 1.0, + gpu: false, + min_ram_gb: None, + legacy_http: false, + public_url: None, + bind: "127.0.0.1:0".parse().unwrap(), + timeout_s: 60, + retries: 0, + bid_window: Duration::from_secs(1), + replicas, + replica_tolerance: crate::train_verify::DEFAULT_TOLERANCE, + penalty_file: None, + } + } + + #[test] + fn job_validation_accepts_every_shape_the_clients_send() { + use crate::train::{DatasetReport, LoraOptions}; + let report = DatasetReport { system_prompt: Some("You are a pricing coach.".into()), ..Default::default() }; + // `c0mpute train lora` with its defaults, and with every flag set to + // each end of the range the CLI itself accepts. + let defaults = LoraOptions { base_model: "Qwen/Qwen2.5-0.5B-Instruct".into(), ..Default::default() }; + let low = LoraOptions { name: "a".into(), epochs: 1, rank: 1, lr: 1e-9, seq_len: 32, batch_size: 1, max_steps: Some(1), system_prompt: None, ..defaults.clone() }; + let high = LoraOptions { + name: "A-b_c.9".repeat(9), + epochs: 20, + rank: 128, + lr: 1e-2, + seq_len: 4096, + batch_size: 64, + max_steps: Some(1_000_000), + system_prompt: Some("s".repeat(8 * 1024)), + ..defaults.clone() + }; + for opts in [&defaults, &low, &high] { + opts.validate().unwrap(); + validate_job(&opts.job(&report)).unwrap_or_else(|e| panic!("{opts:?}: {e}")); + validate_job(&opts.job(&DatasetReport::default())).unwrap(); + } + // With --replicas the buyer pins a seed in the job (and takes + // `replicas` out of it before it is sent). + let mut job = defaults.job(&report); + job["replicas"] = json!(3); + assert_eq!(replica_count(&train_request(0), &mut job).unwrap(), 3); + assert_eq!(job["hyperparams"]["seed"], 0); + validate_job(&job).unwrap(); + // The job file on the /train page (apps/web/src/app/train/page.tsx), + // which has no dataset, output or name the CLI would add. + let page: Value = serde_json::from_str( + r#"{ "name": "my-agent", "kind": "lora", + "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct", + "system_prompt": "You are a concise, practical business strategist.", + "hyperparams": { "max_steps": 200, "batch_size": 2, "seq_len": 256 } }"#, + ) + .unwrap(); + validate_job(&page).unwrap(); + let mut page_replicas = page.clone(); + assert_eq!(replica_count(&train_request(3), &mut page_replicas).unwrap(), 3); + validate_job(&page_replicas).unwrap(); + // The smallest job, the timeout the worker reads, and `peft`. + validate_job(&json!({"kind": "peft", "base_model": "a/b"})).unwrap(); + validate_job(&json!({"kind": "lora", "base_model": "a/b", "limits": {"timeout_s": 600}, "system_prompt": null})).unwrap(); + // And the whole spec the buyer builds around it. + TextTrainSpec { job: high.job(&report), ..p2p_spec() }.validate().unwrap(); + } + + #[test] + fn job_validation_refuses_what_the_trainer_was_not_asked_for() { + let job = |patch: Value| { + let mut j = json!({"kind": "lora", "base_model": "Qwen/Qwen2.5-0.5B-Instruct", "hyperparams": {"lr": 2e-4, "epochs": 1}}); + for (k, v) in patch.as_object().unwrap() { + j[k] = v.clone(); + } + j + }; + let err = |patch: Value| validate_job(&job(patch)).unwrap_err().to_string(); + // Unknown top-level keys are named, with the keys that are accepted. + let e = err(json!({"trust_remote_code": true})); + assert!(e.contains("job.trust_remote_code") && e.contains("hyperparams") && e.contains("system_prompt"), "{e}"); + assert!(err(json!({"replicas": 3})).contains("job.replicas")); + // Unknown hyperparameters would become trainer arguments. + let e = err(json!({"hyperparams": {"output_dir": "/home/op/.ssh"}})); + assert!(e.contains("hyperparams.output_dir") && e.contains("lora_r"), "{e}"); + assert!(err(json!({"hyperparams": {"trust_remote_code": true}})).contains("not accepted")); + // Types and ranges. + for bad in [ + json!({"lr": "2e-4"}), json!({"lr": 0}), json!({"lr": -1e-4}), json!({"lr": 0.5}), json!({"lr": [2e-4]}), + json!({"epochs": 0}), json!({"epochs": 21}), json!({"epochs": 1.5}), json!({"epochs": -1}), json!({"epochs": "1"}), + json!({"batch_size": 65}), json!({"seq_len": 31}), json!({"seq_len": 1_000_000}), json!({"lora_r": 129}), + json!({"max_steps": 0}), json!({"max_steps": 1_000_001}), json!({"seed": -1}), json!({"seed": 4294967296u64}), + json!({"lora_r": {"nested": 1}}), json!({"epochs": null}), + ] { + assert!(err(json!({"hyperparams": bad.clone()})).contains("out of range"), "{bad}"); + } + assert!(err(json!({"hyperparams": [1, 2]})).contains("must be an object")); + assert!(err(json!({"hyperparams": "lr=1"})).contains("must be an object")); + assert!(err(json!({"name": "a b"})).contains("job.name")); + assert!(err(json!({"name": 7})).contains("job.name")); + assert!(err(json!({"system_prompt": "s".repeat(8 * 1024 + 1)})).contains("system_prompt")); + assert!(err(json!({"system_prompt": ["a"]})).contains("system_prompt")); + assert!(err(json!({"limits": {"timeout_s": 60, "max_memory": 1}})).contains("limits")); + assert!(err(json!({"limits": {"timeout_s": "60"}})).contains("limits")); + assert!(err(json!({"dataset": "/etc/passwd"})).contains("job.dataset")); + assert!(validate_job(&json!(["lora"])).is_err()); + // The spec check runs it: a worker never writes such a job to disk. + let mut s = p2p_spec(); + s.job["hyperparams"] = json!({"output_dir": "/tmp/x"}); + assert!(s.validate().is_err()); + // A chat message carries a role and its text, nothing else. + let mut i = infer_spec(); + i.messages = vec![json!({"role": "user", "content": "hi", "tool_calls": [{"name": "x"}]})]; + assert!(i.validate().unwrap_err().to_string().contains("only role and content")); + } + + #[test] + fn adapter_cache_entries_are_filed_under_every_file_and_checked_on_a_hit() { + let tmp = tempfile::tempdir().unwrap(); + let write = |dir: &Path, config: &[u8]| { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("adapter_model.safetensors"), b"weights").unwrap(); + std::fs::write(dir.join("adapter_config.json"), config).unwrap(); + std::fs::write(dir.join("manifest.json"), br#"{"base_model":"org/m"}"#).unwrap(); + }; + let (a, b) = (tmp.path().join("a"), tmp.path().join("b")); + write(&a, b"{\"r\": 8}"); + write(&b, b"{\"r\": 8, \"base_model_name_or_path\": \"/home/op/private\"}"); + let (key_a, weights_a) = adapter_cache_key(&a).unwrap(); + let (key_b, weights_b) = adapter_cache_key(&b).unwrap(); + // Same weights, another config: the old cache key was the same. + assert_eq!(weights_a, sha256_hex(b"weights")); + assert_eq!(weights_a, weights_b); + assert_ne!(key_a, key_b, "the key covers adapter_config.json"); + assert_eq!(adapter_cache_key(&a).unwrap().0, key_a, "and is stable"); + // A missing file is not the same as an empty one. + let (c, d) = (tmp.path().join("c"), tmp.path().join("d")); + write(&c, b""); + write(&d, b""); + std::fs::remove_file(d.join("adapter_config.json")).unwrap(); + assert_ne!(adapter_cache_key(&c).unwrap().0, adapter_cache_key(&d).unwrap().0); + // Anything that is not an adapter file, or no weights: not an entry. + std::fs::write(c.join("extra.py"), b"x").unwrap(); + assert!(adapter_cache_key(&c).is_err()); + std::fs::remove_file(d.join("adapter_model.safetensors")).unwrap(); + assert!(adapter_cache_key(&d).is_err()); + + // A slot holds the entry under its key; a hit recomputes it. + let slot = tmp.path().join("slot"); + let entry = slot.join(&key_a); + write(&entry, b"{\"r\": 8}"); + assert_eq!(cached_adapter(&slot, &weights_a), Some(entry.clone())); + assert_eq!(cached_adapter(&slot, &sha256_hex(b"other weights")), None, "not the weights the request names"); + assert_eq!(cached_adapter(&tmp.path().join("no-such-slot"), &weights_a), None); + // Changed after it was filed (a config swapped in beside the same + // weights): never used again. + std::fs::write(entry.join("adapter_config.json"), b"{\"r\": 8, \"base_model_name_or_path\": \"/home/op/private\"}").unwrap(); + assert_eq!(cached_adapter(&slot, &weights_a), None); + write(&entry, b"{\"r\": 8}"); + std::fs::write(entry.join("tokenizer.json"), b"{}").unwrap(); + assert_eq!(cached_adapter(&slot, &weights_a), None, "a file dropped beside the adapter"); + } + + #[test] + fn a_job_starts_only_with_the_free_space_floor_left() { + let tmp = tempfile::tempdir().unwrap(); + TextConfig { min_free_gb: 0, ..TextConfig::default() }.check_free(Path::new("relative/unknown")).unwrap(); + // Where the disk is known, a floor above its size refuses the job. + if c0mpute_store::free_space_at(tmp.path()).0 != u64::MAX { + let err = TextConfig { min_free_gb: 1 << 30, ..TextConfig::default() }.check_free(tmp.path()).unwrap_err(); + assert!(err.to_string().starts_with("E_DISK:"), "{err}"); + } + } + #[test] fn conversations_require_the_messages_format_tag() { let text = b"{\"title\":\"a\",\"text\":\"b\"}\n{\"title\":\"c\",\"text\":\"the word \\\"messages\\\" in text\"}\n"; diff --git a/node/crates/c0mpute-core/tests/apps_e2e.rs b/node/crates/c0mpute-core/tests/apps_e2e.rs index c21a9806..28fde546 100644 --- a/node/crates/c0mpute-core/tests/apps_e2e.rs +++ b/node/crates/c0mpute-core/tests/apps_e2e.rs @@ -171,6 +171,12 @@ async fn apps_place_three_copies_serve_snapshot_and_fail_over() { hosts, dir: Some(dir.join("apps")), timing: fast(), + // Docker is a script here: no real network to guard (as root the + // guard would add iptables rules to the machine running the test), + // and nothing read from this machine's config file. + egress_guard: Some(false), + require_egress_guard: Some(false), + max_disk_mb: Some(c0mpute_core::apps::DEFAULT_MAX_DISK_MB), ..Default::default() }; let d0 = tmp.path().join("n0"); diff --git a/node/crates/c0mpute-gateway/src/lib.rs b/node/crates/c0mpute-gateway/src/lib.rs index 5315718b..774a2d3d 100644 --- a/node/crates/c0mpute-gateway/src/lib.rs +++ b/node/crates/c0mpute-gateway/src/lib.rs @@ -7,6 +7,7 @@ pub mod auth; pub mod host_guard; +pub mod site_host; pub mod storage_api; use std::net::SocketAddr; @@ -59,6 +60,9 @@ pub async fn serve_with_storage( pub async fn serve_router(app: Router, addr: SocketAddr) -> Result<()> { info!(%addr, "gateway listening"); let listener = tokio::net::TcpListener::bind(addr).await?; + // On a per-site host nothing but that site is reachable, whatever + // routes were merged into `app` (a no-op until a suffix is configured). + let app = site_host::isolate(app); // Peer addresses reach handlers as ConnectInfo (access log). axum::serve(listener, app.into_make_service_with_connect_info::()).await?; Ok(()) diff --git a/node/crates/c0mpute-gateway/src/site_host.rs b/node/crates/c0mpute-gateway/src/site_host.rs new file mode 100644 index 00000000..8c598711 --- /dev/null +++ b/node/crates/c0mpute-gateway/src/site_host.rs @@ -0,0 +1,331 @@ +//! Per-site hostnames: `.` is one hosted site and nothing +//! else. +//! +//! Every hosted site used to be reachable only at `/s//`, +//! one web origin shared by all sites and by the gateway's own APIs (storage +//! writes, git, swarm routes): any site's script could read another site's +//! localStorage and call those APIs as the same origin. With a suffix +//! configured (`[gateway] site_host_suffix`), each site gets an origin of its +//! own, and on that origin the only thing served is the site. +//! +//! This module is the part every router on the gateway shares: telling a +//! per-site host from any other ([`site_of_host`]), and making sure a request +//! on one can reach nothing but site serving ([`isolate`]). The hosting +//! service serves the site itself. + +use std::sync::RwLock; + +use axum::{ + Router, + extract::Request, + http::{HeaderMap, Method, StatusCode, Uri, header}, + middleware::{self, Next}, + response::{IntoResponse, Response}, +}; + +/// Marks a request that arrived on a per-site host. [`isolate`] adds it +/// after rewriting the path to `/s//`; whatever serves sites +/// reads it to know the visitor's URL has no `/s/` prefix. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SiteHost { + pub site: String, +} + +/// The suffix this process serves per-site hosts under, set once the hosting +/// service knows its configuration. +static SUFFIX: RwLock> = RwLock::new(None); + +/// Set the per-site host suffix for this process (see [`isolate`]). +pub fn configure(suffix: &str) { + if let Some(s) = normalize_suffix(suffix) { + *SUFFIX.write().unwrap_or_else(|e| e.into_inner()) = Some(s); + } +} + +/// The suffix set with [`configure`], if any. +pub fn configured() -> Option { + SUFFIX.read().unwrap_or_else(|e| e.into_inner()).clone() +} + +/// A configured suffix in the form hosts are compared with: lower case, no +/// leading `*.` or dots, no trailing dot. `None` when what is left is not a +/// DNS name of at least two labels. +pub fn normalize_suffix(raw: &str) -> Option { + let s = raw.trim().to_ascii_lowercase(); + let s = s.strip_prefix("*.").unwrap_or(&s).trim_matches('.').to_string(); + let label_ok = |l: &str| { + !l.is_empty() + && l.len() <= 63 + && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + && !l.starts_with('-') + && !l.ends_with('-') + }; + // 20 characters of site id and a dot go in front of it. + (s.len() <= 232 && s.contains('.') && s.split('.').all(label_ok)).then_some(s) +} + +/// A site id: 20 lower-case hex characters (it is one DNS label as it is). +pub fn valid_site_id(s: &str) -> bool { + s.len() == 20 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// The site a `Host` value names under `suffix` (already normalized): +/// exactly `.`, with an optional port and trailing dot. +/// Host names are case-insensitive, so the id is matched in lower case; a +/// name with more labels in front, or anything that is not a site id, is +/// not a per-site host. +pub fn site_of_host(host: &str, suffix: &str) -> Option { + let host = host.trim(); + let host = match host.rsplit_once(':') { + Some((h, p)) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => h, + _ => host, + }; + let host = host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase(); + let label = host.strip_suffix(suffix)?.strip_suffix('.')?; + valid_site_id(label).then(|| label.to_string()) +} + +/// The host a request is for: its `Host` header, else the authority of its +/// URI (HTTP/2 requests may carry only that). +pub fn request_host(headers: &HeaderMap, uri: &Uri) -> Option { + headers + .get(header::HOST) + .and_then(|v| v.to_str().ok()) + .map(str::to_string) + .or_else(|| uri.authority().map(|a| a.as_str().to_string())) +} + +/// Where `/s//` lives on the site's own host. `rest` is the path +/// after `/s//` exactly as the visitor sent it (still percent-encoded); +/// the query is kept. `None` for a site id or suffix that could not be part +/// of a URL. +pub fn site_url(site: &str, suffix: &str, rest: &str, query: Option<&str>) -> Option { + if !valid_site_id(site) || normalize_suffix(suffix).as_deref() != Some(suffix) { + return None; + } + let rest = rest.trim_start_matches('/'); + Some(match query { + Some(q) if !q.is_empty() => format!("https://{site}.{suffix}/{rest}?{q}"), + _ => format!("https://{site}.{suffix}/{rest}"), + }) +} + +/// Wrap the whole gateway so that a request on a per-site host can only +/// reach site serving: its path is rewritten to `/s/` before any +/// routing (and marked with [`SiteHost`]), so no other route (storage API, +/// git, swarm, the `/p/` loader) can match it, and anything but GET or HEAD +/// is answered 404 here. Requests on every other host pass through +/// untouched. The suffix is read per request from [`configured`]. +pub fn isolate(app: Router) -> Router { + wrap(app, None) +} + +/// [`isolate`] with the suffix given rather than read from [`configured`]. +pub fn isolate_with(app: Router, suffix: &str) -> Router { + wrap(app, normalize_suffix(suffix)) +} + +fn wrap(app: Router, fixed: Option) -> Router { + // A router with nothing but a fallback: its middleware runs before the + // wrapped router routes, so the rewritten path is what gets routed. + Router::new() + .fallback_service(app) + .layer(middleware::from_fn(move |req: Request, next: Next| { + let suffix = fixed.clone().or_else(configured); + async move { guard(suffix, req, next).await } + })) +} + +async fn guard(suffix: Option, mut req: Request, next: Next) -> Response { + let Some(suffix) = suffix else { + return next.run(req).await; + }; + let site = request_host(req.headers(), req.uri()).and_then(|h| site_of_host(&h, &suffix)); + // Already rewritten for this host by a wrap further out: leave it. + if site.is_some() && req.extensions().get::().map(|m| &m.site) == site.as_ref() { + return next.run(req).await; + } + // Any other mark was not put there for this request's host. + req.extensions_mut().remove::(); + let Some(site) = site else { + return next.run(req).await; + }; + if req.method() != Method::GET && req.method() != Method::HEAD { + return not_found(); + } + let Some(uri) = rewritten(req.uri(), &site) else { + return not_found(); + }; + *req.uri_mut() = uri; + req.extensions_mut().insert(SiteHost { site }); + next.run(req).await +} + +/// `uri` with its path moved under `/s/` (query kept). +fn rewritten(uri: &Uri, site: &str) -> Option { + let path = uri.path(); + let path = if path.starts_with('/') { path } else { "/" }; + let pq = match uri.query() { + Some(q) => format!("/s/{site}{path}?{q}"), + None => format!("/s/{site}{path}"), + }; + let mut parts = uri.clone().into_parts(); + parts.path_and_query = Some(pq.parse().ok()?); + Uri::from_parts(parts).ok() +} + +fn not_found() -> Response { + (StatusCode::NOT_FOUND, "not found").into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::routing::{get, post}; + use tower::ServiceExt; + + const SITE: &str = "0123456789abcdef0123"; + const SUFFIX: &str = "sites.example.com"; + + #[test] + fn a_site_host_is_exactly_a_site_id_under_the_suffix() { + let site = |h: &str| site_of_host(h, SUFFIX); + assert_eq!(site(&format!("{SITE}.{SUFFIX}")).as_deref(), Some(SITE)); + assert_eq!(site(&format!("{SITE}.{SUFFIX}:8443")).as_deref(), Some(SITE)); + assert_eq!(site(&format!("{SITE}.{SUFFIX}.")).as_deref(), Some(SITE)); + // Host names are case-insensitive. + assert_eq!(site(&format!("{}.SITES.Example.com", SITE.to_uppercase())).as_deref(), Some(SITE)); + // Not a site id: too short, too long, not hex, empty. + assert_eq!(site(&format!("0123456789abcdef012.{SUFFIX}")), None); + assert_eq!(site(&format!("0123456789abcdef01234.{SUFFIX}")), None); + assert_eq!(site(&format!("0123456789abcdefghij.{SUFFIX}")), None); + assert_eq!(site(&format!(".{SUFFIX}")), None); + assert_eq!(site(SUFFIX), None); + // More labels in front, another suffix, or the suffix as a mere + // ending of another name. + assert_eq!(site(&format!("evil.{SITE}.{SUFFIX}")), None); + assert_eq!(site(&format!("{SITE}.example.com")), None); + assert_eq!(site(&format!("{SITE}.notsites.example.com")), None); + assert_eq!(site(&format!("{SITE}{SUFFIX}")), None); + assert_eq!(site(&format!("{SITE}.{SUFFIX}.evil.com")), None); + assert_eq!(site(&format!("{SITE}.{SUFFIX}:80:80")), None); + assert_eq!(site(&format!("{SITE}.{SUFFIX}/x")), None); + assert_eq!(site("gateway.example.com"), None); + assert_eq!(site(""), None); + } + + #[test] + fn suffixes_are_normalized_or_refused() { + assert_eq!(normalize_suffix(" *.Sites.Example.com. ").as_deref(), Some(SUFFIX)); + assert_eq!(normalize_suffix(".sites.example.com").as_deref(), Some(SUFFIX)); + assert_eq!(normalize_suffix("localhost"), None, "one label: every host would be under it"); + assert_eq!(normalize_suffix(""), None); + assert_eq!(normalize_suffix("sites.example.com/x"), None); + assert_eq!(normalize_suffix("sites..example.com"), None); + assert_eq!(normalize_suffix("-sites.example.com"), None); + assert_eq!(normalize_suffix(&format!("{}.com", "a".repeat(64))), None); + } + + #[test] + fn the_redirect_target_keeps_path_and_query() { + let url = |rest: &str, q: Option<&str>| site_url(SITE, SUFFIX, rest, q); + assert_eq!(url("", None).unwrap(), format!("https://{SITE}.{SUFFIX}/")); + assert_eq!(url("docs/", None).unwrap(), format!("https://{SITE}.{SUFFIX}/docs/")); + assert_eq!( + url("a%20b/c.html", Some("x=1&y=%2F")).unwrap(), + format!("https://{SITE}.{SUFFIX}/a%20b/c.html?x=1&y=%2F"), + "the path and query go over as they were sent" + ); + assert_eq!(url("p", Some("")).unwrap(), format!("https://{SITE}.{SUFFIX}/p")); + // A leading slash cannot turn the path into another host. + assert_eq!(url("/evil.com/x", None).unwrap(), format!("https://{SITE}.{SUFFIX}/evil.com/x")); + // Only a real site id and a normalized suffix make a URL. + assert_eq!(site_url("evil.com/", SUFFIX, "", None), None); + assert_eq!(site_url(SITE, "Sites.Example.com", "", None), None); + assert_eq!(site_url(SITE, "example.com/@", "", None), None); + } + + /// A gateway with an API route, a write route and site serving that + /// answers with the path it was asked for. + fn gateway() -> Router { + Router::new() + .route("/storage/v1/objects", get(|| async { "api read" }).post(|| async { "api write" })) + .route("/swarm/v1/run", post(|| async { "swarm" })) + .route( + "/s/{site}/{*path}", + get(|req: Request| async move { + let marked = req.extensions().get::().map(|s| s.site.clone()).unwrap_or_default(); + format!("site {} marked={marked}", req.uri()) + }), + ) + } + + async fn ask(app: Router, method: &str, host: &str, uri: &str) -> (StatusCode, String) { + let req = axum::http::Request::builder() + .method(method) + .uri(uri) + .header("host", host) + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + (status, String::from_utf8_lossy(&body).into_owned()) + } + + #[tokio::test] + async fn api_routes_are_out_of_reach_on_a_site_host() { + let app = isolate_with(gateway(), SUFFIX); + let site_host = format!("{SITE}.{SUFFIX}"); + // On the gateway's own name everything is as before. + assert_eq!(ask(app.clone(), "GET", "gateway.example.com", "/storage/v1/objects").await.1, "api read"); + assert_eq!(ask(app.clone(), "POST", "gateway.example.com", "/storage/v1/objects").await.1, "api write"); + let (_, body) = ask(app.clone(), "GET", "gateway.example.com", &format!("/s/{SITE}/a.css")).await; + assert_eq!(body, format!("site /s/{SITE}/a.css marked=")); + // On a site host a write is refused before any route sees it... + for uri in ["/storage/v1/objects", "/swarm/v1/run", "/", &format!("/s/{SITE}/x")] { + let (status, body) = ask(app.clone(), "POST", &site_host, uri).await; + assert_eq!((status, body.as_str()), (StatusCode::NOT_FOUND, "not found"), "POST {uri}"); + assert_eq!(ask(app.clone(), "DELETE", &site_host, uri).await.0, StatusCode::NOT_FOUND); + } + // ...and a read of an API path is a read of that path of the site. + let (status, body) = ask(app.clone(), "GET", &site_host, "/storage/v1/objects?list=1").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, format!("site /s/{SITE}/storage/v1/objects?list=1 marked={SITE}")); + // Another site's /s/ path is a path of this site too, not that site. + let other = "ffffffffffffffffffff"; + let (_, body) = ask(app.clone(), "GET", &site_host, &format!("/s/{other}/secret")).await; + assert_eq!(body, format!("site /s/{SITE}/s/{other}/secret marked={SITE}")); + let (_, body) = ask(app.clone(), "GET", &format!("{SITE}.{SUFFIX}:443"), "/a%20b.html").await; + assert_eq!(body, format!("site /s/{SITE}/a%20b.html marked={SITE}")); + // Wrapped twice, the path is still moved once. + let twice = isolate_with(app.clone(), SUFFIX); + let (_, body) = ask(twice, "GET", &site_host, "/a.css").await; + assert_eq!(body, format!("site /s/{SITE}/a.css marked={SITE}")); + // A name that only looks like one is the gateway's. + let (_, body) = ask(app.clone(), "GET", &format!("x.{SITE}.{SUFFIX}"), "/storage/v1/objects").await; + assert_eq!(body, "api read"); + } + + #[tokio::test] + async fn without_a_suffix_nothing_changes_and_the_mark_cannot_be_forged() { + let plain = wrap(gateway(), None); + let host = format!("{SITE}.{SUFFIX}"); + if configured().is_none() { + assert_eq!(ask(plain, "POST", &host, "/storage/v1/objects").await.1, "api write"); + } + // A request that already carries the mark (only code in this process + // could add it) loses it on the gateway's own name. + let app = isolate_with(gateway(), SUFFIX); + let mut req = axum::http::Request::builder() + .uri(format!("/s/{SITE}/x")) + .header("host", "gateway.example.com") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(SiteHost { site: SITE.into() }); + let resp = app.oneshot(req).await.unwrap(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap(); + assert_eq!(String::from_utf8_lossy(&body), format!("site /s/{SITE}/x marked=")); + } +} diff --git a/node/crates/c0mpute-gateway/src/storage_api.rs b/node/crates/c0mpute-gateway/src/storage_api.rs index 5b7748ad..80ff0c7f 100644 --- a/node/crates/c0mpute-gateway/src/storage_api.rs +++ b/node/crates/c0mpute-gateway/src/storage_api.rs @@ -29,7 +29,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::get; use c0mpute_proto::Hash; use c0mpute_store::{ObjectManifest, Storage, Tier}; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use tracing::{info, warn}; @@ -430,23 +430,38 @@ async fn get_object( ) .into_response()); }; - let bytes = state - .storage - .get_range_with(&manifest, offset, len) - .await - .map_err(|e| ApiError::Internal(format!("{e:#}")))?; - let end = offset.saturating_add(bytes.len() as u64).saturating_sub(1); + // The length and the range are promised in the headers before the + // body is read, so the blocks must add up the way the manifest says. + if !manifest.tiles() { + return Err(ApiError::Internal(format!( + "manifest of {object_hash} does not describe {total} bytes block by block" + ))); + } + // Streamed block by block, like a whole object: `bytes=0-` on a + // multi-GiB object used to be assembled in memory first. Only the + // first block is read before answering, so an object that cannot be + // read is an error rather than a 206 that stops short. + let mut stream = state.storage.read_range_stream(manifest, offset, len); + let first = match stream.next().await { + Some(Ok(bytes)) => bytes, + Some(Err(e)) => return Err(ApiError::Internal(format!("{e:#}"))), + None => return Err(ApiError::Internal(format!("no block of {object_hash} covers the range"))), + }; + // `parse_range` never returns an empty range, so `len - 1` is safe. + let end = offset.saturating_add(len.saturating_sub(1)); + let body = futures::stream::once(async move { Ok(first) }).chain(stream); return Ok(( StatusCode::PARTIAL_CONTENT, [ (header::CONTENT_TYPE, "application/octet-stream".to_string()), + (header::CONTENT_LENGTH, len.to_string()), ( header::CONTENT_RANGE, format!("bytes {offset}-{end}/{total}"), ), (header::ACCEPT_RANGES, "bytes".to_string()), ], - bytes, + Body::from_stream(body), ) .into_response()); } @@ -653,6 +668,113 @@ mod tests { assert_eq!(parse_range("bytes=0-", 0), None); } + async fn api() -> (Router, Storage) { + let dir = std::env::temp_dir().join(format!( + "c0mpute-storage-api-test-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::new(c0mpute_store::ChunkStore::open(&dir).await.unwrap().with_min_free_bytes(0)); + (router(StorageApiState::local(storage.clone())), storage) + } + + async fn get_with_range(app: &Router, hash: &Hash, range: Option<&str>) -> (StatusCode, HeaderMap, Vec) { + use tower::ServiceExt; + let mut req = axum::http::Request::builder().uri(format!("/storage/v1/objects/{}", hash.to_hex())); + if let Some(r) = range { + req = req.header(header::RANGE, r); + } + let resp = app.clone().oneshot(req.body(Body::empty()).unwrap()).await.unwrap(); + let (status, headers) = (resp.status(), resp.headers().clone()); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + (status, headers, body.to_vec()) + } + + fn varied(len: usize) -> Vec { + let mut state: u64 = 0x9e37_79b9_7f4a_7c15; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() + } + + #[tokio::test] + async fn a_ranged_read_streams_the_right_bytes_with_its_length_and_range() { + let (app, storage) = api().await; + // Two blocks and a bit. + let bs = c0mpute_store::DEFAULT_BLOCK_SIZE as usize; + let data = varied(bs * 2 + 321); + let m = storage.put(&data).await.unwrap(); + let total = data.len(); + let h = |headers: &HeaderMap, name: header::HeaderName| { + headers.get(name).and_then(|v| v.to_str().ok()).unwrap_or("").to_string() + }; + // Across a block boundary. + let (from, to) = (bs - 10, bs + 9); + let (status, headers, body) = get_with_range(&app, &m.object_hash, Some(&format!("bytes={from}-{to}"))).await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!(body, &data[from..=to]); + assert_eq!(h(&headers, header::CONTENT_LENGTH), "20"); + assert_eq!(h(&headers, header::CONTENT_RANGE), format!("bytes {from}-{to}/{total}")); + // `bytes=0-`: the whole object as a range, block by block. + let (status, headers, body) = get_with_range(&app, &m.object_hash, Some("bytes=0-")).await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!(body, data); + assert_eq!(h(&headers, header::CONTENT_LENGTH), total.to_string()); + assert_eq!(h(&headers, header::CONTENT_RANGE), format!("bytes 0-{}/{total}", total - 1)); + // A suffix, and an end past the object (clamped). + let (_, headers, body) = get_with_range(&app, &m.object_hash, Some("bytes=-5")).await; + assert_eq!(body, &data[total - 5..]); + assert_eq!(h(&headers, header::CONTENT_RANGE), format!("bytes {}-{}/{total}", total - 5, total - 1)); + let (_, headers, body) = get_with_range(&app, &m.object_hash, Some(&format!("bytes={}-99999999", total - 3))).await; + assert_eq!(body, &data[total - 3..]); + assert_eq!(h(&headers, header::CONTENT_LENGTH), "3"); + // Past the end: 416 with the size. + let (status, headers, _) = get_with_range(&app, &m.object_hash, Some(&format!("bytes={total}-"))).await; + assert_eq!(status, StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(h(&headers, header::CONTENT_RANGE), format!("bytes */{total}")); + // No Range: 200 with everything. + let (status, headers, body) = get_with_range(&app, &m.object_hash, None).await; + assert_eq!((status, body.len()), (StatusCode::OK, total)); + assert_eq!(h(&headers, header::CONTENT_LENGTH), total.to_string()); + } + + #[tokio::test] + async fn a_range_of_an_empty_object_is_unsatisfiable_and_the_object_itself_is_an_empty_200() { + let (app, storage) = api().await; + let m = storage.put(b"").await.unwrap(); + // No byte of it exists, so no range of it does (RFC 9110, 14.1.2): + // 416, not a range whose last byte would be offset minus one. + for range in ["bytes=-5", "bytes=0-", "bytes=0-0", "bytes=-0"] { + let (status, headers, body) = get_with_range(&app, &m.object_hash, Some(range)).await; + assert_eq!(status, StatusCode::RANGE_NOT_SATISFIABLE, "{range}"); + assert_eq!(headers.get(header::CONTENT_RANGE).unwrap(), "bytes */0", "{range}"); + assert!(body.is_empty()); + } + let (status, headers, body) = get_with_range(&app, &m.object_hash, None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers.get(header::CONTENT_LENGTH).unwrap(), "0"); + assert!(body.is_empty()); + } + + #[tokio::test] + async fn a_range_of_a_manifest_whose_blocks_do_not_add_up_is_refused_before_any_header() { + let (app, storage) = api().await; + let mut m = storage.put(&varied(5000)).await.unwrap(); + // Claims more bytes than its blocks hold. + m.original_len = 9000; + storage.write_manifest(&m).await.unwrap(); + let (status, _, _) = get_with_range(&app, &m.object_hash, Some("bytes=6000-6999")).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + } + #[test] fn budget_tracks_and_never_wraps() { let b = DiskBudget::new(Some(1000), 0); diff --git a/node/crates/c0mpute-placement/src/distributed.rs b/node/crates/c0mpute-placement/src/distributed.rs index 1fbe6d10..c3f13263 100644 --- a/node/crates/c0mpute-placement/src/distributed.rs +++ b/node/crates/c0mpute-placement/src/distributed.rs @@ -287,7 +287,15 @@ impl DistributedStorage { errors.push(format!("{peer_id}: served the wrong bytes")); continue; } - received[shard_index as usize] = Some(Shard { + // The index is the manifest's word: one past the + // block's shards is an error, not a slot to write. + let Some(slot) = received.get_mut(shard_index as usize) else { + bail!( + "block {index} of object {}: shard index {shard_index} is outside its {n} shards", + manifest.object_hash + ); + }; + *slot = Some(Shard { index: shard_index, bytes, }); @@ -305,11 +313,18 @@ impl DistributedStorage { // written with keep_local_copy, still reads. if found < k { for shard in &entry.shards { - if received[shard.index as usize].is_some() { + let Some(slot) = received.get_mut(shard.index as usize) else { + bail!( + "block {index} of object {}: shard index {} is outside its {n} shards", + manifest.object_hash, + shard.index + ); + }; + if slot.is_some() { continue; } if let Ok(bytes) = self.local.chunk_store().get(&shard.hash).await { - received[shard.index as usize] = Some(Shard { + *slot = Some(Shard { index: shard.index, bytes, }); @@ -435,6 +450,53 @@ impl BlockState { mod tests { use super::*; + #[tokio::test] + async fn a_shard_index_outside_the_block_is_an_error_not_a_panic() { + use crate::peer::{PeerCatalog, PeerInfo}; + use crate::transport::memory::MemoryTransport; + let dir = std::env::temp_dir().join(format!( + "c0mpute-placement-index-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let local = Storage::new(c0mpute_store::ChunkStore::open(&dir).await.unwrap()); + let peers: Vec = (0..20) + .map(|i| PeerInfo { + peer_id: format!("peer{i}"), + endpoint: format!("http://peer{i}.test:7780"), + reputation: 0.95, + uptime_30d: 0.995, + free_bytes: 1 << 30, + rtt_ms: 20 + i as u32, + asn: Some(64500 + i as u32), + region: None, + ip_prefix: None, + }) + .collect(); + let storage = DistributedStorage::new( + local.clone(), + Arc::new(MemoryTransport::new()), + Arc::new(RwLock::new(PeerCatalog::new(peers))), + ); + let data: Vec = (0..50_000u32).map(|i| (i.wrapping_mul(2654435761) >> 13) as u8).collect(); + // Shards fetched from peers: an index past the block's 14. + let mut placed = storage.put(&data, Tier::Standard).await.unwrap(); + assert_eq!(storage.read_block(&placed, 0).await.unwrap(), data); + for shard in placed.blocks[0].shards.iter_mut() { + shard.index = 200; + } + let err = storage.read_block(&placed, 0).await.unwrap_err().to_string(); + assert!(err.contains("outside its 14 shards"), "{err}"); + // Shards read from the local store (no host hints): the same. + let mut kept = local.put(&data).await.unwrap(); + kept.blocks[0].shards[0].index = 14; + let err = storage.read_block(&kept, 0).await.unwrap_err().to_string(); + assert!(err.contains("outside its 14 shards"), "{err}"); + } + #[test] fn write_quorum_is_k_plus_half_the_parity() { assert_eq!(DistributedStorage::write_quorum(Tier::Standard), 12); // 10 + 2 diff --git a/node/crates/c0mpute-placement/src/repair.rs b/node/crates/c0mpute-placement/src/repair.rs index 0ad5ddbb..14a249df 100644 --- a/node/crates/c0mpute-placement/src/repair.rs +++ b/node/crates/c0mpute-placement/src/repair.rs @@ -675,8 +675,16 @@ impl Repairer { warn!(peer = %peer_id, "served the wrong bytes during repair"); continue; } + // The index is the manifest's word: one past the + // block's shards is an error, not a slot to write. + let Some(slot) = received.get_mut(index as usize) else { + bail!( + "cannot repair block {}: shard index {index} is outside its {n} shards", + plan.block + ); + }; bytes_read += bytes.len() as u64; - received[index as usize] = Some(Shard { index, bytes }); + *slot = Some(Shard { index, bytes }); sources.push(peer_id); found += 1; if found == k { diff --git a/node/crates/c0mpute-store/src/lib.rs b/node/crates/c0mpute-store/src/lib.rs index 49edd6d2..d9675197 100644 --- a/node/crates/c0mpute-store/src/lib.rs +++ b/node/crates/c0mpute-store/src/lib.rs @@ -25,6 +25,7 @@ pub use storage::{ pub use tier::Tier; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -58,6 +59,35 @@ pub struct ChunkStore { probe: FreeSpaceProbe, /// Last reading: when, and bytes free then minus what was written since. free_cache: Arc>>, + /// Chunks whose reads are being watched (see [`ChunkStore::watch_reads`]). + watches: Arc)>>>, + /// How many watches are open, so a read with none pays one atomic load. + watching: Arc, +} + +/// An open watch on reads of one chunk (see [`ChunkStore::watch_reads`]). +/// Dropping it ends the watch. +pub struct ReadWatch { + seen: Arc, + watches: Arc)>>>, + watching: Arc, +} + +impl ReadWatch { + /// Whether the chunk was read from this store since the watch opened. + pub fn seen(&self) -> bool { + self.seen.load(Ordering::Relaxed) + } +} + +impl Drop for ReadWatch { + fn drop(&mut self) { + let mut w = self.watches.lock().unwrap(); + if let Some(i) = w.iter().position(|(_, s)| Arc::ptr_eq(s, &self.seen)) { + w.swap_remove(i); + self.watching.fetch_sub(1, Ordering::Relaxed); + } + } } impl std::fmt::Debug for ChunkStore { @@ -107,9 +137,38 @@ impl ChunkStore { min_free: min_free_from_env(), probe: Arc::new(free_space_at), free_cache: Arc::new(Mutex::new(None)), + watches: Arc::new(Mutex::new(Vec::new())), + watching: Arc::new(AtomicUsize::new(0)), }) } + /// Watch reads of `hash` through this store (every clone of it) until + /// the returned watch is dropped. Hosting opens one around a storage + /// challenge: a peer that answers the challenge by fetching the same + /// chunk from this node while it is being asked shows up as a read. + pub fn watch_reads(&self, hash: &Hash) -> ReadWatch { + let seen = Arc::new(AtomicBool::new(false)); + self.watches.lock().unwrap().push((*hash, seen.clone())); + self.watching.fetch_add(1, Ordering::Relaxed); + ReadWatch { + seen, + watches: self.watches.clone(), + watching: self.watching.clone(), + } + } + + /// Tell open watches that `hash` is being read (found or not). + fn note_read(&self, hash: &Hash) { + if self.watching.load(Ordering::Relaxed) == 0 { + return; + } + for (h, seen) in self.watches.lock().unwrap().iter() { + if h == hash { + seen.store(true, Ordering::Relaxed); + } + } + } + /// Set the free-disk floor in bytes (0 = none). pub fn with_min_free_bytes(mut self, bytes: u64) -> Self { self.min_free = Some(bytes); @@ -264,6 +323,7 @@ impl ChunkStore { } pub async fn read_unchecked(&self, hash: &Hash) -> Result> { + self.note_read(hash); let path = self.chunk_path(hash); fs::read(&path) .await @@ -301,6 +361,29 @@ mod tests { assert_eq!(bytes, b"hello world"); } + #[tokio::test] + async fn a_read_watch_sees_reads_of_its_chunk_through_any_clone() { + let dir = tempdir(); + let store = ChunkStore::open(&dir).await.unwrap(); + let a = store.put(b"watched").await.unwrap(); + let b = store.put(b"other").await.unwrap(); + let watch = store.watch_reads(&a); + store.get(&b).await.unwrap(); + assert!(store.has(&a).await); + assert!(!watch.seen(), "another chunk, or a presence check, is not a read"); + store.clone().get(&a).await.unwrap(); + assert!(watch.seen()); + // A read of a chunk that is not here counts too: the asker wanted it. + let missing = Hash::of(b"missing"); + let w2 = store.watch_reads(&missing); + assert!(store.get(&missing).await.is_err()); + assert!(w2.seen()); + drop(watch); + drop(w2); + assert_eq!(store.watching.load(Ordering::Relaxed), 0); + assert!(store.watches.lock().unwrap().is_empty()); + } + #[tokio::test] async fn refuses_writes_that_would_cross_the_free_disk_floor() { let dir = tempdir(); diff --git a/node/crates/c0mpute-store/src/storage.rs b/node/crates/c0mpute-store/src/storage.rs index d1746f73..0cd4652c 100644 --- a/node/crates/c0mpute-store/src/storage.rs +++ b/node/crates/c0mpute-store/src/storage.rs @@ -17,8 +17,10 @@ //! Block size scales with object size so that manifests stay small — see //! [`block_size_for`]. +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::pin::Pin; +use std::sync::{Arc, Weak}; use anyhow::{Context, Result, anyhow, bail}; use bytes::Bytes; @@ -121,6 +123,20 @@ impl ObjectManifest { let last = ((end - 1) / bs) as usize; first..(last + 1).min(self.blocks.len()) } + + /// Whether the blocks tile the object the way range reads assume: every + /// block but the last is `block_size` long, the last is no longer, and + /// the lengths add up to `original_len`. A manifest that does not would + /// make a ranged read return other bytes than the range asked for. + pub fn tiles(&self) -> bool { + let Some((last, full)) = self.blocks.split_last() else { + return false; + }; + self.block_size > 0 + && full.iter().all(|b| b.len == self.block_size) + && last.len <= self.block_size + && self.blocks.iter().map(|b| b.len as u64).sum::() == self.original_len + } } /// Wire form that accepts both v1 (flat `shards`, one implicit block) and v2 @@ -193,14 +209,83 @@ impl<'de> Deserialize<'de> for ObjectManifest { } } +/// What keeps a delete from removing a shard a write is about to depend on +/// (see [`Storage::delete`]). One per store root in a process, shared by +/// every [`Storage`] opened on it. Nothing here is on disk: after a crash +/// there are no writes in progress to protect. +#[derive(Debug, Default)] +struct Guard { + /// Held while a write commits its manifest, and for the whole of a + /// delete: while a delete looks through the manifests, none appears. + commit: tokio::sync::Mutex<()>, + /// Shards the writes in progress have produced or found already present + /// (a count each: two writes can share a shard). Their manifests are not + /// on disk yet, so this is the only record that they are needed. + writing: std::sync::Mutex>, +} + +/// The guard of each store root opened in this process. +static GUARDS: std::sync::Mutex)>> = std::sync::Mutex::new(Vec::new()); + +fn guard_for(root: &std::path::Path) -> Arc { + let root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + let mut guards = GUARDS.lock().unwrap_or_else(|e| e.into_inner()); + guards.retain(|(_, g)| g.strong_count() > 0); + if let Some(g) = guards.iter().find(|(r, _)| *r == root).and_then(|(_, g)| g.upgrade()) { + return g; + } + let g = Arc::new(Guard::default()); + guards.push((root, Arc::downgrade(&g))); + g +} + +/// The shards one write in progress has registered in [`Guard::writing`]. +/// Dropping it (the write finished, failed, or was cancelled with its +/// request) takes them out again. +struct Writing { + guard: Arc, + hashes: Vec, +} + +impl Writing { + /// Register `hashes` before they are written or relied on. + fn add(&mut self, hashes: &[Hash]) { + let mut writing = self.guard.writing.lock().unwrap_or_else(|e| e.into_inner()); + for h in hashes { + *writing.entry(*h).or_insert(0) += 1; + } + self.hashes.extend_from_slice(hashes); + } + + fn release(&mut self) { + let mut writing = self.guard.writing.lock().unwrap_or_else(|e| e.into_inner()); + for h in self.hashes.drain(..) { + if let Some(n) = writing.get_mut(&h) { + *n -= 1; + if *n == 0 { + writing.remove(&h); + } + } + } + } +} + +impl Drop for Writing { + fn drop(&mut self) { + self.release(); + } +} + #[derive(Clone, Debug)] pub struct Storage { inner: ChunkStore, + guard: Arc, } impl Storage { pub fn new(inner: ChunkStore) -> Self { - Self { inner } + let guard = guard_for(inner.root()); + Self { inner, guard } } pub fn chunk_store(&self) -> &ChunkStore { @@ -263,6 +348,12 @@ impl Storage { let mut written: Vec = Vec::new(); let mut buf: Vec = Vec::with_capacity(block_size as usize); let mut total_len: u64 = 0; + // Every shard this write depends on, new or already here, kept from + // a concurrent delete until the manifest that names them is on disk. + let mut writing = Writing { + guard: self.guard.clone(), + hashes: Vec::new(), + }; let mut stream = Box::pin(stream); let mut failed: Option = None; @@ -284,7 +375,7 @@ impl Storage { buf.extend_from_slice(&rest[..take]); rest = &rest[take..]; if buf.len() == block_size as usize { - match self.seal_block(&buf, blocks.len() as u32, k, parity).await { + match self.seal_block(&buf, blocks.len() as u32, k, parity, &mut writing).await { Ok((entry, created)) => { written.extend(created); blocks.push(entry); @@ -304,7 +395,7 @@ impl Storage { // Trailing partial block. if failed.is_none() && !buf.is_empty() { - match self.seal_block(&buf, blocks.len() as u32, k, parity).await { + match self.seal_block(&buf, blocks.len() as u32, k, parity, &mut writing).await { Ok((entry, created)) => { written.extend(created); blocks.push(entry); @@ -314,6 +405,7 @@ impl Storage { } if let Some(e) = failed { + writing.release(); self.rollback(&written).await; return Err(e); } @@ -322,6 +414,7 @@ impl Storage { if let Some(want) = expected && want != object_hash { + writing.release(); self.rollback(&written).await; bail!("object integrity failure: committed to {want} but body hashes to {object_hash}"); } @@ -329,7 +422,7 @@ impl Storage { // A zero-length object still needs one (empty) block so reads have // something to iterate. if blocks.is_empty() { - let (entry, _) = self.seal_block(&[], 0, k, parity).await?; + let (entry, _) = self.seal_block(&[], 0, k, parity, &mut writing).await?; blocks.push(entry); } @@ -343,7 +436,13 @@ impl Storage { tier, blocks, }; - self.write_manifest(&manifest).await?; + { + // Not while a delete is looking through the manifests: it would + // miss this one. The shards stay registered until it is written. + let _commit = self.guard.commit.lock().await; + self.write_manifest(&manifest).await?; + } + drop(writing); debug!( object_hash = %object_hash, blocks = manifest.blocks.len(), @@ -365,8 +464,13 @@ impl Storage { index: u32, k: usize, parity: usize, + writing: &mut Writing, ) -> Result<(BlockEntry, Vec)> { let (shards, _) = erasure::encode(plaintext, k, parity)?; + // Registered before any of them is written or found present: from + // here on a delete leaves them alone (see [`Storage::delete`]). + let hashes: Vec = shards.iter().map(|s| Hash::of(&s.bytes)).collect(); + writing.add(&hashes); let mut entries = Vec::with_capacity(shards.len()); let mut created = Vec::new(); for s in &shards { @@ -400,11 +504,74 @@ impl Storage { /// hashes. Rolling back every hash the write touched would delete the /// intact object's shards — data loss triggered by one malformed request /// from anyone who can obtain the content. + /// + /// A shard this write created may since have been taken up by another + /// object (written, or being written, with the same content), so the + /// same two checks as [`Storage::delete`] apply. async fn rollback(&self, hashes: &[Hash]) { - for h in hashes { - if let Err(e) = self.inner.delete(h).await { - warn!(hash = %h, err = %e, "rollback failed to remove shard"); + if hashes.is_empty() { + return; + } + let _commit = self.guard.commit.lock().await; + let mut doomed: HashSet = hashes.iter().copied().collect(); + self.spare_referenced(&mut doomed, None).await; + self.remove_unwritten(doomed).await; + } + + /// Take out of `doomed` every shard a manifest on this disk names, + /// leaving aside the manifest of `except` (the object being deleted). + /// Call with [`Guard::commit`] held. + async fn spare_referenced(&self, doomed: &mut HashSet, except: Option<&Hash>) { + let others = match self.list().await { + Ok(v) => v, + Err(e) => { + // Without the list nothing can be shown to be unshared. + warn!(err = %e, "cannot list manifests; keeping every shard"); + doomed.clear(); + return; + } + }; + for other in others { + if doomed.is_empty() { + return; + } + if Some(&other) == except { + continue; + } + match self.read_manifest(&other).await { + Ok(m) => { + for shard in m.blocks.iter().flat_map(|b| &b.shards) { + doomed.remove(&shard.hash); + } + } + // Unreadable as a manifest, so unreadable as an object too: + // it cannot be shown to need any shard. + Err(e) => warn!(object_hash = %other, err = %e, "skipping an unreadable manifest"), + } + } + } + + /// Remove the shards of `doomed` that no write in progress depends on. + /// Each check and removal is one step with respect to + /// [`Writing::add`]: a write that registers a shard afterwards finds it + /// gone and writes it again. + async fn remove_unwritten(&self, doomed: HashSet) { + let (guard, store) = (self.guard.clone(), self.inner.clone()); + let removal = tokio::task::spawn_blocking(move || { + for h in doomed { + let writing = guard.writing.lock().unwrap_or_else(|e| e.into_inner()); + if writing.contains_key(&h) { + continue; + } + match std::fs::remove_file(store.chunk_path(&h)) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!(hash = %h, err = %e, "failed to remove shard"), + } } + }); + if let Err(e) = removal.await { + warn!(err = %e, "shard removal did not finish"); } } @@ -449,6 +616,48 @@ impl Storage { )) } + /// Stream `[offset, offset + len)` of an object (clamped to its end), + /// reconstructing one block at a time: a range of a huge object costs + /// one block of memory, like [`Storage::read_stream`], not the range. + /// + /// The manifest should [tile](ObjectManifest::tiles); a block shorter + /// than its place in the range ends the stream with an error. + pub fn read_range_stream( + &self, + manifest: ObjectManifest, + offset: u64, + len: u64, + ) -> Pin> + Send>> { + let end = offset.saturating_add(len).min(manifest.original_len); + let range = manifest.blocks_for_range(offset, end.saturating_sub(offset)); + let storage = self.clone(); + Box::pin(futures::stream::unfold( + (storage, manifest, range.start, range.end), + move |(storage, manifest, i, stop)| async move { + if i >= stop { + return None; + } + let item = storage.read_block(&manifest, i).await.and_then(|block| { + let bs = manifest.block_size as u64; + let block_start = i as u64 * bs; + let from = offset.max(block_start) - block_start; + let to = end.min(block_start + bs) - block_start; + if to > block.len() as u64 || from > to { + bail!( + "block {i} of object {} is {} bytes, shorter than its place in the object", + manifest.object_hash, + block.len() + ); + } + Ok(Bytes::from(block).slice(from as usize..to as usize)) + }); + // Nothing after a failed block: the bytes would be misplaced. + let next = if item.is_ok() { i + 1 } else { stop }; + Some((item, (storage, manifest, next, stop))) + }, + )) + } + /// Read `[offset, offset + len)` of an object, touching only the blocks /// that range covers. pub async fn get_range(&self, object_hash: &Hash, offset: u64, len: u64) -> Result> { @@ -494,11 +703,22 @@ impl Storage { for shard in &entry.shards { match self.inner.get(&shard.hash).await { Ok(bytes) => { - received[shard.index as usize] = Some(Shard { + // The index is the manifest's word: one past the block's + // shards is an error, not a slot to write. + let Some(slot) = received.get_mut(shard.index as usize) else { + bail!( + "block {index} of object {}: shard index {} is outside its {n} shards", + manifest.object_hash, + shard.index + ); + }; + if slot.is_none() { + found += 1; + } + *slot = Some(Shard { index: shard.index, bytes, }); - found += 1; } Err(e) => { warn!( @@ -573,17 +793,34 @@ impl Storage { Ok(out) } - /// Delete an object's manifest and every shard it points at. + /// Delete an object's manifest and the shards only it points at. + /// + /// Shards are content-addressed, so two objects with a block in common + /// share that block's shards (see [`Storage::rollback`]): a shard goes + /// only when no other manifest on this disk names it and no write in + /// progress depends on it. The other manifests are read to find out, + /// which makes a delete cost one pass over them; there is no reference + /// count to keep right, and none to rebuild or to be wrong after a + /// crash. + /// + /// The shards go first and the manifest last. A crash in between leaves + /// the manifest of a half-deleted object, never shards nothing names: + /// deleting it again finishes the job, and no other object has lost + /// anything. pub async fn delete(&self, object_hash: &Hash) -> Result<()> { + let _commit = self.guard.commit.lock().await; let manifest = match self.read_manifest(object_hash).await { Ok(m) => m, Err(_) => return Ok(()), }; - for block in &manifest.blocks { - for shard in &block.shards { - let _ = self.inner.delete(&shard.hash).await; - } - } + let mut doomed: HashSet = manifest + .blocks + .iter() + .flat_map(|b| &b.shards) + .map(|s| s.hash) + .collect(); + self.spare_referenced(&mut doomed, Some(object_hash)).await; + self.remove_unwritten(doomed).await; let _ = fs::remove_file(self.manifest_path(object_hash)).await; Ok(()) } @@ -964,6 +1201,183 @@ mod tests { assert!(s.get(&m.object_hash).await.is_err()); } + #[tokio::test] + async fn deleting_one_of_two_objects_that_share_a_block_keeps_the_other_intact() { + let s = store().await; + // Two objects with the same first block and different second ones. + let common = varied(DEFAULT_BLOCK_SIZE as usize); + let mut a = common.clone(); + a.extend_from_slice(b"the tail of a"); + let mut b = common.clone(); + b.extend_from_slice(b"and b ends another way"); + let ma = s.put(&a).await.unwrap(); + let mb = s.put(&b).await.unwrap(); + let shared: Vec = ma.blocks[0].shards.iter().map(|x| x.hash).collect(); + assert_eq!(shared, mb.blocks[0].shards.iter().map(|x| x.hash).collect::>()); + assert_ne!(ma.blocks[1].hash, mb.blocks[1].hash); + + s.delete(&ma.object_hash).await.unwrap(); + assert!(!s.has(&ma.object_hash).await); + for h in &shared { + assert!(s.inner.has(h).await, "a shard b still needs was deleted"); + } + // What only a used is gone... + for shard in &ma.blocks[1].shards { + assert!(!s.inner.has(&shard.hash).await); + } + // ...and b reads back whole, with every shard in place. + assert_eq!(s.get(&mb.object_hash).await.unwrap(), b); + for shard in mb.blocks.iter().flat_map(|x| &x.shards) { + assert!(s.inner.has(&shard.hash).await); + } + + // Deleting both removes the shared shards too: nothing is left. + s.delete(&mb.object_hash).await.unwrap(); + let left = walk(&s.inner.root().join("shards")).into_iter().filter(|p| p.is_file()).count(); + assert_eq!(left, 0, "shards nothing names were left behind"); + assert!(s.list().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_delete_leaves_alone_shards_a_write_in_progress_depends_on() { + let s = store().await; + let data = varied(20_000); + let m = s.put(&data).await.unwrap(); + let hashes: Vec = m.blocks[0].shards.iter().map(|x| x.hash).collect(); + // Another write of the same content has sealed its block (the + // shards were already here) but not written its manifest yet. It + // goes through a second handle on the same store, as another part + // of the process would. + let other = Storage::new(s.inner.clone()); + let mut writing = Writing { + guard: other.guard.clone(), + hashes: Vec::new(), + }; + writing.add(&hashes); + s.delete(&m.object_hash).await.unwrap(); + assert!(!s.has(&m.object_hash).await); + for h in &hashes { + assert!(s.inner.has(h).await, "deleted under a write in progress"); + } + // That write commits: its object is whole. + other.write_manifest(&m).await.unwrap(); + drop(writing); + assert_eq!(other.get(&m.object_hash).await.unwrap(), data); + // With no write in progress the shards go with the object. + s.delete(&m.object_hash).await.unwrap(); + for h in &hashes { + assert!(!s.inner.has(h).await); + } + assert!(s.guard.writing.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_failed_write_does_not_take_back_a_shard_another_object_took_up() { + let s = store().await; + let block = varied(DEFAULT_BLOCK_SIZE as usize); + // A write that will fail (wrong committed hash) seals its first + // block, creating the shards, and then waits for more of its body. + let (sealed_tx, sealed_rx) = tokio::sync::oneshot::channel::<()>(); + let (go_tx, go_rx) = tokio::sync::oneshot::channel::<()>(); + let first = Bytes::from(block.clone()); + let stream = futures::stream::once(async move { Ok(first) }).chain(futures::stream::once(async move { + let _ = sealed_tx.send(()); + let _ = go_rx.await; + Ok(Bytes::from_static(b"the rest of a body that does not match")) + })); + let failing = s.put_stream(stream, Some(Hash::of(b"not the data")), Tier::Standard, None); + // Meanwhile the same block is stored for real: its shards are the + // ones the failing write created. + let good = async { + sealed_rx.await.unwrap(); + let m = s.put(&block).await.unwrap(); + go_tx.send(()).unwrap(); + m + }; + let (failed, good) = tokio::join!(failing, good); + assert!(failed.unwrap_err().to_string().contains("integrity failure")); + // The failed write took back only what nothing else names. + assert_eq!(s.get(&good.object_hash).await.unwrap(), block); + let kept: HashSet = good.blocks.iter().flat_map(|b| &b.shards).map(|x| x.hash).collect(); + let left = walk(&s.inner.root().join("shards")).into_iter().filter(|p| p.is_file()).count(); + assert_eq!(left, kept.len(), "exactly the stored object's shards remain"); + } + + #[tokio::test] + async fn a_shard_index_outside_the_block_is_an_error_not_a_panic() { + let s = store().await; + let data = varied(5_000); + let mut m = s.put(&data).await.unwrap(); + m.blocks[0].shards[3].index = 200; + let err = s.read_block(&m, 0).await.unwrap_err().to_string(); + assert!(err.contains("outside its 14 shards"), "{err}"); + // The same slot named twice is one shard, not two. + let mut m = s.read_manifest(&m.object_hash).await.unwrap(); + for shard in m.blocks[0].shards.iter_mut() { + shard.index = 0; + } + let err = s.read_block(&m, 0).await.unwrap_err().to_string(); + assert!(err.contains("need at least"), "{err}"); + } + + #[tokio::test] + async fn a_range_streams_block_by_block_and_matches_the_bytes() { + let s = store().await; + let bs = DEFAULT_BLOCK_SIZE as u64; + let data = varied(bs as usize * 2 + 1000); + let m = s.put(&data).await.unwrap(); + assert!(m.tiles()); + let read = |offset: u64, len: u64| { + let (s, m) = (s.clone(), m.clone()); + async move { + let mut chunks = Vec::new(); + let mut stream = s.read_range_stream(m, offset, len); + while let Some(c) = stream.next().await { + chunks.push(c.unwrap()); + } + chunks + } + }; + // Across all three blocks: three pieces, none bigger than a block. + let chunks = read(100, bs * 2 + 500).await; + assert_eq!(chunks.len(), 3); + assert!(chunks.iter().all(|c| c.len() as u64 <= bs)); + assert_eq!(chunks.concat(), &data[100..(bs * 2 + 600) as usize]); + // Inside one block: one piece of exactly the range. + let chunks = read(bs + 7, 10).await; + assert_eq!(chunks.concat(), &data[(bs + 7) as usize..(bs + 17) as usize]); + assert_eq!(chunks.len(), 1); + // The whole object, a range past the end (clamped), and nothing. + assert_eq!(read(0, u64::MAX).await.concat(), data); + assert_eq!(read(bs * 2 + 900, 5000).await.concat(), &data[(bs * 2 + 900) as usize..]); + assert!(read(data.len() as u64, 10).await.is_empty()); + assert!(read(5, 0).await.is_empty()); + } + + #[tokio::test] + async fn a_range_over_a_manifest_that_does_not_tile_fails_instead_of_misplacing_bytes() { + let s = store().await; + let bs = DEFAULT_BLOCK_SIZE as usize; + let data = varied(bs + 100); + let good = s.put(&data).await.unwrap(); + // A manifest whose first block is the short one. + let mut bad = good.clone(); + bad.blocks.swap(0, 1); + assert!(!bad.tiles()); + let mut stream = s.read_range_stream(bad, 0, bs as u64 + 100); + assert!(stream.next().await.unwrap().is_err()); + assert!(stream.next().await.is_none(), "nothing follows a failed block"); + // Lengths that do not add up, and no blocks at all. + let mut bad = good.clone(); + bad.original_len += 1; + assert!(!bad.tiles()); + let mut bad = good.clone(); + bad.blocks.clear(); + assert!(!bad.tiles()); + // An empty object tiles (one empty block). + assert!(s.put(b"").await.unwrap().tiles()); + } + #[tokio::test] async fn delete_removes_manifest_and_shards() { let s = store().await; diff --git a/package.json b/package.json index 5fe176d8..10796b21 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.3.39", + "version": "0.3.65", "private": true, "workspaces": [ "apps/*", diff --git a/packages/captcha/package.json b/packages/captcha/package.json index 9961bde0..52b286ae 100644 --- a/packages/captcha/package.json +++ b/packages/captcha/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/c0mpute-captcha", - "version": "0.3.39", + "version": "0.3.65", "description": "Solve image, slider, gap, click-text, audio and math captchas on the c0mpute network: library (fetch().then(solveCaptcha)), CLI and stdio MCP server.", "type": "module", "license": "MIT", diff --git a/packages/shared/package.json b/packages/shared/package.json index 303ffda6..c6065bfb 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.3.39", + "version": "0.3.65", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 0673a612..b000c5f9 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index 3509d31b..36bdd7c4 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -38,7 +38,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "partial" diff --git a/plugins/apps/module.toml b/plugins/apps/module.toml index cf4b976f..b0c5ae01 100644 --- a/plugins/apps/module.toml +++ b/plugins/apps/module.toml @@ -18,7 +18,7 @@ [module] id = "apps" name = "Apps — backends with 3x redundancy" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/captcha/module.toml b/plugins/captcha/module.toml index 6310337e..cb1067be 100644 --- a/plugins/captcha/module.toml +++ b/plugins/captcha/module.toml @@ -22,7 +22,7 @@ [module] id = "captcha" name = "Captcha — image, slider, audio and math solving" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index 89686be9..66ea5cf3 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "available" diff --git a/plugins/containers/module.toml b/plugins/containers/module.toml index 47330bf3..9be47d1b 100644 --- a/plugins/containers/module.toml +++ b/plugins/containers/module.toml @@ -15,7 +15,7 @@ [module] id = "containers" name = "Containers — remote docker builds and jobs" -version = "0.3.38" +version = "0.3.65" kind = "workload" category = "developer" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index 55044152..b868e817 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "data" status = "available" diff --git a/plugins/crowdgpt/module.toml b/plugins/crowdgpt/module.toml index 3d9f56f2..0e064bfc 100644 --- a/plugins/crowdgpt/module.toml +++ b/plugins/crowdgpt/module.toml @@ -43,7 +43,7 @@ [module] id = "crowdgpt" name = "CrowdGPT: donate idle compute" -version = "0.3.39" +version = "0.3.65" kind = "donation" category = "ai" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index aba48802..62824e5a 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -28,7 +28,7 @@ [module] id = "ctv-ads" name = "CTV Ads — VAST and VMAP from every gateway" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "partial" diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index 32666f83..99b78617 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index 4bd6742c..6ff7c629 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -19,7 +19,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index 351534f7..4ecb74cd 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -55,7 +55,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/git/module.toml b/plugins/git/module.toml index 955a0a6d..d1bd2072 100644 --- a/plugins/git/module.toml +++ b/plugins/git/module.toml @@ -32,7 +32,7 @@ [module] id = "git" name = "Git — p2p repo hosting" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "data" status = "partial" diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index 812b9103..cceac060 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -30,7 +30,7 @@ [module] id = "hosting" name = "Hosting — p2p static sites" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "data" status = "available" diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index 1e6e38ce..ce4cf3cb 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -22,7 +22,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index a96e5b1b..7b4f6cda 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -34,7 +34,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "media" status = "available" diff --git a/plugins/music/module.toml b/plugins/music/module.toml index 5eb11c66..e6e995aa 100644 --- a/plugins/music/module.toml +++ b/plugins/music/module.toml @@ -15,7 +15,7 @@ [module] id = "music" name = "Music — song generation (ACE-Step 1.5)" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index 09a6cc2a..fa579030 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -19,7 +19,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index 4629de2e..76e8aaa4 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -31,7 +31,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — dynamic ad insertion" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "partial" diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index ec8b7db1..ec521b38 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "media" status = "available" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index cadbdb1f..808e2bb0 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "media" status = "available" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index 32d6b683..28735f1c 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -33,7 +33,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "media" status = "available" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index b6fce53d..c16b4c8a 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -21,7 +21,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "available" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index be91366c..5d3c5e35 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "data" status = "available" diff --git a/plugins/threatcrush/module.toml b/plugins/threatcrush/module.toml index 4150d685..87e15c5d 100644 --- a/plugins/threatcrush/module.toml +++ b/plugins/threatcrush/module.toml @@ -19,7 +19,7 @@ [module] id = "threatcrush" name = "ThreatCrush — protect this node" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "infrastructure" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 778e7b94..4adf235c 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -55,7 +55,7 @@ [module] id = "train" name = "Fine-tune — LoRA training" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index 39c2b9dc..d8301299 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index 43b3b635..1846daf3 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index 6fea2f74..8b15141b 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -65,7 +65,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, Wan 2.1 1.3B on 8 GiB or CPU, LTX-2.3 with audio)" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 9931fbd4..3ba95531 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index f630edbd..b245c1ce 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index 63b88e9a..5608e328 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.3.39" +version = "0.3.65" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/x402/module.toml b/plugins/x402/module.toml index 7a2b3f20..35e502e7 100644 --- a/plugins/x402/module.toml +++ b/plugins/x402/module.toml @@ -21,7 +21,7 @@ [module] id = "x402" name = "x402 — charge bots for hosted files" -version = "0.3.39" +version = "0.3.65" kind = "service" category = "applications" status = "available" diff --git a/scripts/install.sh b/scripts/install.sh index 8984afe0..7dcd97b8 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2,11 +2,11 @@ # c0mpute.com installer. Served at https://c0mpute.com/install.sh. # # Installs the c0mpute v1 stack idempotently: -# mise — runtime version manager (skipped if present) -# bun — JS runtime for TUI / JS-flavoured plugins (skipped if present) -# c0mpute — this repo (https://github.com/profullstack/c0mpute) -# coinpay — routed via https://c0mpute.com/plugins/coinpay/install.sh -# infernet — routed via https://c0mpute.com/plugins/infernet/install.sh +# c0mpute — this repo (https://github.com/profullstack/c0mpute), signed release +# coinpay — plugins/coinpay/install.sh from the signed c0mpute-plugins.tar.gz +# infernet — plugins/infernet/install.sh from the signed c0mpute-plugins.tar.gz +# tui — c0mpute-tui from the signed c0mpute-tui.tar.gz +# bun — pinned JS runtime for the TUI, checked by sha256 (skipped if present) # # Re-running upgrades in place. Each step skips when already installed # unless --force is passed. @@ -50,13 +50,25 @@ C0MPUTE_VERSION="${C0MPUTE_VERSION:-latest}" C0MPUTE_HOME="${C0MPUTE_HOME:-$HOME/.c0mpute}" RELEASE_BASE="${C0MPUTE_RELEASE_BASE:-https://c0mpute.com/releases}" -# Route plugin installs through c0mpute.com so each wrapper can do its -# own error handling; the wrapper at https://c0mpute.com/plugins//install.sh -# is the in-repo plugins//install.sh and chains to upstream itself. +# Plugin installers (the in-repo plugins//install.sh, which chain to +# upstream themselves) run from the release's signed c0mpute-plugins.tar.gz. +# These URLs serve the same files unsigned and are only used for a release +# from before that archive existed (see UNSIGNED_EXTRAS_UNTIL). COINPAY_INSTALL_URL="${COINPAY_INSTALL_URL:-https://c0mpute.com/plugins/coinpay/install.sh}" INFERNET_INSTALL_URL="${INFERNET_INSTALL_URL:-https://c0mpute.com/plugins/infernet/install.sh}" TRANSCODE_INSTALL_URL="${TRANSCODE_INSTALL_URL:-https://c0mpute.com/plugins/transcode/install.sh}" +# Last release WITHOUT the signed extras (c0mpute-plugins.tar.gz, +# c0mpute-tui.tar.gz, AppImage .minisig). Only a release this old may fall back +# to the unverified downloads when an extra is absent; on anything newer a +# missing extra is an error, so hiding the asset cannot force the fallback. +UNSIGNED_EXTRAS_UNTIL="0.3.64" +# The release everything in this run comes from (v): the version in the +# binary's signed comment, or the already-installed binary's. "" = unknown. +RELEASE_TAG="" +SIGNED_VERSION="" +SCRATCH="" + INSTALL_C0MPUTE=1 INSTALL_COINPAY=1 INSTALL_INFERNET=1 @@ -74,8 +86,8 @@ warn() { printf '\033[1;33m!\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; } ok() { printf '\033[1;32m✓\033[0m %s\n' "$*"; } -# Privilege escalation helper for system-wide steps (the /usr/local/bin -# symlink + /etc/profile.d hook). Empty when already root or when sudo isn't +# Privilege escalation helper for system-wide steps (the /etc/profile.d hook, +# apt, a --system-service unit). Empty when already root or when sudo isn't # available; callers try a direct write first and fall back to $SUDO. if [ "$(id -u 2>/dev/null || echo 0)" != "0" ] && command -v sudo >/dev/null 2>&1; then SUDO="sudo" @@ -165,57 +177,105 @@ shell_rc_for() { } # ──────────────────────────────────────────────────────────────────────── -# Idempotent mise + bun install +# bun (pinned) # ──────────────────────────────────────────────────────────────────────── # -# Some plugins are JS/TS-runtime-based (the Bun-built TUI, Node-based -# infernet variants). mise gives us a single tool to manage the runtime -# versions; bun is the JS runtime we standardise on. Both install -# user-locally — no sudo, no system package manager. +# The TUI (c0mpute-tui) is TypeScript run through bun; nothing else on an +# installed node needs a JS runtime, so bun is only installed with the TUI. +# We download the pinned upstream release zip for this OS/arch (not bun's +# installer script) and check it against a hard-coded sha256 before anything +# in it runs, the same way minisign is pinned below. User-local, no sudo. +# +# mise is NOT installed any more: it is the contributor toolchain manager +# (.mise.toml, scripts/dev-setup.sh), not something a node needs at runtime. +# +# Hashes: upstream's SHASUMS256.txt for bun-v1.3.14 +# (github.com/oven-sh/bun/releases), each re-computed from the downloaded zip. +# Bump the version and every hash together. +BUN_PIN_VERSION="1.3.14" +bun_pin_sha256() { + case "$1" in + linux-x64) echo 951ee2aee855f08595aeec6225226a298d3fea83a3dcd6465c09cbccdf7e848f ;; + linux-x64-baseline) echo a063908ae08b7852ca10939bbdc6ceed3ddabce8fb9402dce83d65d73b36e6c7 ;; + linux-x64-musl) echo 14bd9aedeebf1dba67e8def9531c89bc989ecfdf1de42e5bfcaf1b8cd9294719 ;; + linux-x64-musl-baseline) echo 56a7d6806cf155536c0178f0ea5fbd098e684fa509ebdb4fc0a7e19fb65382dc ;; + linux-aarch64) echo a27ffb63a8310375836e0d6f668ae17fa8d8d18b88c37c821c65331973a19a3b ;; + linux-aarch64-musl) echo b98e0ad3625c5c00d1d5b5ff55605c7adddbfae151861e68ade57b2d3b8703bb ;; + darwin-x64) echo 4183df3374623e5bab315c547cfa0974533cd457d86b73b639f7a87974cd6633 ;; + darwin-x64-baseline) echo 3e35ad6f53971a9834bf9e6786e2adf72b5f1921cc9a9c5fde073d2972944076 ;; + darwin-aarch64) echo d8b96221828ad6f97ac7ac0ab7e95872341af763001e8803e8267652c2652620 ;; + esac +} -ensure_mise() { - if command -v mise >/dev/null 2>&1; then - return 0 - fi - say "installing mise" - if ! command -v curl >/dev/null 2>&1; then - warn "curl missing; skipping mise install" - return 1 - fi - curl -fsSL https://mise.run | sh >/dev/null 2>&1 || { - warn "mise install failed" - return 1 - } - if [ -x "$HOME/.local/bin/mise" ]; then - PATH="$HOME/.local/bin:$PATH" - export PATH - ok "mise installed" - fi +# The bun release build for platform $1 on this machine (prints nothing: none). +bun_target() { + case "$1" in + linux-x86_64) bt="linux-x64" ;; + linux-aarch64) bt="linux-aarch64" ;; + darwin-aarch64) bt="darwin-aarch64" ;; + darwin-x86_64) + bt="darwin-x64" + # An x86_64 shell under Rosetta: the machine is Apple silicon. + if [ "$(sysctl -n sysctl.proc_translated 2>/dev/null || true)" = "1" ]; then bt="darwin-aarch64"; fi + ;; + *) return 0 ;; + esac + case "$bt" in + linux-*) + # musl libc (Alpine) needs the musl build. + if [ -f /etc/alpine-release ] || { ldd --version 2>&1 || true; } | grep -qi musl; then + bt="$bt-musl" + fi + ;; + esac + # x64 CPUs without AVX2 need the baseline build (it runs on every x64). + case "$bt" in + linux-x64*) grep -qw avx2 /proc/cpuinfo 2>/dev/null || bt="$bt-baseline" ;; + darwin-x64) [ "$(sysctl -n hw.optional.avx2_0 2>/dev/null || true)" = "1" ] || bt="$bt-baseline" ;; + esac + printf '%s' "$bt" } -ensure_bun() { +ensure_bun() { # if command -v bun >/dev/null 2>&1; then return 0 fi - if command -v mise >/dev/null 2>&1; then - say "installing bun via mise" - mise use --global bun@latest >/dev/null 2>&1 || mise install bun@latest >/dev/null 2>&1 || true - if [ -x "$HOME/.local/share/mise/installs/bun/latest/bin/bun" ] \ - || command -v bun >/dev/null 2>&1; then - ok "bun installed (via mise)" - return 0 - fi - fi - say "installing bun" - curl -fsSL https://bun.sh/install | bash >/dev/null 2>&1 || { - warn "bun install failed" - return 1 - } if [ -x "$HOME/.bun/bin/bun" ]; then PATH="$HOME/.bun/bin:$PATH" export PATH - ok "bun installed" + return 0 + fi + bun_manual="install bun ${BUN_PIN_VERSION} yourself (https://bun.sh/docs/installation), then run: c0mpute tui" + bt=$(bun_target "$1") + bun_want=$(bun_pin_sha256 "$bt") + if [ -z "$bt" ] || [ -z "$bun_want" ]; then + warn "no pinned bun build for ${bt:-$1}; $bun_manual" + return 1 fi + if ! command -v unzip >/dev/null 2>&1; then + warn "unzip is needed to unpack bun: install unzip and re-run, or $bun_manual" + return 1 + fi + say "installing bun ${BUN_PIN_VERSION} (${bt}, pinned by sha256)" + bun_dir="$SCRATCH/bun" + mkdir -p "$bun_dir" + curl -fsSL -o "$bun_dir/bun.zip" \ + "${C0MPUTE_BUN_BASE:-https://github.com/oven-sh/bun/releases/download}/bun-v${BUN_PIN_VERSION}/bun-${bt}.zip" 2>/dev/null \ + || { warn "could not download bun; $bun_manual"; return 1; } + bun_got=$(c0mpute_sha256 "$bun_dir/bun.zip") || { warn "no sha256 tool (sha256sum/shasum/openssl) to check bun"; return 1; } + if [ "$bun_got" != "$bun_want" ]; then + rm -f "$bun_dir/bun.zip" + warn "bun download has sha256 ${bun_got}, expected ${bun_want}; not installing it" + return 1 + fi + unzip -q -o "$bun_dir/bun.zip" "bun-${bt}/bun" -d "$bun_dir" >/dev/null 2>&1 \ + || { warn "could not unzip bun"; return 1; } + mkdir -p "$HOME/.bun/bin" + { mv -f "$bun_dir/bun-${bt}/bun" "$HOME/.bun/bin/bun" && chmod 755 "$HOME/.bun/bin/bun"; } \ + || { warn "could not install bun into $HOME/.bun/bin"; return 1; } + PATH="$HOME/.bun/bin:$PATH" + export PATH + ok "bun ${BUN_PIN_VERSION} installed → $HOME/.bun/bin/bun" } # ──────────────────────────────────────────────────────────────────────── @@ -279,9 +339,117 @@ c0mpute_minisign() { printf '%s\n' "$2/minisign/$ms_member" } +# True when dotted version $1 is lower than $2 (a leading v is ignored). +version_lt() { + vl_a="${1#v}"; vl_b="${2#v}" + [ "$vl_a" != "$vl_b" ] || return 1 + vl_low=$(printf '%s\n%s\n' "$vl_a" "$vl_b" | sort -t. -k1,1n -k2,2n -k3,3n | head -n 1) + [ "$vl_low" = "$vl_a" ] +} + +# x.y.z of the c0mpute already on this machine ("" when none, or it won't run). +installed_c0mpute_version() { + icv_bin="$C0MPUTE_HOME/bin/c0mpute" + [ -x "$icv_bin" ] || icv_bin=$(command -v c0mpute 2>/dev/null || true) + [ -n "$icv_bin" ] || return 0 + "$icv_bin" --version 2>/dev/null \ + | sed -n 's/^c0mpute *v\{0,1\}\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -n 1 +} + +# Verify a downloaded release asset against its minisign signature. +# $1 platform $2 the file ($2.minisig next to it) $3 its name on the release +# $4 the release tag the signature must be for ("" = any: the `latest` URL) +# Fails closed: no verifier, a bad signature, or a valid signature for another +# file or release all abort the install. Sets SIGNED_VERSION to the release tag +# in the signed (trusted) comment, `timestamp: version: file:`. +c0mpute_verify() { + if [ -z "${MS_BIN:-}" ] || [ ! -x "$MS_BIN" ]; then + MS_BIN=$(c0mpute_minisign "$1" "$SCRATCH") \ + || die "cannot verify the download: install minisign (brew install minisign / apt install minisign / https://jedisct1.github.io/minisign/) and re-run" + fi + ms_out=$("$MS_BIN" -V -P "$C0MPUTE_PUBKEY" -m "$2" -x "$2.minisig" 2>&1) \ + || die "signature verification FAILED for $3; not installing it" + ms_comment=$(printf '%s\n' "$ms_out" | sed -n 's/^Trusted comment: //p' | head -n 1) + [ -n "$ms_comment" ] || ms_comment="$ms_out" + # The signed comment names the file and release, so a validly signed + # artifact for another platform or release can't be swapped in. + case " $ms_comment " in + *" file:$3 "*) ;; + *) die "signature for $3 is valid but signs a different file; not installing it" ;; + esac + if [ -n "$4" ]; then + case " $ms_comment " in + *" version:$4 "*) ;; + *) die "signature for $3 is valid but not for release $4; not installing it" ;; + esac + fi + SIGNED_VERSION=$(printf '%s\n' "$ms_comment" | tr ' ' '\n' | sed -n 's/^version://p' | head -n 1) + case "$SIGNED_VERSION" in + *[!A-Za-z0-9._-]*) die "signature for $3 carries a malformed release version; not installing it" ;; + esac +} + +# True when the release being installed is from before the signed extras +# existed, so an absent one is expected rather than suspicious. An unknown +# release is not: only C0MPUTE_ALLOW_UNSIGNED=1 gets the old behaviour then. +legacy_release() { + if [ -z "$RELEASE_TAG" ]; then + [ "${C0MPUTE_ALLOW_UNSIGNED:-0}" = "1" ] + return + fi + ! version_lt "$UNSIGNED_EXTRAS_UNTIL" "$RELEASE_TAG" +} + +# Download release asset $2 for platform $1 from the same release as the +# binary and verify it exactly like the binary. Sets ASSET_PATH. Returns +# 0 downloaded and verified +# 2 the release does not have it (HTTP 404) and predates signed extras: the +# only case where a caller may fall back to the old unverified download +# 1 could not be fetched, or missing from a release that must have it +# An asset that IS there with a missing or bad signature aborts the install. +fetch_signed_asset() { + fa_url="${RELEASE_BASE}/${RELEASE_TAG:-latest}/$2" + mkdir -p "$SCRATCH/assets" + ASSET_PATH="$SCRATCH/assets/$2" + fa_code=$(curl -sSL -o "$ASSET_PATH" -w '%{http_code}' "$fa_url" 2>/dev/null || echo "000") + if [ "$fa_code" = "404" ]; then + rm -f "$ASSET_PATH" + if legacy_release; then return 2; fi + warn "$2 is missing from the ${RELEASE_TAG:-latest} release, which should carry it (${fa_url})" + return 1 + fi + if [ "$fa_code" != "200" ]; then + rm -f "$ASSET_PATH" + warn "could not download ${fa_url} (HTTP ${fa_code})" + return 1 + fi + fa_sig=$(curl -sSL -o "$ASSET_PATH.minisig" -w '%{http_code}' "${fa_url}.minisig" 2>/dev/null || echo "000") + if [ "$fa_sig" != "200" ]; then + rm -f "$ASSET_PATH.minisig" + if [ "${C0MPUTE_ALLOW_UNSIGNED:-0}" = "1" ]; then + warn "no signature at ${fa_url}.minisig (HTTP ${fa_sig}); using $2 UNVERIFIED because C0MPUTE_ALLOW_UNSIGNED=1" + return 0 + fi + die "no signature at ${fa_url}.minisig (HTTP ${fa_sig}); refusing to use an unverified $2 (C0MPUTE_ALLOW_UNSIGNED=1 overrides)" + fi + say "verifying signature for $2" + c0mpute_verify "$1" "$ASSET_PATH" "$2" "$RELEASE_TAG" + ok "signature verified: $2 (${SIGNED_VERSION})" +} + +# The checkout this script was run from, when it was run as a file from one +# (sh scripts/install.sh). Empty under `curl | sh`: there $0 is the shell, and +# a path relative to it would be whatever directory you happen to be in. +repo_root() { + case "$0" in */install.sh) ;; *) return 0 ;; esac + rr=$(cd "$(dirname "$0")/.." 2>/dev/null && pwd) || return 0 + if [ -f "$rr/scripts/install.sh" ] && [ -d "$rr/plugins" ]; then printf '%s' "$rr"; fi +} + install_c0mpute() { platform="$1" target="$C0MPUTE_HOME/bin/c0mpute" + have_ver=$(installed_c0mpute_version) if [ -x "$target" ] && [ "$FORCE" -eq 0 ]; then # Verify the existing binary actually runs. v0.1.0's glibc-linked # build fails on systems with older glibc; v0.1.1+ uses musl. If @@ -289,6 +457,8 @@ install_c0mpute() { # and force reinstall. if "$target" version >/dev/null 2>&1; then say "c0mpute already installed at $target (use --force to reinstall)" + # Plugins and the TUI come from the installed binary's own release. + if [ -n "$have_ver" ]; then RELEASE_TAG="v$have_ver"; fi return 0 fi warn "existing $target appears broken (won't run); reinstalling" @@ -297,7 +467,8 @@ install_c0mpute() { artifact="c0mpute-${platform}.tar.gz" url="${RELEASE_BASE}/${C0MPUTE_VERSION}/${artifact}" sig_url="${url}.minisig" - tmp=$(mktemp -d) + tmp="$SCRATCH/c0mpute" + mkdir -p "$tmp" say "downloading c0mpute ${C0MPUTE_VERSION}" http_code=$(curl -sSL -o "$tmp/$artifact" -w '%{http_code}' "$url" 2>/dev/null || echo "000") @@ -347,35 +518,29 @@ EOF if [ "$sig_code" != "200" ]; then if [ "${C0MPUTE_ALLOW_UNSIGNED:-0}" = "1" ]; then warn "no signature at ${sig_url} (HTTP ${sig_code}); installing UNVERIFIED because C0MPUTE_ALLOW_UNSIGNED=1" + case "$url" in */latest/*) ;; *) RELEASE_TAG="${url%/*}"; RELEASE_TAG="${RELEASE_TAG##*/}" ;; esac else rm -rf "$tmp" die "no signature at ${sig_url} (HTTP ${sig_code}); refusing to install an unverified binary (C0MPUTE_ALLOW_UNSIGNED=1 overrides)" fi else say "verifying signature for c0mpute" - if ! ms_bin=$(c0mpute_minisign "$platform" "$tmp"); then - rm -rf "$tmp" - die "cannot verify the download: install minisign (brew install minisign / apt install minisign / https://jedisct1.github.io/minisign/) and re-run" - fi - if ! ms_out=$("$ms_bin" -V -P "$C0MPUTE_PUBKEY" -m "$tmp/$artifact" -x "$tmp/$artifact.minisig" 2>&1); then - rm -rf "$tmp" - die "signature verification FAILED for ${artifact} from ${url}; not installing it" - fi - # The signed (trusted) comment names the file and release, so a validly - # signed artifact for another platform or release can't be swapped in. + # A pinned release (or one found by walking back) must be signed for that + # tag; through the `latest` URL any release's signature is accepted here + # and its signed version is checked below. ms_tag="" case "$url" in */latest/*) ;; *) ms_tag="${url%/*}"; ms_tag="${ms_tag##*/}" ;; esac - case "$ms_out" in - *"file:${artifact}"*) ;; - *) rm -rf "$tmp"; die "signature for ${artifact} is valid but signs a different file; not installing it" ;; - esac - if [ -n "$ms_tag" ]; then - case "$ms_out" in - *"version:${ms_tag} "*) ;; - *) rm -rf "$tmp"; die "signature for ${artifact} is valid but not for release ${ms_tag}; not installing it" ;; - esac - fi + c0mpute_verify "$platform" "$tmp/$artifact" "$artifact" "$ms_tag" + [ -n "$SIGNED_VERSION" ] || die "signature for ${artifact} names no release version; not installing it" ok "signature verified (minisign key $(printf '%s' "$C0MPUTE_PUBKEY" | cut -c1-12)…)" + # "latest" is whatever the release host answers. The signature proves this + # is a real release, not that it is the newest one, so `latest` never moves + # an existing install backwards. Pin C0MPUTE_VERSION to downgrade on purpose. + if [ "$C0MPUTE_VERSION" = "latest" ] && [ -n "$have_ver" ] && version_lt "$SIGNED_VERSION" "$have_ver"; then + die "the release offered as latest is ${SIGNED_VERSION}, older than the installed c0mpute ${have_ver}; refusing to downgrade (to install it on purpose: C0MPUTE_VERSION=${SIGNED_VERSION})" + fi + RELEASE_TAG="$SIGNED_VERSION" + say "installing c0mpute ${SIGNED_VERSION} (the version named in the signed release comment)" fi tar -xzf "$tmp/$artifact" -C "$C0MPUTE_HOME/bin" @@ -390,22 +555,76 @@ EOF } # ──────────────────────────────────────────────────────────────────────── -# Chain to upstream installers for coinpay + infernet +# Plugin installers (coinpay, infernet, transcode) # ──────────────────────────────────────────────────────────────────────── +# Unpack the release's signed c0mpute-plugins.tar.gz once. PLUGINS_STATE: +# ok PLUGINS_SRC holds the verified plugins//install.sh files +# legacy a release from before that archive: unverified URLs, with a warning +# failed it should be there and could not be had: run nothing unverified +PLUGINS_STATE="" +PLUGINS_SRC="" +fetch_plugins() { + [ -z "$PLUGINS_STATE" ] || return 0 + PLUGINS_STATE=failed + fp_rc=0 + fetch_signed_asset "$1" c0mpute-plugins.tar.gz || fp_rc=$? + case "$fp_rc" in + 0) + mkdir -p "$SCRATCH/plugins-src" + if tar -xzf "$ASSET_PATH" -C "$SCRATCH/plugins-src" 2>/dev/null; then + PLUGINS_SRC="$SCRATCH/plugins-src/plugins" + PLUGINS_STATE=ok + else + warn "could not unpack c0mpute-plugins.tar.gz" + fi + ;; + 2) PLUGINS_STATE=legacy ;; + esac +} + +# Run plugin $1's installer: the copy in this checkout when run from one, else +# the copy in the signed archive, else (old releases only) the unsigned $2. +run_plugin_installer() { + if [ -n "$REPO_ROOT" ] && [ -f "$REPO_ROOT/plugins/$1/install.sh" ]; then + say "running $1 plugin installer (local: $REPO_ROOT/plugins/$1/install.sh)" + sh "$REPO_ROOT/plugins/$1/install.sh" /dev/null 2>&1; then - say "upgrading $name (via $url)" + say "upgrading $name" else - say "installing $name (via $url)" + say "installing $name" fi - if ! curl -fsSL "$url" | sh; then + if ! run_plugin_installer "$name" "$url"; then warn "$name install/upgrade failed (continuing without it)" return 1 fi @@ -437,17 +656,9 @@ house_models_notice() { } # transcode is in-process (no separate binary), but it depends on -# ffmpeg. Delegate to the plugin's own install.sh — local copy if -# we're running in-repo, otherwise the published one. +# ffmpeg. Delegate to the plugin's own install.sh (see run_plugin_installer). install_transcode_deps() { - local_path="$(dirname "$0")/../plugins/transcode/install.sh" - if [ -f "$local_path" ]; then - say "running transcode plugin installer (local: $local_path)" - sh "$local_path" || warn "transcode deps install reported a problem" - return 0 - fi - say "running transcode plugin installer (via $TRANSCODE_INSTALL_URL)" - if ! curl -fsSL "$TRANSCODE_INSTALL_URL" | sh; then + if ! run_plugin_installer transcode "$TRANSCODE_INSTALL_URL"; then warn "transcode deps install failed (continuing without ffmpeg)" return 1 fi @@ -458,8 +669,11 @@ install_transcode_deps() { # ──────────────────────────────────────────────────────────────────────── # The release carries a version-less AppImage per arch # (c0mpute-desktop-linux-.AppImage, see .github/workflows/release.yml), -# checked here against its .sha256. The app updates itself from GitHub -# releases afterwards. macOS and Windows builds are on the releases page. +# signed like the binary (.minisig) and verified here. Only a release +# from before desktop signatures falls back to its .sha256, which comes from +# the same place as the file and so only catches a corrupt download. The app +# updates itself from GitHub releases afterwards. macOS and Windows builds are +# on the releases page. install_desktop() { plat=$(detect_platform) case "$plat" in @@ -474,21 +688,40 @@ install_desktop() { dest="$bin_dir/c0mpute-desktop" mkdir -p "$bin_dir" "$HOME/.local/share/applications" say "installing the desktop app ($asset)" - if ! curl -fsSL -o "$dest.tmp" "${RELEASE_BASE}/${C0MPUTE_VERSION}/${asset}" \ - || ! curl -fsSL -o "$dest.tmp.sha256" "${RELEASE_BASE}/${C0MPUTE_VERSION}/${asset}.sha256"; then - rm -f "$dest.tmp" "$dest.tmp.sha256" - warn "desktop app: $asset is not on the ${C0MPUTE_VERSION} release (yet); skipped" + rel="${RELEASE_TAG:-$C0MPUTE_VERSION}" + base="${RELEASE_BASE}/${rel}" + mkdir -p "$SCRATCH/assets" + dl="$SCRATCH/assets/$asset" + if ! curl -fsSL -o "$dl" "$base/$asset"; then + rm -f "$dl" + warn "desktop app: $asset is not on the ${rel} release (yet); skipped" return 0 fi - want=$(cut -d' ' -f1 < "$dest.tmp.sha256") - if command -v sha256sum >/dev/null 2>&1; then got=$(sha256sum "$dest.tmp" | cut -d' ' -f1) - else got=$(shasum -a 256 "$dest.tmp" | cut -d' ' -f1); fi - rm -f "$dest.tmp.sha256" - if [ "$want" != "$got" ]; then - rm -f "$dest.tmp" - warn "desktop app: checksum mismatch for $asset; not installed" + sig_code=$(curl -sSL -o "$dl.minisig" -w '%{http_code}' "$base/$asset.minisig" 2>/dev/null || echo "000") + if [ "$sig_code" = "200" ]; then + c0mpute_verify "$plat" "$dl" "$asset" "$RELEASE_TAG" + ok "desktop app: signature verified (${SIGNED_VERSION})" + elif [ "$sig_code" = "404" ] && legacy_release; then + warn "desktop app: c0mpute ${rel} predates signed desktop builds; checking only the .sha256 from the same release" + if ! curl -fsSL -o "$dl.sha256" "$base/$asset.sha256"; then + rm -f "$dl" "$dl.sha256" + warn "desktop app: no checksum for $asset on the ${rel} release; not installed" + return 0 + fi + want=$(cut -d' ' -f1 < "$dl.sha256") + got=$(c0mpute_sha256 "$dl" || true) + if [ -z "$want" ] || [ "$want" != "$got" ]; then + rm -f "$dl" "$dl.sha256" + warn "desktop app: checksum mismatch for $asset; not installed" + return 0 + fi + else + # The desktop builds are signed a few minutes after a release goes public. + rm -f "$dl" "$dl.minisig" + warn "desktop app: no signature for $asset on the ${rel} release yet (HTTP ${sig_code}); not installed. Re-run with --desktop later." return 0 fi + chmod 755 "$dl" && mv -f "$dl" "$dest" chmod 755 "$dest.tmp" && mv "$dest.tmp" "$dest" cat > "$HOME/.local/share/applications/c0mpute-desktop.desktop" <&2 + mkdir -p "$SCRATCH/tui-branch" + if curl -fsSL "https://github.com/profullstack/c0mpute/archive/refs/heads/$1.tar.gz" \ + | tar -xz -C "$SCRATCH/tui-branch" 2>/dev/null; then + find "$SCRATCH/tui-branch" -type d -path '*/apps/tui' 2>/dev/null | head -1 + fi +} + install_tui() { bun_bin="$(command -v bun || true)" [ -x "$HOME/.bun/bin/bun" ] && bun_bin="${bun_bin:-$HOME/.bun/bin/bun}" @@ -537,26 +781,31 @@ install_tui() { dest="$C0MPUTE_HOME/tui" - # Locate the TUI source: in-repo copy first, else fetch the repo tarball. - local_tui="$(dirname "$0")/../apps/tui" + # Locate the TUI source: in-repo copy first, else the signed release archive. src="" - cleanup_tmp="" - if [ -d "$local_tui/src" ]; then - src="$local_tui" + if [ -n "$REPO_ROOT" ] && [ -d "$REPO_ROOT/apps/tui/src" ]; then + src="$REPO_ROOT/apps/tui" + elif [ -n "${C0MPUTE_TUI_REF:-}" ]; then + # An explicit branch is a developer override; nothing signs a branch. + warn "C0MPUTE_TUI_REF=${C0MPUTE_TUI_REF}: installing c0mpute-tui UNVERIFIED from that branch" + src=$(fetch_tui_branch "$C0MPUTE_TUI_REF") else - say "fetching c0mpute-tui source" - tmp=$(mktemp -d) - cleanup_tmp="$tmp" - ref="${C0MPUTE_TUI_REF:-master}" - tarball="https://github.com/profullstack/c0mpute/archive/refs/heads/${ref}.tar.gz" - if curl -fsSL "$tarball" | tar -xz -C "$tmp" 2>/dev/null; then - src=$(find "$tmp" -type d -path '*/apps/tui' 2>/dev/null | head -1) - fi - if [ -z "$src" ] || [ ! -d "$src/src" ]; then - warn "couldn't fetch c0mpute-tui source; skipping (run: c0mpute tui later to retry)" - [ -n "$cleanup_tmp" ] && rm -rf "$cleanup_tmp" - return 1 - fi + tui_rc=0 + fetch_signed_asset "$platform" c0mpute-tui.tar.gz || tui_rc=$? + case "$tui_rc" in + 0) + mkdir -p "$SCRATCH/tui-src" + if tar -xzf "$ASSET_PATH" -C "$SCRATCH/tui-src" 2>/dev/null; then src="$SCRATCH/tui-src/tui"; fi + ;; + 2) + warn "c0mpute ${RELEASE_TAG:-(unknown release)} predates the signed TUI archive; fetching c0mpute-tui source UNVERIFIED" + src=$(fetch_tui_branch master) + ;; + esac + fi + if [ -z "$src" ] || [ ! -d "$src/src" ]; then + warn "couldn't fetch c0mpute-tui source; skipping (run: c0mpute tui later to retry)" + return 1 fi say "installing c0mpute-tui → $dest" @@ -568,7 +817,6 @@ install_tui() { # copied node_modules would be dangling symlinks). if ! ( cd "$dest" && "$bun_bin" install --production --no-save >/dev/null 2>&1 ); then warn "c0mpute-tui dependency install failed; skipping" - [ -n "$cleanup_tmp" ] && rm -rf "$cleanup_tmp" return 1 fi @@ -579,7 +827,6 @@ BUN="\$(command -v bun || echo "\$HOME/.bun/bin/bun")" exec "\$BUN" run "$dest/src/index.ts" "\$@" EOF chmod +x "$wrapper" - [ -n "$cleanup_tmp" ] && rm -rf "$cleanup_tmp" ok "installed c0mpute-tui → $wrapper" } @@ -587,33 +834,45 @@ EOF # PATH + diagnostics # ──────────────────────────────────────────────────────────────────────── -# Make `c0mpute` reachable no matter the shell config. Shell rc files -# (ensure_path) only help NEW shells and silently do nothing when they're -# root-owned (cloud GPU images) — that's how you get `c0mpute: command not -# found` right after a "successful" install. Mirror infernet's installer: -# 1. Symlink the binary into a system bin dir (already on every PATH), so it -# works in the CURRENT shell immediately. -# 2. Write /etc/profile.d so future login shells also see ~/.c0mpute/bin — -# which is where the peer binaries (coinpay, infernet) and c0mpute-tui -# live; the single symlink only covers the main binary. -# Both try a direct write first, then $SUDO. All best-effort. +# `c0mpute` is reached through PATH, never through a root-owned directory: +# ~/.c0mpute/bin holds files this user can rewrite, and a symlink to one in +# /usr/local/bin would put a user-writable program on root's PATH (anything +# that can write as this user would run as root the next time root, or sudo, +# typed `c0mpute`). So: +# - as a normal user: PATH comes from the shell rc files (ensure_path) and, +# when sudo is there, the /etc/profile.d hook below. That hook only names +# $HOME, so for every account it points at that account's own directory. +# - as root (cloud GPU images): root owns the binary, so the symlink is safe +# and makes `c0mpute` work in the current shell at once. +# All best-effort. link_system_bin() { target="$C0MPUTE_HOME/bin/c0mpute" [ -x "$target" ] || return 0 # nothing installed to link - linked="" - for sysbin in /usr/local/bin /usr/bin /opt/bin; do - [ -d "$sysbin" ] || continue - if ln -sf "$target" "$sysbin/c0mpute" 2>/dev/null; then - ln -sf "$target" "$sysbin/git-remote-c0mpute" 2>/dev/null || true - linked="$sysbin/c0mpute"; ok "symlinked $linked → $target"; break - fi - if [ -n "$SUDO" ] && $SUDO ln -sf "$target" "$sysbin/c0mpute" 2>/dev/null; then - $SUDO ln -sf "$target" "$sysbin/git-remote-c0mpute" 2>/dev/null || true - linked="$sysbin/c0mpute"; ok "symlinked $linked → $target (via sudo)"; break - fi - done - [ -n "$linked" ] || warn "no writable system bin dir — relying on shell rc / profile.d for PATH" + if [ "$(id -u 2>/dev/null || echo 1)" = "0" ]; then + for sysbin in /usr/local/bin /usr/bin /opt/bin; do + [ -d "$sysbin" ] || continue + if ln -sf "$target" "$sysbin/c0mpute" 2>/dev/null; then + ln -sf "$target" "$sysbin/git-remote-c0mpute" 2>/dev/null || true + ok "symlinked $sysbin/c0mpute → $target"; break + fi + done + else + # Earlier installers made that symlink with sudo. Removing it needs root, + # so say how instead of doing it behind a sudo prompt. + for sysbin in /usr/local/bin /usr/bin /opt/bin; do + for old in "$sysbin/c0mpute" "$sysbin/git-remote-c0mpute"; do + [ -L "$old" ] || continue + old_to=$(readlink "$old" 2>/dev/null || true) + case "$old_to" in + "$HOME"/*|/home/*|/Users/*) + warn "$old is a symlink into a home directory ($old_to), left by an earlier c0mpute installer." + warn " It puts a user-writable program on root's PATH. Remove it: sudo rm $old" + ;; + esac + done + done + fi if [ -d /etc/profile.d ]; then line='export PATH="$HOME/.c0mpute/bin:$HOME/.local/bin:$PATH"' @@ -624,20 +883,27 @@ link_system_bin() { ok "wrote /etc/profile.d/c0mpute.sh (via sudo)" fi fi - unset target linked sysbin line + unset target sysbin line old old_to } ensure_path() { - # Bash / Zsh / sh-style profile. Each rc file gets the PATH line + - # the right `mise activate ` invocation. Idempotent — won't - # duplicate lines on re-run. + # The rc file of the shell you actually use must exist, or a fresh account + # (zsh with no ~/.zshrc yet) would get no PATH line at all. + path_rc=$(shell_rc_for "$(detect_shell)") + case "$path_rc" in + */config.fish) ;; + *) [ -e "$path_rc" ] || touch "$path_rc" 2>/dev/null || true ;; + esac + + # Bash / Zsh / sh-style profile. Each rc file gets the PATH line. + # Idempotent — won't duplicate lines on re-run. for rc in "$HOME/.bashrc" "$HOME/.zshrc" "$HOME/.profile"; do [ -f "$rc" ] || continue # Must be writable. An rc file created root-owned by a prior sudo install # (common on cloud GPU images) makes `>> "$rc"` fail with a raw shell-level # "cannot create ...: Permission denied" that 2>/dev/null can't suppress. - # Pre-check and skip — link_system_bin() already put c0mpute on PATH via - # the /usr/local/bin symlink + /etc/profile.d hook, so this is best-effort. + # Pre-check and skip — link_system_bin() covers that case through the + # /etc/profile.d hook, so this is best-effort. if [ ! -w "$rc" ]; then warn "$rc not writable (try: sudo chown \$USER:\$USER $rc) — skipping PATH append" continue @@ -649,15 +915,6 @@ ensure_path() { printf 'export PATH="$HOME/.c0mpute/bin:$HOME/.local/bin:$PATH"\n' } >> "$rc" fi - - # mise activation per shell. .profile is sh-only and mise's - # `activate sh` doesn't exist; we skip it there. - if command -v mise >/dev/null 2>&1 && ! grep -q 'mise activate' "$rc"; then - case "$rc" in - *.bashrc) printf 'eval "$(mise activate bash)"\n' >> "$rc" ;; - *.zshrc) printf 'eval "$(mise activate zsh)"\n' >> "$rc" ;; - esac - fi done # Fish — different syntax, different config path. @@ -671,9 +928,16 @@ ensure_path() { printf 'fish_add_path -p $HOME/.c0mpute/bin $HOME/.local/bin\n' } >> "$fish_rc" fi - if command -v mise >/dev/null 2>&1 && ! grep -q 'mise activate' "$fish_rc"; then - printf 'mise activate fish | source\n' >> "$fish_rc" - fi + fi + + if grep -q '\.c0mpute/bin' "$path_rc" 2>/dev/null; then + case "$path_rc" in + */config.fish) ok "PATH: $path_rc has: fish_add_path -p \$HOME/.c0mpute/bin \$HOME/.local/bin" ;; + *) ok "PATH: $path_rc has: export PATH=\"\$HOME/.c0mpute/bin:\$HOME/.local/bin:\$PATH\"" ;; + esac + else + warn "could not add c0mpute to PATH in $path_rc; add this line yourself:" + warn " export PATH=\"\$HOME/.c0mpute/bin:\$HOME/.local/bin:\$PATH\"" fi } @@ -1007,12 +1271,10 @@ main() { platform=$(detect_platform) mkdir -p "$C0MPUTE_HOME/bin" - - # Install runtime tooling some plugins need (idempotent — skipped if - # already present). mise manages tool versions; bun runs the TUI and - # any future JS-flavoured plugins. - ensure_mise || true - ensure_bun || true + # Downloads land here and are verified here; gone when the script ends. + SCRATCH=$(mktemp -d) + trap 'rm -rf "$SCRATCH"' EXIT + REPO_ROOT=$(repo_root) if [ "$INSTALL_C0MPUTE" -eq 1 ]; then install_c0mpute "$platform"; fi if [ "$INSTALL_COINPAY" -eq 1 ]; then chain_install coinpay "$COINPAY_INSTALL_URL"; fi @@ -1030,8 +1292,8 @@ main() { # NO_STORAGE_RELOCATE=1. Same approach as infernet's installer. detect_storage_volume || true - # System-wide PATH (symlink + profile.d) FIRST — works even when the shell - # rc files below are root-owned/unwritable. Prevents `command not found`. + # profile.d hook FIRST — works even when the shell rc files below are + # root-owned/unwritable. Prevents `command not found`. link_system_bin ensure_path @@ -1042,8 +1304,12 @@ main() { # exits the script, and the worker never started). if [ "$INSTALL_TRANSCODE" -eq 1 ]; then install_transcode_deps || true; fi - # c0mpute-tui: Bun + hqtui terminal UI launched by `c0mpute tui`. - if [ "$INSTALL_TUI" -eq 1 ]; then install_tui || true; fi + # c0mpute-tui: Bun + hqtui terminal UI launched by `c0mpute tui`. bun (a + # pinned, checksummed release) is only installed for it. + if [ "$INSTALL_TUI" -eq 1 ]; then + ensure_bun "$platform" || true + install_tui || true + fi # Desktop app (apps/desktop): opt-in, a node is usually headless. if [ "$INSTALL_DESKTOP" -eq 1 ]; then install_desktop || true; fi @@ -1144,6 +1410,7 @@ EOF user_shell=$(detect_shell) shell_bin="${SHELL:-/bin/bash}" + rm -rf "$SCRATCH" # exec never returns, so the EXIT trap would not run printf '\n\033[1;36m→\033[0m starting fresh %s shell with c0mpute on $PATH\n' "$user_shell" printf ' (type \033[1;36mexit\033[0m to return to your original shell)\n\n' case "$user_shell" in diff --git a/scripts/pack-release-extras.sh b/scripts/pack-release-extras.sh new file mode 100755 index 00000000..9fc7676e --- /dev/null +++ b/scripts/pack-release-extras.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env sh +# Build the two platform-independent release archives that scripts/install.sh +# and the CLI install from, so they can be signed like the binaries: +# +# c0mpute-plugins.tar.gz plugins//install.sh + plugins//module.toml +# c0mpute-tui.tar.gz tui/src, tui/package.json, tui/tsconfig.json +# +# Usage: scripts/pack-release-extras.sh +# +# Called by .github/workflows/release.yml (job `extras`) on the tagged tree; +# run it by hand to see exactly what a release ships. Writes and +# .sha256 into . Needs GNU tar (ubuntu runners, any Linux). +set -eu + +root="${1:?usage: pack-release-extras.sh }" +out="${2:?usage: pack-release-extras.sh }" +[ -d "$root/plugins" ] || { echo "no plugins/ under $root" >&2; exit 1; } +[ -f "$root/apps/tui/src/index.ts" ] || { echo "no apps/tui/src/index.ts under $root" >&2; exit 1; } +[ -f "$root/apps/tui/package.json" ] || { echo "no apps/tui/package.json under $root" >&2; exit 1; } +mkdir -p "$out" +out=$(cd "$out" && pwd) +root=$(cd "$root" && pwd) + +stage=$(mktemp -d) +trap 'rm -rf "$stage"' EXIT + +# Same bytes for the same tree: fixed order, owner and mtime, no gzip header time. +pack() { # + tar --sort=name --owner=0 --group=0 --numeric-owner --mtime='2020-01-01 00:00:00Z' \ + -C "$stage" -cf - "$2" | gzip -n -9 > "$out/$1" + ( cd "$out" && sha256sum "$1" > "$1.sha256" ) +} + +# Plugins: every installer, plus its manifest. The installers are +# self-contained (they read nothing else from the repo at install time). +mkdir -p "$stage/plugins" +n=0 +for dir in "$root"/plugins/*/; do + id=$(basename "$dir") + [ -f "$dir/install.sh" ] || continue + mkdir -p "$stage/plugins/$id" + cp "$dir/install.sh" "$stage/plugins/$id/install.sh" + chmod 755 "$stage/plugins/$id/install.sh" + if [ -f "$dir/module.toml" ]; then + cp "$dir/module.toml" "$stage/plugins/$id/module.toml" + chmod 644 "$stage/plugins/$id/module.toml" + fi + n=$((n + 1)) +done +[ "$n" -gt 0 ] || { echo "no plugins/*/install.sh under $root" >&2; exit 1; } +chmod -R u=rwX,go=rX "$stage/plugins" +# The installers scripts/install.sh runs by default must be there. +for id in coinpay infernet transcode; do + [ -f "$stage/plugins/$id/install.sh" ] || { echo "plugins/$id/install.sh missing" >&2; exit 1; } +done +pack c0mpute-plugins.tar.gz plugins + +# TUI: exactly what the installers copy into ~/.c0mpute/tui (never node_modules). +mkdir -p "$stage/tui" +cp -R "$root/apps/tui/src" "$stage/tui/src" +cp "$root/apps/tui/package.json" "$stage/tui/package.json" +[ -f "$root/apps/tui/tsconfig.json" ] && cp "$root/apps/tui/tsconfig.json" "$stage/tui/tsconfig.json" +chmod -R u=rwX,go=rX "$stage/tui" +pack c0mpute-tui.tar.gz tui + +echo "packed $n plugin installer(s) and the TUI into $out:" +ls -l "$out/c0mpute-plugins.tar.gz" "$out/c0mpute-tui.tar.gz" From 3c4b72cfc9a5c1f232a9df22a3f6de1effa4b7ca Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 13 Nov 2026 10:23:38 +0000 Subject: [PATCH 2/2] hosting: proof-required counting is opt-in ([hosting] require_proofs) With it on, hosting_market_e2e over-replicates: challenges are voided while a small site is still being copied, so every host takes a copy. Default keeps the 0.3.64 fill rule; drops still need live proofs from three others. Co-Authored-By: Claude Fable 5.2 --- docs/hosting.md | 11 +++++++++-- node/crates/c0mpute-core/src/config.rs | 8 ++++++++ node/crates/c0mpute-core/src/hosting/mod.rs | 13 ++++++++++++- node/crates/c0mpute-core/src/hosting/replicas.rs | 11 +++++++++++ node/crates/c0mpute-core/src/supervisor.rs | 1 + 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/hosting.md b/docs/hosting.md index 0de97d6b..2e3a2ee6 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -44,8 +44,15 @@ Git repositories ride on the same machinery: see [git hosting](git-hosting.md). ### An announced copy counts once it is proven -Announcing a copy is free: any key can gossip "I hold this site". So an announcement -alone does not fill one of the three copies, and does not let anyone drop theirs. +Announcing a copy is free: any key can gossip "I hold this site". A node never drops +its own copy on announcements: it first fetches a random chunk from three other holders. + +The stricter rules below (an announcement does not fill a slot either, and a drop +needs holders with standing) are **off by default** and turned on with +`[hosting] require_proofs = true` (or `C0MPUTE_HOSTING_REQUIRE_PROOFS=1`). With them +on, while a small site is still being copied its chunks are read constantly, which +voids challenges, so extra hosts can take a copy and keep it for hours. Until that is +settled, the default counts announcements for filling, as every version before did. - **Filling.** A holder counts toward the three once the node deciding has seen it pass a challenge (a random chunk, fetched straight from that peer and checked against its diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index 6e3424b2..11fe2cd0 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -270,6 +270,13 @@ pub struct HostingConfig { /// space above the free-disk floor (see `hosting::auto_budget`). #[serde(default = "default_hosting_max_bytes")] pub max_bytes: u64, + /// Count another node's copy of a site only after it has answered a + /// storage challenge, and rely on it before giving a copy up only once + /// it has answered several over hours. Off by default: an announcement + /// counts, as before 0.3.65 (`C0MPUTE_HOSTING_REQUIRE_PROOFS=1` also + /// turns it on). + #[serde(default)] + pub require_proofs: bool, /// What this node asks to be a preferred (paid) host, USD per GiB-month. /// Published in the OpenDisk descriptor and in hosting gossip; a site /// offering less does not count this node as preferred. @@ -309,6 +316,7 @@ impl Default for HostingConfig { Self { enabled: true, max_bytes: default_hosting_max_bytes(), + require_proofs: false, price_usd_per_gib_month: default_hosting_price(), payout_wallet: None, payout_coin: crate::payments::default_coin(), diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs index 91510a08..1f187209 100644 --- a/node/crates/c0mpute-core/src/hosting/mod.rs +++ b/node/crates/c0mpute-core/src/hosting/mod.rs @@ -374,6 +374,13 @@ pub struct Options { pub catalog_path: Option, pub timing: Timing, pub dns: dns::Verifier, + /// `[hosting] require_proofs`: count a holder toward a site's copies only + /// after this node saw it pass a challenge, and rely on it for a drop + /// only once it has standing. Off by default: announcements count, as + /// before 0.3.65, until the stricter rule has been run on real networks + /// (while a small site is still being copied its chunks are read + /// constantly, challenges are voided, and every host takes a copy). + pub require_proofs: bool, /// Key that signs proof-of-storage receipts; no challenges without it. pub challenger: Option, pub challenge: ChallengeScope, @@ -404,6 +411,7 @@ impl Default for Options { catalog_path: None, timing: Timing::default(), dns: dns::Verifier::default(), + require_proofs: false, challenger: None, challenge: ChallengeScope::Own, challenge_every: Duration::from_secs(600), @@ -476,7 +484,10 @@ impl Hosting { hosts: opts.hosts, max_bytes: opts.max_bytes, timing: opts.timing, - map: RwLock::new(ReplicaMap::default()), + map: RwLock::new(ReplicaMap { + count_claims: !(opts.require_proofs || std::env::var("C0MPUTE_HOSTING_REQUIRE_PROOFS").is_ok_and(|v| v == "1")), + ..Default::default() + }), catalog: RwLock::new(Catalog::default()), catalog_path: opts.catalog_path, records_path, diff --git a/node/crates/c0mpute-core/src/hosting/replicas.rs b/node/crates/c0mpute-core/src/hosting/replicas.rs index 87019723..b56e7705 100644 --- a/node/crates/c0mpute-core/src/hosting/replicas.rs +++ b/node/crates/c0mpute-core/src/hosting/replicas.rs @@ -97,6 +97,11 @@ pub struct ReplicaMap { pub receipts: HashMap<(String, String), Vec>, /// Challenges this node saw pass, per (site id, holder). pub proofs: HashMap<(String, String), Proof>, + /// Count a holder on its announcement alone, as every version up to + /// 0.3.64 did (`[hosting] require_proofs` off, the default). With it + /// false a holder counts only once proven ([`Self::proven`]) and may only + /// be relied on for a drop once it has standing ([`Self::has_standing`]). + pub count_claims: bool, } impl ReplicaMap { @@ -423,6 +428,9 @@ impl ReplicaMap { /// or payer (the keys [`proof::demoted`] trusts) is a pass no older than /// `ttl`. `now_secs` is the wall clock receipts are dated with. pub fn proven(&self, site_id: &str, holder: &str, now: Instant, now_secs: u64, ttl: Duration) -> bool { + if self.count_claims { + return true; + } let Some(rec) = self.records.get(site_id) else { return false; }; @@ -450,6 +458,9 @@ impl ReplicaMap { /// with no gap of `span` between passes since. Receipts from other keys /// do not count here: only what this node asked for itself. pub fn has_standing(&self, site_id: &str, holder: &str, now: Instant, passes: u32, span: Duration) -> bool { + if self.count_claims { + return true; + } let Some(rec) = self.records.get(site_id) else { return false; }; diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index a4719180..b419ad18 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -946,6 +946,7 @@ impl Supervisor { crate::hosting::Options { hosts, max_bytes: cfg.max_bytes, + require_proofs: cfg.require_proofs, catalog_path, challenger, challenge,