diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 41323dc6..9c106fd4 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -15,6 +15,12 @@ name: release
# production minisign key (repo secrets MINISIGN_SECRET_KEY + MINISIGN_PASSWORD;
# public key minisign.pub) and uploads .minisig next to it, which
# scripts/install.sh requires. A release is never published unsigned.
+# The same goes for the two platform-independent archives the `extras` job
+# packs from the tagged tree (scripts/pack-release-extras.sh):
+# c0mpute-plugins.tar.gz (plugins//install.sh) and c0mpute-tui.tar.gz
+# (apps/tui), which the installer and the CLI install from instead of fetching
+# unsigned scripts. The desktop artifacts are built after publishing and signed
+# by `desktop-sign`.
#
# A failed platform leaves the draft unpublished (Latest stays on the
# previous release); fix and re-run the failed jobs.
@@ -157,10 +163,49 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: gh release upload "$TAG" "$ARTIFACT" "$ARTIFACT_SHA" --clobber
+ # Platform-independent archives the installer and the CLI install from: the
+ # plugin installers and the TUI source, packed from the tagged tree. Uploaded
+ # to the draft with a .sha256 like the binaries; `publish` signs them.
+ extras:
+ name: plugins + tui archives
+ needs: draft
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ with:
+ ref: ${{ env.TAG }}
+ path: tree
+
+ # The packer comes from the commit this workflow runs at, so re-running
+ # an older tag (workflow_dispatch) from before the script existed still
+ # works. On a tag push that commit is the tag itself.
+ - uses: actions/checkout@v5
+ with:
+ ref: ${{ github.sha }}
+ path: tools
+ sparse-checkout: scripts/pack-release-extras.sh
+ sparse-checkout-cone-mode: false
+
+ - name: Pack and upload to the draft release
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ set -eu
+ sh tools/scripts/pack-release-extras.sh "$GITHUB_WORKSPACE/tree" "$RUNNER_TEMP/extras"
+ cd "$RUNNER_TEMP/extras"
+ sha256sum -c c0mpute-plugins.tar.gz.sha256
+ sha256sum -c c0mpute-tui.tar.gz.sha256
+ tar -tzf c0mpute-plugins.tar.gz
+ tar -tzf c0mpute-tui.tar.gz
+ gh release upload "$TAG" \
+ c0mpute-plugins.tar.gz c0mpute-plugins.tar.gz.sha256 \
+ c0mpute-tui.tar.gz c0mpute-tui.tar.gz.sha256 --clobber
+
publish:
name: publish + mark latest
- # Runs only when every platform job succeeded (needs + default success()).
- needs: release
+ # Runs only when every platform job and the extras job succeeded (needs +
+ # default success()).
+ needs: [release, extras]
runs-on: ubuntu-latest
steps:
# Only for minisign.pub, the public key every signature must verify against.
@@ -200,7 +245,7 @@ jobs:
work="$RUNNER_TEMP/sign"; mkdir -p "$work"; cd "$work"
printf '%s\n' "$MINISIGN_SECRET_KEY" > "$RUNNER_TEMP/minisign.key"
trap 'rm -f "$RUNNER_TEMP/minisign.key"' EXIT
- want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip"
+ want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip c0mpute-plugins.tar.gz c0mpute-tui.tar.gz"
for a in $want; do
gh release download "$TAG" -p "$a" -p "$a.sha256" --clobber
# Sign only what the platform build uploaded: check its checksum first.
@@ -217,7 +262,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: |
set -eu
- want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip"
+ want="c0mpute-linux-x86_64.tar.gz c0mpute-linux-aarch64.tar.gz c0mpute-darwin-x86_64.tar.gz c0mpute-darwin-aarch64.tar.gz c0mpute-windows-x86_64.zip c0mpute-plugins.tar.gz c0mpute-tui.tar.gz"
have=$(gh release view "$TAG" --json assets -q '.assets[].name')
for a in $want; do
for f in "$a" "$a.sha256" "$a.minisig"; do
@@ -326,5 +371,65 @@ jobs:
[ ${#files[@]} -gt 0 ] || { echo "no desktop artifacts built" >&2; ls -la; exit 1; }
for f in "${files[@]}"; do
if command -v sha256sum >/dev/null; then sha256sum "$f" > "$f.sha256"; else shasum -a 256 "$f" > "$f.sha256"; fi
+ # On a re-run the build differs from the one the old signature
+ # covers: drop it, `desktop-sign` uploads a fresh one.
+ gh release delete-asset "$TAG" "$f.minisig" -y >/dev/null 2>&1 || true
gh release upload "$TAG" "$f" "$f.sha256" --clobber
done
+
+ # Sign the desktop artifacts with the same key as everything else. They are
+ # built after `publish` (a desktop failure must not hold back a CLI release),
+ # so this is its own job, and the key never enters a job that runs
+ # `bun install` or electron-builder. It signs whatever desktop builds are on
+ # the release, so one failed platform does not leave the others unsigned;
+ # after re-running a failed desktop job, re-run this one too.
+ # scripts/install.sh --desktop requires the AppImage's .minisig.
+ desktop-sign:
+ name: sign desktop artifacts
+ needs: [publish, desktop]
+ if: ${{ !cancelled() && needs.publish.result == 'success' }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ with:
+ ref: ${{ env.TAG }}
+ sparse-checkout: minisign.pub
+ sparse-checkout-cone-mode: false
+
+ - name: Install minisign (pinned)
+ run: |
+ set -eu
+ curl -fsSL -o "$RUNNER_TEMP/minisign.tar.gz" https://github.com/jedisct1/minisign/releases/download/0.12/minisign-0.12-linux.tar.gz
+ echo "9a599b48ba6eb7b1e80f12f36b94ceca7c00b7a5173c95c3efc88d9822957e73 $RUNNER_TEMP/minisign.tar.gz" | sha256sum -c -
+ tar -xzf "$RUNNER_TEMP/minisign.tar.gz" -C "$RUNNER_TEMP" minisign-linux/x86_64/minisign
+ install -m 755 "$RUNNER_TEMP/minisign-linux/x86_64/minisign" /usr/local/bin/minisign 2>/dev/null \
+ || sudo install -m 755 "$RUNNER_TEMP/minisign-linux/x86_64/minisign" /usr/local/bin/minisign
+ minisign -v
+
+ - name: Sign desktop artifacts and upload .minisig files
+ env:
+ GH_TOKEN: ${{ github.token }}
+ MINISIGN_SECRET_KEY: ${{ secrets.MINISIGN_SECRET_KEY }}
+ MINISIGN_PASSWORD: ${{ secrets.MINISIGN_PASSWORD }}
+ run: |
+ set -eu
+ [ -n "$MINISIGN_SECRET_KEY" ] && [ -n "$MINISIGN_PASSWORD" ] || { echo "MINISIGN_SECRET_KEY / MINISIGN_PASSWORD secrets missing; cannot sign" >&2; exit 1; }
+ umask 077
+ pub="$GITHUB_WORKSPACE/minisign.pub"
+ [ -s "$pub" ] || { echo "minisign.pub missing at $TAG; not signing" >&2; exit 1; }
+ work="$RUNNER_TEMP/sign-desktop"; mkdir -p "$work"; cd "$work"
+ printf '%s\n' "$MINISIGN_SECRET_KEY" > "$RUNNER_TEMP/minisign.key"
+ trap 'rm -f "$RUNNER_TEMP/minisign.key"' EXIT
+ names=$(gh release view "$TAG" --json assets -q '.assets[].name' \
+ | grep -E '^c0mpute-desktop-[A-Za-z0-9._-]+\.(AppImage|deb|dmg|zip|exe)$' || true)
+ [ -n "$names" ] || { echo "no desktop artifacts on $TAG; nothing to sign" >&2; exit 1; }
+ for a in $names; do
+ gh release download "$TAG" -p "$a" -p "$a.sha256" --clobber
+ # Sign only what the desktop build uploaded: check its checksum first.
+ sha256sum -c "$a.sha256"
+ printf '%s\n' "$MINISIGN_PASSWORD" | minisign -S -s "$RUNNER_TEMP/minisign.key" -m "$a" \
+ -t "timestamp:$(date +%s) version:$TAG file:$a" >/dev/null
+ minisign -V -p "$pub" -m "$a"
+ gh release upload "$TAG" "$a.minisig" --clobber
+ rm -f "$a" "$a.sha256" "$a.minisig"
+ done
diff --git a/Cargo.lock b/Cargo.lock
index 74aacb8b..f6313642 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -486,7 +486,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "c0mpute-api"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"c0mpute-proto",
@@ -501,7 +501,7 @@ dependencies = [
[[package]]
name = "c0mpute-bench"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"blake3",
@@ -512,7 +512,7 @@ dependencies = [
[[package]]
name = "c0mpute-cli"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"async-trait",
@@ -549,7 +549,7 @@ dependencies = [
[[package]]
name = "c0mpute-core"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"aes-gcm",
"anyhow",
@@ -603,7 +603,7 @@ dependencies = [
[[package]]
name = "c0mpute-doctor"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"serde",
@@ -615,7 +615,7 @@ dependencies = [
[[package]]
name = "c0mpute-envelope"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"blake3",
"bs58",
@@ -628,7 +628,7 @@ dependencies = [
[[package]]
name = "c0mpute-gateway"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"axum",
@@ -650,7 +650,7 @@ dependencies = [
[[package]]
name = "c0mpute-market"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"c0mpute-envelope",
"serde",
@@ -660,7 +660,7 @@ dependencies = [
[[package]]
name = "c0mpute-net"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"async-trait",
@@ -681,7 +681,7 @@ dependencies = [
[[package]]
name = "c0mpute-placement"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"async-trait",
@@ -702,7 +702,7 @@ dependencies = [
[[package]]
name = "c0mpute-proto"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"blake3",
"hex",
@@ -714,7 +714,7 @@ dependencies = [
[[package]]
name = "c0mpute-sealed"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"aes",
"aes-gcm",
@@ -736,7 +736,7 @@ dependencies = [
[[package]]
name = "c0mpute-secure-chat"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"aes-gcm",
"anyhow",
@@ -759,7 +759,7 @@ dependencies = [
[[package]]
name = "c0mpute-store"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"blake3",
@@ -778,7 +778,7 @@ dependencies = [
[[package]]
name = "c0mpute-transcode"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"c0mpute-proto",
@@ -791,7 +791,7 @@ dependencies = [
[[package]]
name = "c0mpute-update"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"hex",
@@ -806,7 +806,7 @@ dependencies = [
[[package]]
name = "c0mpute-verify"
-version = "0.3.64"
+version = "0.3.65"
dependencies = [
"anyhow",
"blake3",
diff --git a/Cargo.toml b/Cargo.toml
index 73d5ed2f..8ed9cd48 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,7 @@ members = [
]
[workspace.package]
-version = "0.3.64"
+version = "0.3.65"
edition = "2024"
license = "MIT"
repository = "https://github.com/profullstack/c0mpute"
diff --git a/README.md b/README.md
index 122f3b4a..d29b57f2 100644
--- a/README.md
+++ b/README.md
@@ -508,7 +508,9 @@ cargo build --bin c0mpute
- `c0mpute` Rust binary builds; clap surface for `doctor`, `worker`,
`job`, `plugin`, `transcode`, `coinpay` (passthrough), `infernet`
(passthrough), `tui`, `version`
-- `c0mpute plugin install ` chain-calls upstream installers
+- `c0mpute plugin install ` installs a first-party plugin from the signed
+ release archive; `c0mpute plugin install --unverified ` runs a
+ third-party installer
- `c0mpute doctor` cross-checks `coinpay` and `infernet` on PATH
- `c0mpute bench` runs fib / matmul / blake3 at 1…N threads, writes
`bench.json` (same `metadata` + `results` layout as
diff --git a/SECURITY.md b/SECURITY.md
index f0f5215a..f56af0c2 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -8,9 +8,16 @@ rather than a public issue.
## Release signing
-Every artifact on a [c0mpute release](https://github.com/profullstack/c0mpute/releases)
-(`c0mpute--.tar.gz`, `c0mpute-windows-x86_64.zip`) has a
-[minisign](https://jedisct1.github.io/minisign/) signature next to it, `.minisig`.
+Every artifact a [c0mpute release](https://github.com/profullstack/c0mpute/releases) is
+installed from has a [minisign](https://jedisct1.github.io/minisign/) signature next to it,
+`.minisig`:
+
+| Artifact | What it is |
+| --- | --- |
+| `c0mpute--.tar.gz`, `c0mpute-windows-x86_64.zip` | the `c0mpute` binary |
+| `c0mpute-plugins.tar.gz` | every `plugins//install.sh` (and its `module.toml`) of that tag |
+| `c0mpute-tui.tar.gz` | the terminal UI source (`apps/tui`: `src/`, `package.json`, `tsconfig.json`) |
+| `c0mpute-desktop-*.{AppImage,deb,dmg,zip,exe}` | the desktop app builds |
Public key ([`minisign.pub`](minisign.pub), key id `01A1B54137EC392A`):
@@ -21,26 +28,111 @@ RWQqOew3QbWhAWH36TA5roreh18RgRqgozSwhE3yZtdza7eeLKmbDbJM
The signed (trusted) comment reads `timestamp: version: file:`, so a
signature only vouches for that one file of that one release.
-What checks it:
+### What signs
+
+**The release workflow** (`.github/workflows/release.yml`). The platform jobs build the
+binaries and the `extras` job packs the two archives from the tagged tree
+(`scripts/pack-release-extras.sh`; run it yourself to see exactly what ships). The `publish`
+job signs all of them and refuses to publish a release that is missing any binary, either
+archive, or any of their `.minisig` files. The desktop builds are made after the release is
+public and are signed by the `desktop-sign` job a few minutes later; until then they have no
+signature and the installer will not install them.
+
+### What verifies
+
+- **`scripts/install.sh`** (served at `https://c0mpute.com/install.sh`):
+ - **The binary.** Downloads the artifact and its `.minisig`, verifies the signature with
+ the embedded key and checks that the signed comment names the artifact (and the release
+ tag when one was asked for). A missing signature, a failed verification or a mismatched
+ comment aborts the install.
+ - **Which release.** Everything else in the run comes from the release named in the
+ binary's signed comment (or, when the binary is already installed, from that binary's own
+ release), and each signature must be for that tag.
+ - **Plugin installers** (coinpay, infernet, transcode) run from the verified
+ `c0mpute-plugins.tar.gz`, and **the TUI** is installed from the verified
+ `c0mpute-tui.tar.gz`. If an archive is on the release, a missing or bad signature aborts
+ the install.
+ - **The desktop AppImage** (`--desktop`) is verified the same way. No signature means it
+ is not installed.
+ - **Releases up to v0.3.64** predate these signed extras. Only for those, and only when
+ the release really does not have the asset (HTTP 404), the installer falls back to the
+ old downloads (`https://c0mpute.com/plugins//install.sh`, the GitHub source tarball,
+ the AppImage's `.sha256`) and prints a warning that they are unverified. On any newer
+ release a missing extra is an error, so hiding the asset cannot force the fallback.
+ - **minisign.** If `minisign` is not installed the script downloads the pinned upstream
+ minisign 0.12 release and checks it against a hard-coded sha256 before running it (Linux
+ x86_64/aarch64, macOS arm64; on Intel macs install minisign with `brew install minisign`).
+ - **bun** (the runtime the TUI needs; only installed with the TUI). The script downloads
+ the pinned upstream bun 1.3.14 release zip for the OS/arch and checks it against a
+ hard-coded sha256 before unpacking it. It no longer pipes `bun.sh/install` to a shell, and
+ it no longer installs mise at all (a contributor tool, not needed on a node). A platform
+ without a pinned hash gets a message and no bun.
+- **`c0mpute` self-update** (`node/crates/c0mpute-update`) verifies the binary's signature
+ with the key compiled into the binary, after the sha256 check and before swapping the
+ binary. No signature or a bad one means no update.
+- **`c0mpute tui`** and **`c0mpute plugin install `** download `c0mpute-tui.tar.gz` /
+ `c0mpute-plugins.tar.gz` of the running binary's own release and verify them with that same
+ compiled-in key (file and release tag included) before anything is unpacked or run. There
+ is no unverified fallback.
+
+`C0MPUTE_ALLOW_UNSIGNED=1` makes the installer accept a missing signature. It never accepts
+a bad one.
+
+### Rollback: what `latest` does and does not promise
+
+A signature proves an artifact is a real c0mpute release. It does not prove it is the newest
+one: whoever answers for `https://c0mpute.com/releases/latest/` (a redirect to GitHub's
+latest release) could serve an older, genuinely signed release with a known bug.
-- **`scripts/install.sh`** (served at `https://c0mpute.com/install.sh`) downloads the
- artifact and its `.minisig`, verifies the signature with the embedded key and checks the
- trusted comment names the artifact (and the release tag when it is known). It aborts on a
- missing signature, a failed verification or a mismatched comment. If `minisign` is not
- installed it downloads the pinned upstream minisign 0.12 release and checks it against a
- hard-coded sha256 before running it (Linux x86_64/aarch64, macOS arm64; on Intel macs
- install minisign with `brew install minisign`).
-- **`c0mpute` self-update** (`node/crates/c0mpute-update`) verifies the same signature with
- the key compiled into the binary, after the sha256 check and before swapping the binary.
- No signature or a bad one means no update.
-- **The release workflow** (`.github/workflows/release.yml`) signs in its `publish` job and
- refuses to publish a release that is missing any `.minisig`.
+What the installer does about it, without trusting a second server: it reads the release
+version out of the signed comment, prints it (`installing c0mpute v`), and when you
+did not ask for a specific version it refuses to install a release older than the `c0mpute`
+already on the machine. `C0MPUTE_VERSION=v` installs an older release on purpose.
+Self-update likewise only moves to a higher version.
-Verify a download by hand:
+What that does not prevent:
+
+- A **first install** has nothing to compare against, so it can still be handed an older
+ signed release. Check the printed version against the
+ [releases page](https://github.com/profullstack/c0mpute/releases) if that matters to you.
+- Being **held at the current version** (served the release you already have, never the
+ newer one).
+
+### What is still not verified
+
+- **`install.sh` itself** is fetched over HTTPS from c0mpute.com; read it before piping it
+ to `sh` if you prefer (`curl -fsSL https://c0mpute.com/install.sh | less`).
+- **What the plugin installers fetch.** The signed `plugins/coinpay/install.sh` and
+ `plugins/infernet/install.sh` are thin wrappers that download and run the upstream
+ installers (`https://coinpayportal.com/install.sh`, `https://infernetprotocol.com/install.sh`)
+ over HTTPS; `plugins/threatcrush/install.sh` does the same for ThreatCrush. Those upstream
+ scripts and what they install are not covered by the c0mpute signature. The transcode
+ installer installs ffmpeg through your system package manager. The video installer pulls a
+ container image pinned by digest, and builds from a clone of the repository's default
+ branch if that pull fails.
+- **The TUI's npm dependencies.** The TUI source is signed, but `bun install` resolves its
+ dependencies (`@profullstack/hqtui` and what it pulls in) from the npm registry at install
+ time, without a lockfile.
+- **`c0mpute plugin install --unverified `** runs somebody else's script. It
+ refuses without the flag and warns with it.
+- **`C0MPUTE_TUI_REF=`** (a developer override) installs the TUI from a branch
+ tarball, with a warning.
+- **The desktop app's own updater** and the macOS/Windows builds' platform code signing
+ (there are no certificates yet).
+
+### PATH, not root
+
+The installer puts `~/.c0mpute/bin` on your PATH through your shell's rc file and prints the
+line. It does not symlink the binary into `/usr/local/bin` with sudo any more: that put a
+program your account can rewrite on root's PATH. If an earlier install left such a symlink,
+the installer tells you; remove it with `sudo rm /usr/local/bin/c0mpute
+/usr/local/bin/git-remote-c0mpute`. (Run as root, the installer still links there, since root
+then owns the binary.) It does still write `/etc/profile.d/c0mpute.sh`, using sudo when it is
+available: one `export PATH="$HOME/.c0mpute/bin:$HOME/.local/bin:$PATH"` line, which for
+every account names that account's own home directory.
+
+### Verify a download by hand
```sh
minisign -Vm c0mpute-linux-x86_64.tar.gz -P RWQqOew3QbWhAWH36TA5roreh18RgRqgozSwhE3yZtdza7eeLKmbDbJM
```
-
-The installer script itself is fetched over HTTPS from c0mpute.com; read it before piping it
-to `sh` if you prefer (`curl -fsSL https://c0mpute.com/install.sh | less`).
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 7b04d4ee..711f5bff 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -2,7 +2,7 @@
"name": "@c0mpute/desktop",
"productName": "c0mpute",
"desktopName": "c0mpute-desktop.desktop",
- "version": "0.3.39",
+ "version": "0.3.65",
"private": true,
"type": "module",
"description": "c0mpute desktop: chat, agent with your MCP servers and skills, node manager, hosting and tasks, with the c0mpute CLI bundled.",
diff --git a/apps/tui/package.json b/apps/tui/package.json
index 2e55b6da..d5c29373 100644
--- a/apps/tui/package.json
+++ b/apps/tui/package.json
@@ -1,6 +1,6 @@
{
"name": "@c0mpute/tui",
- "version": "0.3.39",
+ "version": "0.3.65",
"private": true,
"type": "module",
"bin": {
diff --git a/apps/web/package.json b/apps/web/package.json
index 3189fd0e..63b5e321 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -1,6 +1,6 @@
{
"name": "c0mpute",
- "version": "0.3.39",
+ "version": "0.3.65",
"private": true,
"type": "module",
"scripts": {
diff --git a/apps/web/src/app/docs/page.tsx b/apps/web/src/app/docs/page.tsx
index 2839dc59..d40854a6 100644
--- a/apps/web/src/app/docs/page.tsx
+++ b/apps/web/src/app/docs/page.tsx
@@ -411,12 +411,12 @@ c0mpute tui # terminal app: chat, jobs, node, network`}
-{`# Install from a plugin's signed install.sh
-c0mpute plugin install https://example.com/my-plugin/install.sh
-
-# Install by id from the c0mpute marketplace
+{`# Install a first-party plugin by id (from the signed release archive)
c0mpute plugin install my-plugin
+# Install a third-party plugin from its install.sh (not verified by c0mpute)
+c0mpute plugin install --unverified https://example.com/my-plugin/install.sh
+
# List installed
c0mpute plugin list
diff --git a/apps/web/src/app/plugins/page.tsx b/apps/web/src/app/plugins/page.tsx
index cdc56825..1cc81241 100644
--- a/apps/web/src/app/plugins/page.tsx
+++ b/apps/web/src/app/plugins/page.tsx
@@ -43,7 +43,7 @@ export default function PluginsPage() {
Third-party plugins install with{" "}
- c0mpute plugin install <url> where{" "}
+ c0mpute plugin install --unverified <url> where{" "}
<url> points at the plugin's{" "}
install.sh. See{" "}
diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts
index 9a2241eb..ada47fcc 100644
--- a/apps/web/src/app/releases/latest.json/route.ts
+++ b/apps/web/src/app/releases/latest.json/route.ts
@@ -28,7 +28,7 @@ export const dynamic = "force-dynamic";
* this in sync with the workspace version in `Cargo.toml` on each release so a
* freshly built CLI reports "already latest" rather than a phantom upgrade.
*/
-const FALLBACK_VERSION = "0.3.39";
+const FALLBACK_VERSION = "0.3.65";
const FALLBACK_MIN_REQUIRED = "0.0.1";
type Channel = "stable" | "beta" | "nightly";
diff --git a/docs/apps.md b/docs/apps.md
index 8c721aee..3d206590 100644
--- a/docs/apps.md
+++ b/docs/apps.md
@@ -62,6 +62,8 @@ NODE_ENV = "production"
| Secrets | Secret env plus the app key travel sealed to each holder's X25519 key (`Seal` gossip): from the owner's CLI at deploy, then from any holder to newcomers, so the owner can go offline. The record carries only their blake3. |
| Serving | Gateways forward to the primary over the `apps.http` libp2p RPC, so NAT'd primaries work. Requests up to 700 KiB, responses up to 5 MiB, 15 s upstream timeout, no websockets. The app sees `X-Forwarded-For/Host/Proto` and `X-Forwarded-Prefix: /a/`. |
| Sandbox | `--cap-drop ALL` (plus CHOWN, SETUID, SETGID, DAC_OVERRIDE, FOWNER, NET_BIND_SERVICE), `no-new-privileges`, memory/CPU/pid limits, 64 MiB tmpfs `/tmp`, published on loopback only, no restart policy (the supervisor decides what runs). |
+| Network | On a root worker the container joins the `c0mpute-apps` network behind an egress guard: it reaches the public internet but not private, loopback, link-local (cloud metadata) or CGNAT addresses, and not the host. On a non-root worker there is no guard (see below). |
+| Disk | State up to 1 GiB, and by default 2 GiB for state plus everything written inside the container. An app over the limit is stopped and restarts from its last snapshot. |
## Custom domains
@@ -86,10 +88,38 @@ c0mpute worker start --roles ...,apps
[apps]
max_apps = 4 # held at once (primary or standby)
max_memory_mb = 2048 # reserved by held apps together
+owners = [] # owner keys whose apps this node runs; empty = anyone's
+egress_guard = true # keep apps off your LAN, the host and metadata addresses
+require_egress_guard = false # true: run no apps unless that guard is in place
+max_disk_mb = 2048 # per app: state volume plus writes inside the container
```
Every gateway routes to apps whether or not it runs any.
+**Whose apps.** With `owners` empty the node runs containers for anyone on
+the network, and says so in its log at start.
+
+**Network.** The egress guard is a set of iptables rules on the apps bridge
+(`c0apps0`), the same ones `container.run` uses plus one that lets the app
+answer the node's own proxy. Only a root worker can add them. A non-root
+worker keeps apps on docker's default bridge, where a container **can reach
+your LAN, the host and cloud metadata addresses**; the node logs that once at
+start. Set `require_egress_guard = true` to run no apps in that case. An app
+that must reach a private address (a database on your LAN) needs
+`egress_guard = false`. The guard is IPv4 only.
+
+**Disk.** Every minute the node asks docker how much a running app has written
+inside its container and adds the size of its last state snapshot. Over
+`max_disk_mb` (0 = no per-app limit), or with state over 1 GiB, the app is
+stopped, the reason is recorded (`error` in `GET /apps/v1/apps/` on that
+node, and its log), and another holder takes over from the last snapshot. The
+stopped app's state volume is left on the node: what it holds past the last
+snapshot exists nowhere else, so removing it is the owner's decision. Apps are also stopped, and
+none started, while the docker disk has less free than the smaller of 20 GiB
+and 10% (the floor `container.run` keeps). Where docker can enforce a quota
+itself (overlay2 on xfs with project quotas) the container is also created
+with `--storage-opt size=`.
+
## Limits, honestly
- **One writer.** A stateful app has a single primary. A healed partition can
diff --git a/docs/hosting.md b/docs/hosting.md
index dcb7f666..2e3a2ee6 100644
--- a/docs/hosting.md
+++ b/docs/hosting.md
@@ -42,6 +42,48 @@ Git repositories ride on the same machinery: see [git hosting](git-hosting.md).
| Healing | A site with fewer than 3 live copies: each hosting node ranks the candidates (rendezvous hashing on site id + peer id). The top-ranked ones fetch a copy. A node that stops announcing stops counting after ~200 s and another node takes over. Above 5 copies the lowest-ranked holder lets go. A publisher seeding its own site is a source, not a copy. |
| Serving | Any gateway: local chunks when it holds them, otherwise fetched over libp2p from the fastest live holder (measured round-trip, round-robin among equally fast ones), failing over to the next and then to the DHT. Every chunk is checked against its hash. |
+### An announced copy counts once it is proven
+
+Announcing a copy is free: any key can gossip "I hold this site". A node never drops
+its own copy on announcements: it first fetches a random chunk from three other holders.
+
+The stricter rules below (an announcement does not fill a slot either, and a drop
+needs holders with standing) are **off by default** and turned on with
+`[hosting] require_proofs = true` (or `C0MPUTE_HOSTING_REQUIRE_PROOFS=1`). With them
+on, while a small site is still being copied its chunks are read constantly, which
+voids challenges, so extra hosts can take a copy and keep it for hours. Until that is
+settled, the default counts announcements for filling, as every version before did.
+
+- **Filling.** A holder counts toward the three once the node deciding has seen it pass
+ a challenge (a random chunk, fetched straight from that peer and checked against its
+ hash) within the last 6 hours, or the site's owner or payer signed a passing receipt
+ for it in that time. Only the nodes next in line for the site ask (the same top three
+ on every node, by the rendezvous ranking), in the background, at most 32 challenges
+ per repair round, 4 per peer and 5 per site. While they ask, the announced holders
+ keep their slots for one holder TTL (200 s), so slow or briefly unreachable honest
+ holders are asked again each round and a healthy site gains no extra copy. After
+ that, holders that still have not proven anything stop counting and the next-ranked
+ nodes take real copies. Keys that only announce can delay a site's copies by those
+ 200 s, once; new keys do not restart the clock. A node that has just started knows no
+ proofs: it works through its sites at the rate above and takes announced holders at
+ their word until a site's turn comes.
+- **Dropping.** A node lets a surplus copy go only when at least three other holders
+ prove their copy to it at that moment, and each of them has passed its challenges at
+ 3 or more separate times over at least 6 hours. A surplus copy therefore stays for at
+ least 6 hours after it appears.
+- **Relays.** A peer with no copy can pass a challenge by fetching the chunk from a real
+ holder while it is being asked. If the holder it fetches from is the challenger, the
+ challenger sees the chunk being read from its own store during the challenge and does
+ not count the answer. A holder also has a rule ready that refuses chunk fetches from
+ a peer that announces the same site whole (such a peer has no reason to fetch it);
+ it takes effect once the node's chunk server passes the requesting peer to hosting.
+ What remains: a peer can relay from a holder other than the challenger, under a
+ second key. That costs it the bandwidth of every challenge, for as long as it wants
+ to be counted. It can thin a site's honest copies over hours, down to one honest
+ holder (which cannot be talked into dropping by peers that hold nothing), but not to
+ none. A peer that really stores the site for 6 hours and then deletes it is not a
+ relay and is not caught by any of this.
+
## Limits
| | |
@@ -152,6 +194,65 @@ the rate (`--sequential` fetches the old way, one chunk at a time, to compare).
`--roles` without `hosting` means the node holds no copies; a gateway still serves
sites by proxying.
+## Site origins
+
+A browser treats everything under one scheme, host and port as one origin. Served at
+`/s//`, every site would share the gateway's origin with every other
+site and with the gateway's own APIs: one site's script could read another site's
+localStorage and call those APIs as the same origin. Two settings keep sites apart:
+
+```toml
+# ~/.config/c0mpute/config.toml
+[gateway]
+sandbox_shared_sites = true # the default
+site_host_suffix = "sites.example.com" # unset by default
+```
+
+**`sandbox_shared_sites`** (default `true`, `C0MPUTE_SANDBOX_SHARED_SITES=0` to turn off).
+Pages (HTML, SVG, XML) under `/s//` are sent with
+
+```
+Content-Security-Policy: sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads allow-pointer-lock allow-presentation
+Referrer-Policy: no-referrer
+```
+
+so each page runs in an origin of its own. Scripts, forms and downloads work.
+**localStorage, sessionStorage, IndexedDB, cookies and service workers do not** under
+`/s/`, and a page's `fetch()` of its own files is a cross-origin request, which is why
+every `/s/` response also carries `Access-Control-Allow-Origin: *` and
+`Cross-Origin-Resource-Policy: cross-origin` (everything under `/s/` is public and
+served without credentials). A site that needs storage or a service worker works on a
+custom domain or a per-site host, where none of these headers are sent. Turning the
+setting off restores the old behaviour, shared origin included.
+
+**`site_host_suffix`** (unset by default, `C0MPUTE_SITE_HOST_SUFFIX` overrides). With
+`site_host_suffix = "sites.example.com"`:
+
+- `https://.sites.example.com/` serves that site at `/`, as an ordinary page
+ with its own origin.
+- On such a host nothing else is served: every other gateway route (storage API, git,
+ swarm, `/p/`, `/hosting/v1/...`, another site's `/s/...`) is not reachable there, and
+ anything but GET or HEAD answers 404.
+- A browser opening `https:///s//?` is redirected (307,
+ cacheable for 5 minutes) to `https://.sites.example.com/?`.
+ Requests that are not page navigations (podcast apps, feed readers, `curl`, the CLI,
+ the `/p/` loader reading its ciphertext) are answered under `/s/` as before, with the
+ sandbox headers above.
+
+The operator has to provide, before setting it:
+
+1. **Wildcard DNS**: `*.sites.example.com` (A/AAAA or CNAME) pointing at the gateway.
+ Site ids are 20 hex characters, one DNS label.
+2. **Wildcard TLS**: a certificate for `*.sites.example.com` on the proxy in front of
+ `:7777` (a DNS-01 ACME challenge; HTTP-01 cannot issue wildcards). The redirect
+ always goes to `https://`.
+3. **The Host header** passed through by that proxy unchanged (the same requirement as
+ custom domains).
+
+Use a suffix that serves nothing else, ideally a separate registrable domain from the
+gateway's own name (as with user content elsewhere), so that cookies set for the
+gateway's domain are never sent to site hosts.
+
## Custom domains
1. Publish with `--domain blog.example.org` (repeatable; part of the signed record).
@@ -254,7 +355,9 @@ cannot demote anybody. The site's copies are kept up by the normal fill meanwhil
This proves the peer returned the right bytes for a chunk it could not predict, within the
timeout. It does not prove the peer keeps its own copy rather than relaying from another
-holder.
+holder; see "An announced copy counts once it is proven" above for what is done about
+relays and what is not. A challenger that sees the challenged chunk read from its own
+store while it waits for the answer signs no receipt for that challenge.
### Pricing and settlement
diff --git a/docs/node-safety.md b/docs/node-safety.md
index 4f206cff..fce1ed7d 100644
--- a/docs/node-safety.md
+++ b/docs/node-safety.md
@@ -34,10 +34,10 @@ open an issue; this page is meant to be checkable against the code.
Container workloads (video, game servers) start only pinned, well-known images, and only
when Docker or Podman already works on the box.
- Jobs run as child processes of the worker with CPU-thread and GPU limits.
- **This is not a sandbox.** A job runs with your user's permissions, so a
- bug in a workload runtime could read files your user can read. If that is
- not acceptable for your main machine, run the node in a VM, a container or
- on a spare box. It earns the same there.
+ **This is not a sandbox** (Blender renders are the exception, below). A job
+ runs with your user's permissions, so a bug in a workload runtime could read
+ files your user can read. If that is not acceptable for your main machine,
+ run the node in a VM, a container or on a spare box. It earns the same there.
- The node keeps itself up to date (`C0MPUTE_NO_AUTO_UPDATE=1` turns that
off). From v0.2.91 on, updates are verified against the same minisign key before
the binary is swapped; a node on an older version checks only the sha256 until it
@@ -45,6 +45,69 @@ open an issue; this page is meant to be checkable against the code.
- `c0mpute plugin install` still runs the plugin's own install script without a
signature check. Only install plugins you trust.
+## Blender renders
+
+`render` is a default role on any machine with `blender` installed. A buyer
+sends a `.blend` scene, and a scene is more than pixels: it names files by
+path (textures, movie clips, fonts, linked libraries, caches) and can ask
+Blender to write files. The node does two things about that
+(source: [`render.rs`](../node/crates/c0mpute-core/src/render.rs)).
+
+**Scene sanitisation, always, on every OS.** Blender starts with
+`--disable-autoexec`, so no Python in the scene runs. After the file loads and
+before the render, the node runs its own fixed script (`SANITIZE_EXPR`; nothing
+from the job is written into it) that:
+
+- empties every file path that is absolute or leaves the job directory: images,
+ movie clips, sounds, fonts, volumes, Alembic/USD cache files, sequencer
+ strips and their proxies, point caches, shader nodes that read a file;
+- deletes everything that came from a linked library outside the job directory;
+- removes compositor File Output nodes, turns Open Shading Language off (a
+ compiled shader can open any image by name), and points fluid caches and
+ every scene's output at the job directory;
+- turns off the render stamps that would write this machine's hostname or the
+ scene's path on your disk into the image.
+
+Data **packed** into the `.blend` is untouched, and an honest scene renders the
+same pixels as before. A scene that needed a file it did not pack renders with
+Blender's pink "missing" texture (or the built-in font, or without that strip):
+buyers must pack their assets. If the script cannot finish, the frame is not
+rendered at all.
+
+**A sandbox, on Linux, where bubblewrap works.** At start the node tries to run
+Blender inside [bubblewrap](https://github.com/containers/bubblewrap) (`bwrap`)
+and, if that works, renders that way: no network, its own process and hostname
+namespaces, a read-only view of the system directories and of Blender's own
+install directory, an empty private `/tmp` and home directory, and the job
+directory as the only thing it can write. GPU device nodes (`/dev/dri`,
+`/dev/nvidia*`) and driver libraries stay visible so GPU renders keep working;
+a GPU job the sandbox cannot serve (the probe checks) runs outside it.
+`c0mpute` logs which one it is at start. Install it with
+`sudo apt install bubblewrap`.
+
+| | sanitisation only | with bubblewrap |
+|---|---|---|
+| Read a file the scene names by path into a frame | blocked by the script, for every kind of path listed above | blocked twice (the file is not there) |
+| Write outside the job directory when rendering | blocked by the script, for the ways listed above (output path, File Output nodes, fluid caches) | not possible |
+| What Blender does while **opening** the file, before the script runs: open a linked library by path (its data is deleted before rendering), create an empty cache directory where the scene says | **not confined** | confined |
+| A bug in Blender's file readers turned into code execution | **not confined**: runs as your user | no network, no home directory, read-only system; it can still read world-readable system files and use the GPU |
+| CPU, RAM and disk the render uses | thread limit and timeout only | the same |
+| macOS, Windows | this column | not available |
+
+The sandbox is not used when `bwrap` is missing, when unprivileged user
+namespaces are switched off (Ubuntu 24.04 does this with AppArmor unless
+bubblewrap's profile is installed), or when Blender is a snap or sits behind a
+wrapper script the sandbox cannot follow. Choose with:
+
+```toml
+[render]
+sandbox = "auto" # default: bubblewrap when it works, else sanitisation only
+# sandbox = "require" # serve no render (and advertise none) without the sandbox
+# sandbox = "off" # never use bubblewrap
+```
+
+`C0MPUTE_RENDER_SANDBOX` overrides the file.
+
## Models it serves
- **House models** (on by default): the models Infernet lists at
@@ -53,6 +116,24 @@ open an issue; this page is meant to be checkable against the code.
`infernet house-models off`, or `--no-house-models` at install.
- **Uncensored house models are off unless you opt in** (`--uncensored-models`
at install, or `infernet house-models uncensored on`).
+- **Text fine-tunes** (`c0mpute infernet text serve`, opt-in) download the
+ buyer's base model from Hugging Face and load it. A worker does that only
+ for the models you list; a job naming another gets no bid:
+
+ ```toml
+ [text]
+ # The default: the two small models c0mpute's own commands and docs use.
+ base_models = ["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"]
+ # "org/*" allows every model of one organisation, "*" any model.
+ min_free_gb = 0 # free disk kept; 0 = the smaller of 20 GiB and 10% of the disk
+ ```
+
+ A job config may set only the keys c0mpute's clients send (`name`, `kind`,
+ `base_model`, `system_prompt`, `limits.timeout_s` and the hyperparameters
+ `lr`, `batch_size`, `epochs`, `seq_len`, `lora_r`, `max_steps`, `seed`, each
+ within a fixed range); anything else is refused before the trainer sees it.
+ The downloaded models live in Infernet's Hugging Face cache, which c0mpute
+ does not size or clean.
## Content policy (images and video)
diff --git a/docs/private-storage.md b/docs/private-storage.md
index 9a2fef75..12fa3063 100644
--- a/docs/private-storage.md
+++ b/docs/private-storage.md
@@ -107,10 +107,15 @@ open (any gateway serves `/s//o`), exactly like public objects.
and refuses older ones, so a node cannot quietly undo a revocation or a
volume write on a machine that saw the newer version. A machine that never
saw it cannot tell, and the browser loader does not keep this memory.
-- **The browser loader shares an origin** with the public sites that gateway
- serves. The key stays in the fragment and the decrypted page is sandboxed,
- but a gateway you do not trust can serve a different loader page; for
- anything sensitive, `c0mpute storage get` is the stronger path.
+- **The browser loader is served from the gateway's own origin**, the one
+ public sites under `/s/` are served from too. With the default
+ `[gateway] sandbox_shared_sites = true` those public pages are sandboxed
+ into origins of their own and cannot script the loader's origin; an
+ operator who turns that off puts them back on the loader's origin (see
+ [hosting](hosting.md#site-origins)). The key stays in the fragment and
+ the decrypted page is sandboxed either way, but a gateway you do not
+ trust can serve a different loader page; for anything sensitive,
+ `c0mpute storage get` is the stronger path.
- **Post-quantum.** Key wrapping is X25519. Data with a decades-long secrecy
horizon should wait for a hybrid ML-KEM wrap (the primitives are already in
the tree for `c0mpute dm`).
diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs
index fc0dc787..61988d43 100644
--- a/node/crates/c0mpute-cli/src/main.rs
+++ b/node/crates/c0mpute-cli/src/main.rs
@@ -646,10 +646,19 @@ enum PluginCmd {
List,
/// Install a plugin by id (from the c0mpute marketplace) or by URL.
///
+ /// An id runs the installer shipped in this release's signed plugins
+ /// archive. A URL is somebody else's script that nothing verifies, so it
+ /// needs --unverified.
+ ///
/// Examples:
/// c0mpute plugin install transcode
- /// c0mpute plugin install https://example.com/my-plugin/install.sh
- Install { target: String },
+ /// c0mpute plugin install --unverified https://example.com/my-plugin/install.sh
+ Install {
+ target: String,
+ /// Run an installer from a URL even though nothing verifies it.
+ #[arg(long)]
+ unverified: bool,
+ },
/// Enable a previously disabled plugin.
Enable { id: String },
/// Disable a plugin without uninstalling.
@@ -3016,13 +3025,107 @@ mod worker_stop_tests {
#[test]
fn plugin_targets_cannot_carry_a_command() {
- assert_eq!(resolve_plugin_target("tts").unwrap(), "https://c0mpute.com/plugins/tts/install.sh");
- assert_eq!(resolve_plugin_target("https://example.com/p/install.sh").unwrap(), "https://example.com/p/install.sh");
- for bad in ["x; rm -rf ~", "../x", "-o/tmp/x", "a b", "", "https://", "https://e.com/x; id", "https://e.com/$(id)", "https://e.com/x | sh"] {
+ assert_eq!(resolve_plugin_target("tts").unwrap(), PluginTarget::Id("tts".into()));
+ assert_eq!(resolve_plugin_target(" ad-generator-agent ").unwrap(), PluginTarget::Id("ad-generator-agent".into()));
+ assert_eq!(
+ resolve_plugin_target("https://example.com/p/install.sh").unwrap(),
+ PluginTarget::Url("https://example.com/p/install.sh".into())
+ );
+ for bad in ["x; rm -rf ~", "../x", "-o/tmp/x", "a b", "", "https://", "https://e.com/x; id", "https://e.com/$(id)", "https://e.com/x | sh", "http://e.com/install.sh", "a/b", "Tts"] {
assert!(resolve_plugin_target(bad).is_err(), "{bad}");
}
}
+ #[test]
+ fn a_plugin_url_needs_the_unverified_flag() {
+ let url = "https://example.com/p/install.sh";
+ // An id is always the signed installer, flag or not.
+ assert_eq!(plugin_install_plan("infernet", false).unwrap(), PluginPlan::Signed("infernet".into()));
+ assert_eq!(plugin_install_plan("infernet", true).unwrap(), PluginPlan::Signed("infernet".into()));
+ // A URL is refused unless the operator said --unverified.
+ let refused = plugin_install_plan(url, false).unwrap_err().to_string();
+ assert!(refused.contains("--unverified"), "{refused}");
+ assert_eq!(plugin_install_plan(url, true).unwrap(), PluginPlan::Unverified(url.into()));
+ // The flag does not make a bad target acceptable.
+ assert!(plugin_install_plan("http://example.com/p/install.sh", true).is_err());
+ assert!(plugin_install_plan("x; id", true).is_err());
+ }
+
+ #[test]
+ fn plugin_install_parses_the_unverified_flag() {
+ let parse = |args: &[&str]| Cli::try_parse_from(args.iter().copied()).map(|c| c.command);
+ match parse(&["c0mpute", "plugin", "install", "transcode"]).unwrap() {
+ Some(Cmd::Plugin { cmd: PluginCmd::Install { target, unverified } }) => {
+ assert_eq!(target, "transcode");
+ assert!(!unverified);
+ }
+ other => panic!("parsed as {other:?}"),
+ }
+ match parse(&["c0mpute", "plugin", "install", "--unverified", "https://example.com/i.sh"]).unwrap() {
+ Some(Cmd::Plugin { cmd: PluginCmd::Install { target, unverified } }) => {
+ assert_eq!(target, "https://example.com/i.sh");
+ assert!(unverified);
+ }
+ other => panic!("parsed as {other:?}"),
+ }
+ }
+
+ #[test]
+ fn signed_assets_come_from_this_binarys_own_release() {
+ let tag = own_release_tag();
+ assert_eq!(tag, format!("v{}", env!("CARGO_PKG_VERSION")));
+ assert_eq!(
+ release_asset_url("v0.3.65", PLUGINS_ARCHIVE),
+ "https://github.com/profullstack/c0mpute/releases/download/v0.3.65/c0mpute-plugins.tar.gz"
+ );
+ assert_eq!(
+ release_asset_url(&tag, TUI_ARCHIVE),
+ format!("https://github.com/profullstack/c0mpute/releases/download/{tag}/c0mpute-tui.tar.gz")
+ );
+ }
+
+ #[test]
+ fn a_tui_branch_override_is_a_plain_branch_name() {
+ assert_eq!(
+ tui_branch_url("master").unwrap(),
+ "https://github.com/profullstack/c0mpute/archive/refs/heads/master.tar.gz"
+ );
+ assert!(tui_branch_url("feat/tui-2.0_x").is_ok());
+ for bad in ["", "-x", "a b", "a'b", "../x", "x/../y", "a;b", "$(id)", "a?b", "a#b"] {
+ assert!(tui_branch_url(bad).is_err(), "{bad}");
+ }
+ }
+
+ #[test]
+ fn a_scratch_dir_is_removed_on_drop() {
+ let base = tempfile::tempdir().unwrap();
+ let dir = base.path().join("scratch");
+ std::fs::create_dir_all(dir.join("stale")).unwrap();
+ {
+ let s = ScratchDir::create(&dir).unwrap();
+ assert!(s.path().is_dir());
+ assert!(!s.path().join("stale").exists(), "an existing directory is emptied first");
+ }
+ assert!(!dir.exists());
+ }
+
+ #[test]
+ fn a_signed_asset_is_refused_unless_it_is_this_file_of_this_release() {
+ // The verifier `fetch_signed_release_asset` calls. A real published
+ // signature (v0.3.64, linux x86_64) over bytes that are not the
+ // artifact must not verify, whatever name and tag are asked for.
+ let sig = "untrusted comment: signature from minisign secret key\n\
+ RUQqOew3QbWhATNMJDo0v4saAncRTUdP557zY9X/DggMKQNkZ/id8b+vpXGezphgXnVM2+X7cb1TCB8U0gGJHTejkJBcWiFD+gw=\n\
+ trusted comment: timestamp:1794555478 version:v0.3.64 file:c0mpute-linux-x86_64.tar.gz\n\
+ Lh5cSGzX/jrC/n/bobJZWpYxtHeq9unNog/A/UC6Wonf2RWPCUqrWTH12osx3NiUqbHoIo6oWyuDG1g3EklxDg==\n";
+ let verify = |bytes: &[u8], asset: &str, tag: &str| {
+ c0mpute_update::verify_release_signature(c0mpute_update::RELEASE_PUBKEY, bytes, sig, asset, Some(tag))
+ };
+ assert!(verify(b"not the artifact", "c0mpute-linux-x86_64.tar.gz", "v0.3.64").is_err());
+ assert!(verify(b"not the artifact", PLUGINS_ARCHIVE, &own_release_tag()).is_err());
+ assert!(verify(b"", TUI_ARCHIVE, "v0.3.64").is_err());
+ }
+
#[test]
fn rpc_sharding_is_off_unless_asked_for() {
assert!(!rpc_sharding_opted_in(None));
@@ -5221,7 +5324,7 @@ fn run_plugin(cmd: PluginCmd) -> Result<()> {
}
Ok(())
}
- PluginCmd::Install { target } => install_plugin(&target),
+ PluginCmd::Install { target, unverified } => install_plugin(&target, unverified),
PluginCmd::Enable { id } => {
println!("[stub] enable {id}");
Ok(())
@@ -5280,20 +5383,67 @@ fn preset_to_spec(preset: &str) -> Result {
/// Resolve a `c0mpute plugin install ` argument and dispatch.
///
/// Accepts either:
-/// - a plugin **id** registered in the c0mpute marketplace, in which
-/// case we chain to `https://c0mpute.com/plugins//install.sh`
-/// - or any **install.sh URL** (third-party plugins published at
-/// their own URL — e.g. a GitHub raw URL)
-///
-/// Both flows pipe `curl -fsSL ` into `sh`. Future: minisign
-/// signature verification before execution (per DIP-0006).
-fn install_plugin(target: &str) -> Result<()> {
- let url = resolve_plugin_target(target)?;
+/// - a plugin **id**: its `plugins//install.sh` is taken from the
+/// `c0mpute-plugins.tar.gz` asset of this binary's own release, after
+/// the archive's minisign signature verified against the key compiled
+/// into this binary
+/// - or any **https install.sh URL** (third-party plugins published at
+/// their own URL). Nothing verifies those, so they need `--unverified`.
+fn install_plugin(target: &str, unverified: bool) -> Result<()> {
+ match plugin_install_plan(target, unverified)? {
+ PluginPlan::Signed(id) => install_signed_plugin(&id),
+ PluginPlan::Unverified(url) => {
+ eprintln!("! {url} is not part of a c0mpute release: nothing verifies this script before it runs (--unverified)");
+ run_unverified_installer(&url)
+ }
+ }
+}
+
+/// What `plugin install` will do for a target, before anything is fetched.
+#[derive(Debug, PartialEq)]
+enum PluginPlan {
+ /// Run this id's installer out of the signed release archive.
+ Signed(String),
+ /// Run the script at this URL; only with `--unverified`.
+ Unverified(String),
+}
+
+fn plugin_install_plan(target: &str, unverified: bool) -> Result {
+ match resolve_plugin_target(target)? {
+ PluginTarget::Id(id) => Ok(PluginPlan::Signed(id)),
+ PluginTarget::Url(url) if unverified => Ok(PluginPlan::Unverified(url)),
+ PluginTarget::Url(url) => anyhow::bail!(
+ "{url} is not part of a c0mpute release, so nothing verifies it before it runs. \
+ To run it anyway: c0mpute plugin install --unverified {url}"
+ ),
+ }
+}
+
+/// Run plugin `id`'s installer from the signed plugins archive of this release.
+fn install_signed_plugin(id: &str) -> Result<()> {
+ let tag = own_release_tag();
+ let scratch = ScratchDir::create(&c0mpute_home()?.join(format!("plugin-install-{}", std::process::id())))?;
+ let archive = fetch_signed_release_asset(PLUGINS_ARCHIVE, scratch.path())?;
+ untar(&archive, scratch.path())?;
+ let script = scratch.path().join("plugins").join(id).join("install.sh");
+ if !script.is_file() {
+ anyhow::bail!("plugin '{id}' has no installer in the signed plugins archive of c0mpute {tag}");
+ }
+ println!("→ installing plugin {id} (signed installer from c0mpute {tag})");
+ let status = Command::new("sh").arg(&script).status()?;
+ if !status.success() {
+ anyhow::bail!("plugin install failed (exit {status})");
+ }
+ Ok(())
+}
+
+/// The old flow, for `--unverified` URLs only: pipe `curl -fsSL ` into `sh`.
+fn run_unverified_installer(url: &str) -> Result<()> {
println!("→ installing plugin via {url}");
// No shell string: the URL is one curl argument, and curl's output is
// sh's stdin, so nothing in the target can add a command.
let mut curl = std::process::Command::new("curl")
- .args(["-fsSL", "--", url.as_str()])
+ .args(["-fsSL", "--", url])
.stdout(std::process::Stdio::piped())
.spawn()?;
let script = curl.stdout.take().context("curl has no stdout")?;
@@ -5308,22 +5458,129 @@ fn install_plugin(target: &str) -> Result<()> {
Ok(())
}
+// Release assets the installer (scripts/install.sh) and this CLI install from.
+// Packed by scripts/pack-release-extras.sh and signed with the release key in
+// .github/workflows/release.yml, like the binaries.
+const PLUGINS_ARCHIVE: &str = "c0mpute-plugins.tar.gz";
+const TUI_ARCHIVE: &str = "c0mpute-tui.tar.gz";
+
+/// The release tag this binary was built from (`v`).
+fn own_release_tag() -> String {
+ format!("v{}", env!("CARGO_PKG_VERSION"))
+}
+
+/// Where `asset` of release `tag` is downloaded from.
+fn release_asset_url(tag: &str, asset: &str) -> String {
+ format!("https://github.com/profullstack/c0mpute/releases/download/{tag}/{asset}")
+}
+
+/// `~/.c0mpute`: scratch for installs lives here, not in the shared /tmp where
+/// another local user could pre-create the directory and plant files in it.
+fn c0mpute_home() -> Result {
+ let home = std::env::var_os("HOME")
+ .or_else(|| std::env::var_os("USERPROFILE"))
+ .ok_or_else(|| anyhow::anyhow!("HOME not set"))?;
+ Ok(PathBuf::from(home).join(".c0mpute"))
+}
+
+/// A directory that is emptied and created now and removed on drop.
+struct ScratchDir(PathBuf);
+
+impl ScratchDir {
+ fn create(dir: &std::path::Path) -> Result {
+ let _ = std::fs::remove_dir_all(dir);
+ std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
+ Ok(Self(dir.to_path_buf()))
+ }
+
+ fn path(&self) -> &std::path::Path {
+ &self.0
+ }
+}
+
+impl Drop for ScratchDir {
+ fn drop(&mut self) {
+ let _ = std::fs::remove_dir_all(&self.0);
+ }
+}
+
+/// Download `url` to `dest` over https only (redirects included).
+fn curl_to_file(url: &str, dest: &std::path::Path) -> Result<()> {
+ let status = Command::new("curl")
+ .args(["-fsSL", "--proto", "=https", "--proto-redir", "=https", "-o"])
+ .arg(dest)
+ .args(["--", url])
+ .status()
+ .context("run curl")?;
+ if !status.success() {
+ anyhow::bail!("download failed ({status}): {url}");
+ }
+ Ok(())
+}
+
+/// Download `asset` of this binary's own release into `dir` and verify it the
+/// way self-update verifies a binary: the minisign signature must check out
+/// against the key compiled into this binary and its signed comment must name
+/// this asset and this release. Anything else is an error; there is no
+/// unverified fallback. Returns the verified file.
+fn fetch_signed_release_asset(asset: &str, dir: &std::path::Path) -> Result {
+ let tag = own_release_tag();
+ let url = release_asset_url(&tag, asset);
+ let file = dir.join(asset);
+ let sig = dir.join(format!("{asset}.minisig"));
+ curl_to_file(&url, &file)
+ .with_context(|| format!("{asset} is not available for c0mpute {tag} (a build that is not a published release has none)"))?;
+ curl_to_file(&format!("{url}.minisig"), &sig)
+ .with_context(|| format!("no signature for {asset} on c0mpute {tag}; refusing to use it"))?;
+ let bytes = std::fs::read(&file).with_context(|| format!("read {}", file.display()))?;
+ let sig_text = std::fs::read_to_string(&sig).with_context(|| format!("read {}", sig.display()))?;
+ c0mpute_update::verify_release_signature(c0mpute_update::RELEASE_PUBKEY, &bytes, &sig_text, asset, Some(&tag))?;
+ Ok(file)
+}
+
+/// Unpack a (verified) `.tar.gz` into `dir`.
+fn untar(archive: &std::path::Path, dir: &std::path::Path) -> Result<()> {
+ let status = Command::new("tar")
+ .arg("-xzf")
+ .arg(archive)
+ .arg("-C")
+ .arg(dir)
+ .status()
+ .context("run tar")?;
+ if !status.success() {
+ anyhow::bail!("could not unpack {} ({status})", archive.display());
+ }
+ Ok(())
+}
+
/// Whether `C0MPUTE_RPC_SHARDING` asks for the llama.cpp RPC slice.
fn rpc_sharding_opted_in(value: Option<&str>) -> bool {
matches!(value.map(str::trim), Some("1" | "true" | "yes" | "on"))
}
+/// What a `plugin install` argument names.
+#[derive(Debug, PartialEq)]
+enum PluginTarget {
+ /// A plugin id from this repo's `plugins/` directory.
+ Id(String),
+ /// Somebody's own installer script.
+ Url(String),
+}
+
/// A plugin id is lowercase letters, digits and dashes; anything else must be
-/// an http(s) URL with no whitespace, quotes or shell metacharacters.
-fn resolve_plugin_target(target: &str) -> Result {
+/// an https URL with no whitespace, quotes or shell metacharacters.
+fn resolve_plugin_target(target: &str) -> Result {
let t = target.trim();
- if t.starts_with("http://") || t.starts_with("https://") {
+ if t.starts_with("http://") {
+ anyhow::bail!("plugin URLs must be https: {t}");
+ }
+ if t.starts_with("https://") {
let rest = t.split_once("://").map(|(_, r)| r).unwrap_or("");
let bad = |c: char| c.is_whitespace() || c.is_control() || matches!(c, '"' | '\'' | '`' | '$' | ';' | '|' | '<' | '>' | '\\');
if rest.is_empty() || rest.starts_with('/') || t.chars().any(bad) {
anyhow::bail!("not a plugin URL: {t}");
}
- return Ok(t.to_string());
+ return Ok(PluginTarget::Url(t.to_string()));
}
let id_ok = !t.is_empty()
&& t.len() <= 64
@@ -5332,10 +5589,9 @@ fn resolve_plugin_target(target: &str) -> Result {
if !id_ok {
anyhow::bail!("plugin id must be lowercase letters, digits and dashes (or a full https URL): {t}");
}
- // Treat anything else as a plugin id and resolve through the
- // c0mpute marketplace. The route at c0mpute.com/plugins//install.sh
- // serves the manifest-checked-in install script for that plugin.
- Ok(format!("https://c0mpute.com/plugins/{t}/install.sh"))
+ // Anything else is a plugin id: its installer comes out of the signed
+ // plugins archive of this release (see install_signed_plugin).
+ Ok(PluginTarget::Id(t.to_string()))
}
// ────────────────────────────────────────────────────────────────────────
@@ -5605,10 +5861,12 @@ fn run_menu() -> Result<()> {
/// `c0mpute tui` — launch the terminal UI, installing it on demand the first
/// time. The TUI is a Bun app (apps/tui, on @profullstack/hqtui), not part of
-/// the Rust binary, so `c0mpute tui` fetches the source, runs `bun install`,
-/// and writes a launcher — the user never has to know about the underlying
-/// `c0mpute-tui` binary. After an update it reinstalls the TUI from this
-/// release's tag, so the CLI and the TUI move together.
+/// the Rust binary, so `c0mpute tui` fetches the source (the signed
+/// `c0mpute-tui.tar.gz` of this release, verified with the release key
+/// compiled into this binary), runs `bun install`, and writes a launcher —
+/// the user never has to know about the underlying `c0mpute-tui` binary.
+/// After an update it reinstalls the TUI from this release, so the CLI and
+/// the TUI move together.
fn run_tui(args: &[String]) -> Result<()> {
let stale = installed_tui_version().is_some_and(|v| tui_is_stale(&v));
if tui_launcher().is_none() {
@@ -5685,57 +5943,87 @@ fn ensure_bun() -> Option {
mise_which(&mise).or_else(|| which_on_path("bun"))
}
+/// The GitHub tarball of a branch, for the `C0MPUTE_TUI_REF` developer
+/// override. Nothing signs a branch, so this is the unverified path.
+fn tui_branch_url(branch: &str) -> Result {
+ let ok = !branch.is_empty()
+ && !branch.starts_with('-')
+ && !branch.contains("..")
+ && branch
+ .chars()
+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '/'));
+ if !ok {
+ anyhow::bail!("C0MPUTE_TUI_REF is not a branch name: {branch}");
+ }
+ Ok(format!("https://github.com/profullstack/c0mpute/archive/refs/heads/{branch}.tar.gz"))
+}
+
+/// Fetch the TUI source into `scratch` and return its directory (holding
+/// `src/`, `package.json`, `tsconfig.json`). Normally that is the signed
+/// `c0mpute-tui.tar.gz` of this binary's own release, so the TUI matches the
+/// CLI and is verified like it. `C0MPUTE_TUI_REF=` takes a branch
+/// instead, unverified, with a warning.
+fn fetch_tui_source(scratch: &std::path::Path) -> Result {
+ if let Some(branch) = std::env::var("C0MPUTE_TUI_REF").ok().filter(|b| !b.is_empty()) {
+ let url = tui_branch_url(&branch)?;
+ eprintln!("! C0MPUTE_TUI_REF={branch}: installing the terminal UI UNVERIFIED from that branch");
+ let tarball = scratch.join("branch.tar.gz");
+ curl_to_file(&url, &tarball)?;
+ untar(&tarball, scratch)?;
+ // The tarball has one top-level directory, -.
+ for entry in std::fs::read_dir(scratch)?.flatten() {
+ let tui = entry.path().join("apps").join("tui");
+ if tui.is_dir() {
+ return Ok(tui);
+ }
+ }
+ anyhow::bail!("TUI source not found on branch {branch}");
+ }
+ let archive = fetch_signed_release_asset(TUI_ARCHIVE, scratch).with_context(|| {
+ "could not get the signed terminal UI (developers: C0MPUTE_TUI_REF= installs it unverified from a branch)"
+ })?;
+ untar(&archive, scratch)?;
+ Ok(scratch.join("tui"))
+}
+
fn install_tui() -> Result<()> {
println!("c0mpute tui: installing the terminal UI for v{}…", env!("CARGO_PKG_VERSION"));
- let home = std::env::var("HOME")
- .map(PathBuf::from)
- .map_err(|_| anyhow::anyhow!("HOME not set"))?;
+ let c0mpute_dir = c0mpute_home()?;
let bun = ensure_bun().ok_or_else(|| {
anyhow::anyhow!(
- "bun is required for the TUI and couldn't be installed via mise. \
- Install it, then rerun `c0mpute tui`:\n curl -fsSL https://bun.sh/install | bash"
+ "bun is required for the TUI. Re-run the c0mpute installer (it installs a pinned, \
+ checksum-verified bun), or install bun yourself (https://bun.sh/docs/installation), \
+ then rerun `c0mpute tui`"
)
})?;
- let tui_dir = home.join(".c0mpute/tui");
- let wrapper = home.join(".c0mpute/bin/c0mpute-tui");
+ let tui_dir = c0mpute_dir.join("tui");
+ let wrapper = c0mpute_dir.join("bin/c0mpute-tui");
// Scratch under the user's own ~/.c0mpute, not a fixed name in the shared
// /tmp: there another local user could pre-create the directory and plant
- // an `apps/tui` that this script would install and run.
- let tmp = home.join(format!(".c0mpute/tui-install-{}", std::process::id()));
- // This release's tag, so the TUI matches the CLI; master if the tag is
- // not there (a dev build). C0MPUTE_TUI_REF picks a branch instead.
- let archive = match std::env::var("C0MPUTE_TUI_REF") {
- Ok(branch) if !branch.is_empty() => format!("https://github.com/profullstack/c0mpute/archive/refs/heads/{branch}.tar.gz"),
- _ => format!("https://github.com/profullstack/c0mpute/archive/refs/tags/v{}.tar.gz", env!("CARGO_PKG_VERSION")),
- };
- let fallback = "https://github.com/profullstack/c0mpute/archive/refs/heads/master.tar.gz";
-
- // Fetch the repo, copy apps/tui into ~/.c0mpute/tui, and materialise deps.
- let script = format!(
- "set -e\n\
- rm -rf '{tmp}' && mkdir -p '{tmp}'\n\
- curl -fsSL '{archive}' | tar -xz -C '{tmp}' || {{ rm -rf '{tmp}' && mkdir -p '{tmp}' && curl -fsSL '{fallback}' | tar -xz -C '{tmp}'; }}\n\
- src=$(find '{tmp}' -type d -path '*/apps/tui' | head -1)\n\
- [ -d \"$src/src\" ] || {{ echo 'TUI source not found' >&2; exit 1; }}\n\
- rm -rf '{tui}' && mkdir -p '{tui}'\n\
- cp -R \"$src/src\" \"$src/package.json\" \"$src/tsconfig.json\" '{tui}/'\n\
- cd '{tui}' && '{bun}' install --production --no-save >/dev/null 2>&1\n\
- rm -rf '{tmp}'\n",
- tmp = tmp.display(),
- archive = archive,
- fallback = fallback,
- tui = tui_dir.display(),
- bun = bun.display(),
- );
- let ok = Command::new("sh")
- .arg("-c")
- .arg(&script)
+ // an `apps/tui` that this would install and run.
+ let scratch = ScratchDir::create(&c0mpute_dir.join(format!("tui-install-{}", std::process::id())))?;
+ let src = fetch_tui_source(scratch.path())?;
+ if !src.join("src").is_dir() || !src.join("package.json").is_file() {
+ anyhow::bail!("TUI source not found in the downloaded archive");
+ }
+
+ // Swap the source in (same filesystem: a rename), then materialise deps.
+ if tui_dir.exists() {
+ std::fs::remove_dir_all(&tui_dir).with_context(|| format!("remove {}", tui_dir.display()))?;
+ }
+ std::fs::rename(&src, &tui_dir).with_context(|| format!("install {}", tui_dir.display()))?;
+ drop(scratch);
+ let ok = Command::new(&bun)
+ .args(["install", "--production", "--no-save"])
+ .current_dir(&tui_dir)
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false);
if !ok {
- anyhow::bail!("failed to fetch/build the TUI (need network + bun)");
+ anyhow::bail!("failed to install the TUI's dependencies (need network + bun)");
}
// Launcher — runs the Bun source. Kept as source rather than a
@@ -5747,7 +6035,7 @@ fn install_tui() -> Result<()> {
BUN=\"$(command -v bun 2>/dev/null || true)\"\n\
[ -z \"$BUN\" ] && [ -x \"$HOME/.bun/bin/bun\" ] && BUN=\"$HOME/.bun/bin/bun\"\n\
[ -z \"$BUN\" ] && command -v mise >/dev/null 2>&1 && BUN=\"$(mise which bun 2>/dev/null || true)\"\n\
- [ -n \"$BUN\" ] || {{ echo 'bun not found; run: mise use --global bun@latest' >&2; exit 1; }}\n\
+ [ -n \"$BUN\" ] || {{ echo 'bun not found; re-run the c0mpute installer (it installs a pinned bun)' >&2; exit 1; }}\n\
exec \"$BUN\" run \"{}/src/index.ts\" \"$@\"\n",
tui_dir.display()
);
diff --git a/node/crates/c0mpute-core/src/apps/mod.rs b/node/crates/c0mpute-core/src/apps/mod.rs
index ae5bc151..64fb7863 100644
--- a/node/crates/c0mpute-core/src/apps/mod.rs
+++ b/node/crates/c0mpute-core/src/apps/mod.rs
@@ -64,6 +64,15 @@ pub const RPC_LOGS: &str = "apps.logs";
pub const RPC_RECORD: &str = "apps.record";
/// Apps per announcement, so one message stays well under gossipsub's limit.
const APPS_PER_MESSAGE: usize = 100;
+/// `[apps] max_disk_mb` when the operator sets none: the 1 GiB of state an
+/// app may keep ([`snapshot::MAX_STATE_BYTES`]) plus as much again written
+/// inside its container.
+pub const DEFAULT_MAX_DISK_MB: u64 = 2048;
+/// How often a running app's disk use is measured (docker walks the
+/// container's writable layer to answer).
+const DISK_CHECK_EVERY: Duration = Duration::from_secs(60);
+/// Logged once per run by a node that runs apps without the egress guard.
+const OPEN_EGRESS_WARNING: &str = "apps: no egress guard on this node (it needs a root worker, iptables and [apps] egress_guard on): app containers can reach this machine's LAN, the host itself and cloud metadata addresses. Set [apps] require_egress_guard = true to run apps only behind the guard";
/// How often things happen. Tests shrink these.
#[derive(Clone, Debug)]
@@ -144,6 +153,15 @@ struct Local {
last_snap: Option,
#[serde(skip)]
failures: u32,
+ /// Size of the state directory at the last snapshot (its tar), or one
+ /// byte over the cap when a snapshot was refused for size.
+ #[serde(skip)]
+ state_bytes: u64,
+ #[serde(skip)]
+ disk_checked: Option,
+ /// Why this node last stopped the app itself (over its disk limit).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ error: Option,
}
impl Local {
@@ -162,6 +180,9 @@ impl Local {
busy: false,
last_snap: None,
failures: 0,
+ state_bytes: 0,
+ disk_checked: None,
+ error: None,
}
}
@@ -207,6 +228,12 @@ struct Inner {
max_apps: usize,
max_memory_mb: u32,
owners: Vec,
+ egress_guard: bool,
+ require_egress_guard: bool,
+ /// Most disk one app may use (state plus writable layer); 0 = no limit.
+ max_disk_bytes: u64,
+ /// The "apps can reach the LAN" warning has been logged this run.
+ warned_open_egress: std::sync::atomic::AtomicBool,
timing: Timing,
x25519: crypto::KeyPair,
hosting: Option,
@@ -236,6 +263,12 @@ pub struct Options {
pub dir: Option,
/// Run only apps signed by these owner keys (hex ed25519); empty = anyone's.
pub owners: Vec,
+ /// `[apps] egress_guard`, `require_egress_guard` and `max_disk_mb`.
+ /// `None` reads the node's config file, so a caller that predates these
+ /// settings still gets what the operator wrote.
+ pub egress_guard: Option,
+ pub require_egress_guard: Option,
+ pub max_disk_mb: Option,
pub timing: Timing,
/// To read images published as hosting sites.
pub hosting: Option,
@@ -254,6 +287,9 @@ impl Default for Options {
max_memory_mb: 2048,
dir: None,
owners: Vec::new(),
+ egress_guard: None,
+ require_egress_guard: None,
+ max_disk_mb: None,
timing: Timing::default(),
hosting: None,
dns: dns::Verifier::default(),
@@ -277,6 +313,10 @@ pub struct AppInfo {
pub snapshot: Option,
pub domains: Vec,
pub path: String,
+ /// Why the answering node last stopped this app itself (over its disk
+ /// limit), when it holds a copy and did.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub error: Option,
}
/// The apps service: one per node. Cheap to clone.
@@ -359,21 +399,49 @@ impl Apps {
.into_iter()
.filter(|(_, r)| r.verify())
.collect();
+ // Settings the caller did not pass come from the node's config file.
+ let file = match (opts.egress_guard, opts.require_egress_guard, opts.max_disk_mb) {
+ (Some(_), Some(_), Some(_)) => crate::config::AppsConfig::default(),
+ _ if !opts.hosts => crate::config::AppsConfig::default(),
+ _ => crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default().apps,
+ };
+ let require_egress_guard = opts.require_egress_guard.unwrap_or(file.require_egress_guard);
+ let egress_guard = opts.egress_guard.unwrap_or(file.egress_guard) || require_egress_guard;
+ let max_disk_mb = opts.max_disk_mb.unwrap_or(file.max_disk_mb);
+ let mut hosts = opts.hosts;
+ if hosts && require_egress_guard && !crate::container_run::is_root() {
+ // The guard is iptables rules, which only a root worker can add.
+ warn!("apps: [apps] require_egress_guard is set and this worker is not root, so the egress guard cannot be put in place; running no apps");
+ hosts = false;
+ }
+ if hosts && opts.owners.is_empty() {
+ warn!("apps: [apps] owners is empty, so this node runs containers for any owner on the network; list owner keys there to run only theirs");
+ }
+ // Known already at start for a non-root worker or a guard turned
+ // off; a root worker finds out when docker and iptables are asked.
+ let open_egress = hosts && (!egress_guard || !crate::container_run::is_root());
+ if open_egress {
+ warn!("{OPEN_EGRESS_WARNING}");
+ }
let a = Apps(Arc::new(Inner {
net: net.clone(),
store,
me,
- hosts: opts.hosts,
+ hosts,
max_apps: opts.max_apps.max(1),
max_memory_mb: opts.max_memory_mb,
owners: opts.owners,
+ egress_guard,
+ require_egress_guard,
+ max_disk_bytes: max_disk_mb.saturating_mul(1 << 20),
+ warned_open_egress: std::sync::atomic::AtomicBool::new(open_egress),
timing: opts.timing,
x25519,
hosting: opts.hosting,
dns: opts.dns,
records: RwLock::new(records),
nodes: RwLock::new(HashMap::new()),
- local: RwLock::new(if opts.hosts { local } else { HashMap::new() }),
+ local: RwLock::new(if hosts { local } else { HashMap::new() }),
owned: RwLock::new(persisted.owned),
sealed_to: RwLock::new(HashMap::new()),
released: RwLock::new(HashMap::new()),
@@ -911,6 +979,7 @@ impl Apps {
} else if let Some(port) = host_port {
let ok = runtime::healthy(port, &rec.health_path).await;
let alive = ok || runtime::running(&id).await;
+ let mut disk_due = false;
let mut local = self.0.local.write().await;
if let Some(l) = local.get_mut(&id) {
if l.healthy != ok {
@@ -921,9 +990,15 @@ impl Apps {
warn!(app = %id, "apps: container stopped; restarting");
l.running_seq = None;
l.failures += 1;
+ } else if l.disk_checked.is_none_or(|t| t.elapsed() >= DISK_CHECK_EVERY) {
+ l.disk_checked = Some(Instant::now());
+ disk_due = true;
}
}
drop(local);
+ if disk_due {
+ self.spawn_disk_check(id.clone());
+ }
if alive && snap_due {
self.spawn_snapshot(id);
}
@@ -1159,6 +1234,15 @@ impl Apps {
};
let secret_env = secrets.as_ref().map(|s| s.env.clone()).unwrap_or_default();
let _ = runtime::remove(id).await;
+ let create = runtime::CreateOpts {
+ network: self.app_network().await?,
+ layer_quota_mb: runtime::layer_quota_mb(self.0.max_disk_bytes).await,
+ };
+ // Same floor as the disk watchdog: an app it would stop is not started.
+ let (free, floor) = runtime::host_free().await;
+ if free < floor {
+ anyhow::bail!("this node's docker disk is down to {} GiB free (it keeps {} GiB)", free >> 30, floor >> 30);
+ }
if let Some(state_path) = &rec.state_path {
let secrets = secrets.context("no secrets for a stateful app")?;
let key = secrets.key()?;
@@ -1188,18 +1272,83 @@ impl Apps {
runtime::restore_state(id, &tar).await?;
info!(app = %id, epoch = s.epoch, n = s.n, bytes = tar.len(), path = %state_path, "apps: state restored");
}
- runtime::create(&rec, &secret_env).await?;
+ runtime::create(&rec, &secret_env, &create).await?;
} else {
- runtime::create(&rec, &secret_env).await?;
+ runtime::create(&rec, &secret_env, &create).await?;
}
} else {
- runtime::create(&rec, &secret_env).await?;
+ runtime::create(&rec, &secret_env, &create).await?;
}
runtime::start(id).await?;
let port = runtime::host_port(id, rec.port).await?;
Ok(port)
}
+ /// The docker network this node's app containers join: the guarded apps
+ /// network when the egress guard is in place. Without it (a non-root
+ /// worker, or `egress_guard = false`) apps stay on docker's default
+ /// bridge, said once per run; with `require_egress_guard` nothing starts.
+ async fn app_network(&self) -> Result> {
+ let guarded = self.0.egress_guard && runtime::ensure_guarded_network().await;
+ let network = runtime::network_for(guarded, self.0.require_egress_guard).map_err(anyhow::Error::msg)?;
+ if network.is_none() && !self.0.warned_open_egress.swap(true, std::sync::atomic::Ordering::Relaxed) {
+ warn!("{OPEN_EGRESS_WARNING}");
+ }
+ Ok(network)
+ }
+
+ /// Measure one running app's disk use and stop it when it is over its
+ /// limit or the host is under its free-space floor.
+ fn spawn_disk_check(&self, id: String) {
+ let a = self.clone();
+ tokio::spawn(async move {
+ let Some(state) = a.0.local.read().await.get(&id).map(|l| l.state_bytes) else {
+ return;
+ };
+ // A size docker cannot report counts as nothing written: an app
+ // is never stopped on a failed measurement.
+ let layer = runtime::layer_bytes(&id).await.unwrap_or(0);
+ let (free, floor) = runtime::host_free().await;
+ if let Some(stop) = runtime::disk_verdict(layer, state, a.0.max_disk_bytes, free, floor) {
+ a.stop_for_disk(&id, stop).await;
+ }
+ });
+ }
+
+ /// Stop an app this node runs for the disk it uses: record why, give the
+ /// primary role up so another holder can take over from the last
+ /// snapshot, and remove the container (and the state volume when the
+ /// state itself is over the cap).
+ async fn stop_for_disk(&self, id: &str, stop: runtime::DiskStop) {
+ {
+ let mut local = self.0.local.write().await;
+ let Some(l) = local.get_mut(id) else { return };
+ // Mid-start or mid-snapshot: the next check decides.
+ if l.busy || l.role != AppRole::Primary {
+ return;
+ }
+ warn!(app = %id, reason = %stop.reason, "apps: stopping the app");
+ l.error = Some(format!("stopped at {}: {}", now_secs(), stop.reason));
+ l.role = AppRole::Standby;
+ l.running_seq = None;
+ l.host_port = None;
+ l.healthy = false;
+ l.ready_seq = None;
+ l.failures += 1;
+ l.disk_checked = None;
+ if stop.drop_state {
+ l.volume_epoch = None;
+ l.state_bytes = 0;
+ }
+ }
+ let _ = runtime::remove(id).await;
+ if stop.drop_state {
+ let _ = runtime::remove_volume(id).await;
+ }
+ self.persist().await;
+ self.announce().await;
+ }
+
async fn snap_manifest(&self, hash_hex: &str) -> Result {
let h = Hash::from_hex(hash_hex)?;
SnapManifest::from_bytes(&self.0.store.get(&h).await?)
@@ -1227,7 +1376,15 @@ impl Apps {
a.persist().await;
}
Ok(None) => {}
- Err(e) => warn!(app = %id, err = %e, "apps: snapshot failed"),
+ Err(e) => {
+ warn!(app = %id, err = %e, "apps: snapshot failed");
+ if runtime::over_state_cap(&e) {
+ // More state than an app may keep: the disk
+ // check on the next pass stops it.
+ l.state_bytes = snapshot::MAX_STATE_BYTES as u64 + 1;
+ l.disk_checked = None;
+ }
+ }
}
}
});
@@ -1248,6 +1405,9 @@ impl Apps {
return Ok(None);
};
let tar = runtime::snapshot_state(id, path).await?;
+ if let Some(l) = self.0.local.write().await.get_mut(id) {
+ l.state_bytes = tar.len() as u64;
+ }
let n = match &prev {
Some(p) if p.epoch == epoch => p.n + 1,
_ => 1,
@@ -1530,6 +1690,7 @@ impl Apps {
snapshot: placement::newest_snapshot(&rec.app_id, view).map(|(_, s)| s),
domains: rec.domains.clone(),
path: format!("/a/{}/", rec.app_id),
+ error: self.0.local.read().await.get(&rec.app_id).and_then(|l| l.error.clone()),
}
}
diff --git a/node/crates/c0mpute-core/src/apps/runtime.rs b/node/crates/c0mpute-core/src/apps/runtime.rs
index d7e5396f..31f05a77 100644
--- a/node/crates/c0mpute-core/src/apps/runtime.rs
+++ b/node/crates/c0mpute-core/src/apps/runtime.rs
@@ -117,8 +117,64 @@ pub async fn load(tar: &Path) -> Result<()> {
.map(|_| ())
}
+/// The docker network guarded app containers join, and its Linux interface.
+/// Not `container.run`'s jobs network: that one drops everything a container
+/// sends to the host, replies included, and an app must answer the node's
+/// proxy on its loopback-published port.
+pub const APPS_NETWORK: &str = "c0mpute-apps";
+pub const APPS_BRIDGE: &str = "c0apps0";
+
+/// The egress guard for the apps bridge: `container.run`'s rules for private,
+/// loopback, link-local (metadata), CGNAT and reserved ranges, and for the
+/// host itself everything except replies to connections the host opened
+/// (docker's loopback port publishing, which is how the proxy and the health
+/// probe reach the app).
+pub fn guard_rules(bridge: &str) -> Vec> {
+ let mut rules = crate::container_run::guard_rules(bridge);
+ rules.retain(|r| r.first().map(String::as_str) != Some("INPUT"));
+ rules.push(s(&["INPUT", "-i", bridge, "-m", "conntrack", "!", "--ctstate", "ESTABLISHED", "-j", "DROP"]));
+ rules
+}
+
+/// Make sure the apps network exists with the egress guard on it. True when
+/// app containers can be put behind the guard; false for a non-root worker
+/// (it cannot change the firewall) or when docker or iptables refuse.
+pub async fn ensure_guarded_network() -> bool {
+ if !crate::container_run::is_root() {
+ return false;
+ }
+ if let Err(e) = crate::container_run::ensure_bridge_network(APPS_NETWORK, APPS_BRIDGE, "c0mpute.apps=1").await {
+ tracing::warn!(err = %format!("{e:#}"), "apps: could not create the apps network");
+ return false;
+ }
+ crate::container_run::ensure_rules(guard_rules(APPS_BRIDGE)).await
+}
+
+/// Which network an app container joins: the guarded apps network when the
+/// guard is in, docker's default bridge (`None`) when it is not, or an error
+/// when the operator runs apps only behind the guard.
+pub fn network_for(guarded: bool, required: bool) -> std::result::Result, String> {
+ match (guarded, required) {
+ (true, _) => Ok(Some(APPS_NETWORK)),
+ (false, true) => Err(
+ "[apps] require_egress_guard is set and the egress guard is not in place (it needs a root worker, docker and iptables)".into(),
+ ),
+ (false, false) => Ok(None),
+ }
+}
+
+/// Where and how tightly an app's container is created.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct CreateOpts {
+ /// The docker network to join; `None` is docker's default bridge.
+ pub network: Option<&'static str>,
+ /// A hard quota on the container's writable layer, MiB, where the
+ /// storage driver has one ([`quota_supported`]).
+ pub layer_quota_mb: Option,
+}
+
/// The `docker create` argv for an app. Pure, so it is tested without docker.
-pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Vec {
+pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap, opts: &CreateOpts) -> Vec {
let name = container_name(&rec.app_id);
let mut a: Vec = vec![
"create".into(),
@@ -166,6 +222,14 @@ pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Ve
"--publish",
]));
a.push(format!("127.0.0.1::{}", rec.port));
+ if let Some(network) = opts.network {
+ a.push("--network".into());
+ a.push(network.into());
+ }
+ if let Some(mb) = opts.layer_quota_mb {
+ a.push("--storage-opt".into());
+ a.push(format!("size={mb}m"));
+ }
if let Some(p) = &rec.state_path {
a.push("--volume".into());
a.push(format!("{}:{}", volume_name(&rec.app_id), p));
@@ -182,11 +246,154 @@ pub fn create_args(rec: &AppRecord, secret_env: &BTreeMap) -> Ve
a
}
-pub async fn create(rec: &AppRecord, secret_env: &BTreeMap) -> Result<()> {
+pub async fn create(rec: &AppRecord, secret_env: &BTreeMap, opts: &CreateOpts) -> Result<()> {
let _ = remove(&rec.app_id).await;
- docker(&create_args(rec, secret_env), None, Duration::from_secs(60))
+ let made = docker(&create_args(rec, secret_env, opts), None, Duration::from_secs(60)).await;
+ if made.is_err() && opts.layer_quota_mb.is_some() {
+ // The quota was detected, not promised: a daemon that refuses it
+ // still runs the app, and the disk watchdog stays the limit.
+ let _ = remove(&rec.app_id).await;
+ let without = CreateOpts { layer_quota_mb: None, ..*opts };
+ return docker(&create_args(rec, secret_env, &without), None, Duration::from_secs(60)).await.map(|_| ());
+ }
+ made.map(|_| ())
+}
+
+/// Does `docker create --storage-opt size=` work here? Only overlay2 on an
+/// xfs filesystem mounted with project quotas enforces it; on ext4 (most
+/// hosts) docker refuses the flag. `mount_options` are those of the
+/// filesystem that holds docker's root directory.
+pub fn quota_supported(driver: &str, backing_fs: &str, mount_options: &str) -> bool {
+ driver == "overlay2"
+ && backing_fs == "xfs"
+ && mount_options.split(',').any(|o| matches!(o.trim(), "pquota" | "prjquota"))
+}
+
+/// The mount options of the filesystem holding `path`, from a
+/// `/proc/self/mounts` listing (the longest mount point that is a prefix).
+pub fn mount_options<'a>(mounts: &'a str, path: &Path) -> Option<&'a str> {
+ mounts
+ .lines()
+ .filter_map(|l| {
+ let mut f = l.split_whitespace();
+ let (_dev, point, _fs, options) = (f.next()?, f.next()?, f.next()?, f.next()?);
+ path.starts_with(point).then_some((point.len(), options))
+ })
+ .max_by_key(|(len, _)| *len)
+ .map(|(_, options)| options)
+}
+
+/// What `docker info` says about where and how containers are stored.
+#[derive(Clone, Debug, Default)]
+struct Storage {
+ root: Option,
+ quota: bool,
+}
+
+/// Asked once per process: the storage driver does not change under a
+/// running node.
+async fn storage() -> &'static Storage {
+ static STORAGE: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new();
+ STORAGE
+ .get_or_init(|| async {
+ let format = "{{.DockerRootDir}}\n{{.Driver}}\n{{range .DriverStatus}}{{index . 0}}={{index . 1}}\n{{end}}";
+ let Ok(out) = docker(&s(&["info", "--format", format]), None, Duration::from_secs(20)).await else {
+ return Storage::default();
+ };
+ let text = String::from_utf8_lossy(&out).into_owned();
+ let mut lines = text.lines().map(str::trim);
+ let root = lines.next().filter(|r| r.starts_with('/')).map(std::path::PathBuf::from);
+ let driver = lines.next().unwrap_or("").to_string();
+ let backing = lines.find_map(|l| l.strip_prefix("Backing Filesystem=")).unwrap_or("").to_string();
+ let mounts = std::fs::read_to_string("/proc/self/mounts").unwrap_or_default();
+ let quota = root
+ .as_deref()
+ .and_then(|r| mount_options(&mounts, r))
+ .is_some_and(|o| quota_supported(&driver, &backing, o));
+ Storage { root, quota }
+ })
.await
- .map(|_| ())
+}
+
+/// The writable-layer quota to create an app with, MiB: its disk limit where
+/// the storage driver enforces one, else none (the watchdog is the limit).
+pub async fn layer_quota_mb(limit_bytes: u64) -> Option {
+ (limit_bytes > 0 && storage().await.quota).then_some((limit_bytes >> 20).max(1))
+}
+
+/// Bytes the app has written inside its container (docker's own count of
+/// the writable layer; the state volume is not in it).
+pub async fn layer_bytes(app_id: &str) -> Option {
+ let out = docker(
+ &["inspect".into(), "--size".into(), "--format".into(), "{{.SizeRw}}".into(), container_name(app_id)],
+ None,
+ Duration::from_secs(60),
+ )
+ .await
+ .ok()?;
+ String::from_utf8_lossy(&out).trim().parse().ok()
+}
+
+/// Free bytes on the disk docker stores containers and volumes on, and the
+/// free space `container.run` keeps on such a disk (its default floor).
+pub async fn host_free() -> (u64, u64) {
+ let Some(root) = storage().await.root.as_deref() else {
+ return (u64::MAX, 0);
+ };
+ let (free, total) = c0mpute_store::free_space_at(root);
+ (free, crate::container_run::min_free_bytes(&crate::container_run::ContainersConfig::default(), total))
+}
+
+/// Why an app is being stopped for the disk it uses.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct DiskStop {
+ pub reason: String,
+ /// Remove the state volume as well. No rule asks for it today: an app
+ /// over its cap is stopped with its data left on this node, because the
+ /// part written since the last snapshot exists nowhere else and deleting
+ /// it is the owner's call, not the node's.
+ pub drop_state: bool,
+}
+
+/// The disk rule for one running app. `layer` is what it wrote inside its
+/// container, `state` the size of its state directory (the last snapshot's
+/// tar; anything over [`super::snapshot::MAX_STATE_BYTES`] when a snapshot
+/// was refused for size). `limit` 0 means no per-app limit. `None`: fine.
+pub fn disk_verdict(layer: u64, state: u64, limit: u64, free: u64, floor: u64) -> Option {
+ let mib = |b: u64| b >> 20;
+ let state_cap = super::snapshot::MAX_STATE_BYTES as u64;
+ let state_limit = if limit > 0 { limit.min(state_cap) } else { state_cap };
+ if state > state_limit {
+ return Some(DiskStop {
+ reason: format!("its state directory is over {} MiB, the most an app may keep here; the data is kept on this node", mib(state_limit)),
+ drop_state: false,
+ });
+ }
+ if limit > 0 && layer.saturating_add(state) > limit {
+ return Some(DiskStop {
+ reason: format!(
+ "it uses {} MiB of disk, over its {} MiB limit ({} MiB written inside the container, {} MiB of state)",
+ mib(layer.saturating_add(state)),
+ mib(limit),
+ mib(layer),
+ mib(state)
+ ),
+ drop_state: false,
+ });
+ }
+ if free < floor {
+ return Some(DiskStop {
+ reason: format!("this node's docker disk is down to {} GiB free (it keeps {} GiB)", free >> 30, floor >> 30),
+ drop_state: false,
+ });
+ }
+ None
+}
+
+/// Did a snapshot fail because the state is over the size cap (and not
+/// because docker did)?
+pub fn over_state_cap(err: &anyhow::Error) -> bool {
+ format!("{err:#}").contains("byte limit")
}
pub async fn start(app_id: &str) -> Result<()> {
@@ -460,9 +667,11 @@ mod tests {
let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
let rec = AppRecord::sign(&kp, spec("demo"), 1).unwrap();
let secret = BTreeMap::from([("DB_PASSWORD".to_string(), "hunter2".to_string())]);
- let a = create_args(&rec, &secret);
+ let a = create_args(&rec, &secret, &CreateOpts::default());
let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v);
assert_eq!(a[0], "create");
+ // Without the guard: docker's default bridge, as before, and no quota flag.
+ assert!(!a.iter().any(|x| x == "--network" || x == "--storage-opt"));
assert!(pair("--memory", "256m") && pair("--memory-swap", "256m"));
assert!(pair("--cpus", "0.500"));
assert!(pair("--cap-drop", "ALL"));
@@ -485,6 +694,104 @@ mod tests {
assert!(!a.iter().any(|x| x.starts_with("SYS_") || x == "NET_ADMIN"));
}
+ #[test]
+ fn guarded_apps_join_the_apps_network_and_stay_published_on_loopback() {
+ let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let rec = AppRecord::sign(&kp, spec("demo"), 1).unwrap();
+ let opts = CreateOpts { network: Some(APPS_NETWORK), layer_quota_mb: Some(2048) };
+ let a = create_args(&rec, &BTreeMap::new(), &opts);
+ let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v);
+ assert!(pair("--network", "c0mpute-apps"));
+ assert!(pair("--storage-opt", "size=2048m"));
+ // The proxy still reaches it the same way, and the image is still last.
+ assert!(pair("--publish", "127.0.0.1::8080"));
+ assert_eq!(a.last().unwrap(), rec.image_ref());
+ assert_eq!(a.iter().filter(|x| *x == "--network").count(), 1);
+ }
+
+ #[test]
+ fn the_guard_decides_the_network_and_require_refuses_without_it() {
+ assert_eq!(network_for(true, false), Ok(Some(APPS_NETWORK)));
+ assert_eq!(network_for(true, true), Ok(Some(APPS_NETWORK)));
+ // No guard (a non-root worker): the default bridge, as before.
+ assert_eq!(network_for(false, false), Ok(None));
+ let err = network_for(false, true).unwrap_err();
+ assert!(err.contains("require_egress_guard"), "{err}");
+ }
+
+ #[test]
+ fn apps_guard_blocks_private_ranges_and_the_host_but_lets_replies_back() {
+ let rules = guard_rules(APPS_BRIDGE);
+ let has = |needle: &[&str]| rules.iter().any(|r| r.iter().map(String::as_str).eq(needle.iter().copied()));
+ // The same forward rules container.run puts on its bridge.
+ for range in crate::container_run::GUARDED_RANGES {
+ assert!(has(&["DOCKER-USER", "-i", "c0apps0", "-d", range, "-j", "DROP"]), "{range}");
+ }
+ assert!(crate::container_run::GUARDED_RANGES.contains(&"169.254.0.0/16"), "cloud metadata");
+ // To the host: only replies to connections the host opened (the
+ // published loopback port); a blanket drop would cut the proxy off.
+ assert!(has(&["INPUT", "-i", "c0apps0", "-m", "conntrack", "!", "--ctstate", "ESTABLISHED", "-j", "DROP"]));
+ assert!(!has(&["INPUT", "-i", "c0apps0", "-j", "DROP"]));
+ assert_eq!(rules.iter().filter(|r| r[0] == "INPUT").count(), 1);
+ assert!(rules.iter().all(|r| r.contains(&"c0apps0".to_string()) && r.last().unwrap() == "DROP"));
+ }
+
+ #[test]
+ fn layer_quota_only_on_overlay2_over_xfs_with_project_quotas() {
+ assert!(quota_supported("overlay2", "xfs", "rw,relatime,attr2,inode64,prjquota"));
+ assert!(quota_supported("overlay2", "xfs", "rw,pquota"));
+ assert!(!quota_supported("overlay2", "xfs", "rw,relatime,noquota"));
+ assert!(!quota_supported("overlay2", "extfs", "rw,relatime,prjquota"), "ext4, like most hosts");
+ assert!(!quota_supported("overlayfs", "xfs", "rw,prjquota"), "the containerd image store");
+ assert!(!quota_supported("", "", ""));
+ let mounts = "/dev/sda1 / ext4 rw,relatime 0 0\n/dev/sdb1 /var/lib/docker xfs rw,noatime,prjquota 0 0\nproc /proc proc rw 0 0\n";
+ assert_eq!(mount_options(mounts, Path::new("/var/lib/docker")), Some("rw,noatime,prjquota"));
+ assert_eq!(mount_options(mounts, Path::new("/var/lib/docker/overlay2")), Some("rw,noatime,prjquota"));
+ assert_eq!(mount_options(mounts, Path::new("/var/lib/dockerx")), Some("rw,relatime"), "a prefix of the name is not the mount");
+ assert_eq!(mount_options("", Path::new("/var/lib/docker")), None);
+ }
+
+ #[test]
+ fn disk_limit_arithmetic() {
+ const MIB: u64 = 1 << 20;
+ const GIB: u64 = 1 << 30;
+ let limit = 2048 * MIB;
+ let (free, floor) = (100 * GIB, 20 * GIB);
+ // Under the limit with room on the host: nothing to do.
+ assert_eq!(disk_verdict(500 * MIB, 900 * MIB, limit, free, floor), None);
+ assert_eq!(disk_verdict(1024 * MIB, 1024 * MIB, limit, free, floor), None, "exactly the limit is fine");
+ // Layer plus state over the limit: stopped, state kept.
+ let stop = disk_verdict(1500 * MIB, 600 * MIB, limit, free, floor).unwrap();
+ assert!(!stop.drop_state);
+ assert!(stop.reason.contains("2100 MiB") && stop.reason.contains("2048 MiB limit"), "{}", stop.reason);
+ // A state directory over what an app may keep (a snapshot refused
+ // for size reports one byte more than the cap): stopped, and the
+ // volume is kept, since what it holds past the last snapshot is the
+ // only copy.
+ let cap = crate::apps::snapshot::MAX_STATE_BYTES as u64;
+ let stop = disk_verdict(0, cap + 1, limit, free, floor).unwrap();
+ assert!(!stop.drop_state && stop.reason.contains("1024 MiB"), "{}", stop.reason);
+ // An operator limit below the state cap caps the state too.
+ assert!(!disk_verdict(0, 300 * MIB, 256 * MIB, free, floor).unwrap().drop_state);
+ // No per-app limit (0): the state cap and the host floor still hold.
+ assert_eq!(disk_verdict(50 * GIB, 0, 0, free, floor), None);
+ assert!(disk_verdict(0, cap + 1, 0, free, floor).is_some());
+ // The host under the floor container.run keeps: stopped, state kept.
+ let stop = disk_verdict(MIB, MIB, limit, 19 * GIB, floor).unwrap();
+ assert!(!stop.drop_state && stop.reason.contains("19 GiB free"), "{}", stop.reason);
+ // Sizes that would overflow do not wrap into "fine".
+ assert!(disk_verdict(u64::MAX, 1, limit, free, floor).is_some());
+ // An unknown docker disk (free MAX, floor 0) never stops an app by itself.
+ assert_eq!(disk_verdict(0, 0, limit, u64::MAX, 0), None);
+ }
+
+ #[test]
+ fn a_snapshot_over_the_cap_is_told_from_a_docker_failure() {
+ let over = anyhow::anyhow!("output is over the 1073741824 byte limit").context("docker cp");
+ assert!(over_state_cap(&over));
+ assert!(!over_state_cap(&anyhow::anyhow!("docker cp exited with exit status: 1")));
+ }
+
#[test]
fn restore_unpacks_into_the_volume_offline() {
let a = restore_args("abc");
diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs
index 8e8e43fd..11fe2cd0 100644
--- a/node/crates/c0mpute-core/src/config.rs
+++ b/node/crates/c0mpute-core/src/config.rs
@@ -107,6 +107,14 @@ pub struct Config {
/// and pays bounties this node owes.
#[serde(default)]
pub swarm: crate::agent_swarm::SwarmConfig,
+ /// Blender renders (`render` role): whether Blender runs inside a
+ /// bubblewrap sandbox (`sandbox = "auto" | "off" | "require"`).
+ #[serde(default)]
+ pub render: crate::render::RenderConfig,
+ /// Text fine-tunes (`c0mpute infernet text serve`): the base models this
+ /// node downloads and runs for buyers, and the free disk it keeps.
+ #[serde(default)]
+ pub text: crate::text::TextConfig,
}
/// Adult mode on the buyer side. Every worker serves adult jobs and
@@ -208,6 +216,16 @@ pub struct AppsConfig {
/// Run only apps signed by these owner keys (hex ed25519, as `c0mpute
/// apps status` shows them). Empty = anyone's apps.
pub owners: Vec,
+ /// Keep app containers off private, loopback, link-local (metadata) and
+ /// CGNAT addresses and off this host, with the iptables guard
+ /// `container.run` uses (needs root; without it apps stay on docker's
+ /// default bridge and the node says so at start).
+ pub egress_guard: bool,
+ /// Run no apps at all unless that guard is in place.
+ pub require_egress_guard: bool,
+ /// Most disk one app may use, MiB: its state volume plus what it writes
+ /// inside its container. An app over it is stopped.
+ pub max_disk_mb: u64,
}
impl Default for AppsConfig {
@@ -216,6 +234,9 @@ impl Default for AppsConfig {
max_apps: 4,
max_memory_mb: 2048,
owners: Vec::new(),
+ egress_guard: true,
+ require_egress_guard: false,
+ max_disk_mb: crate::apps::DEFAULT_MAX_DISK_MB,
}
}
}
@@ -249,6 +270,13 @@ pub struct HostingConfig {
/// space above the free-disk floor (see `hosting::auto_budget`).
#[serde(default = "default_hosting_max_bytes")]
pub max_bytes: u64,
+ /// Count another node's copy of a site only after it has answered a
+ /// storage challenge, and rely on it before giving a copy up only once
+ /// it has answered several over hours. Off by default: an announcement
+ /// counts, as before 0.3.65 (`C0MPUTE_HOSTING_REQUIRE_PROOFS=1` also
+ /// turns it on).
+ #[serde(default)]
+ pub require_proofs: bool,
/// What this node asks to be a preferred (paid) host, USD per GiB-month.
/// Published in the OpenDisk descriptor and in hosting gossip; a site
/// offering less does not count this node as preferred.
@@ -288,6 +316,7 @@ impl Default for HostingConfig {
Self {
enabled: true,
max_bytes: default_hosting_max_bytes(),
+ require_proofs: false,
price_usd_per_gib_month: default_hosting_price(),
payout_wallet: None,
payout_coin: crate::payments::default_coin(),
@@ -840,6 +869,8 @@ impl Default for Config {
threatcrush: Default::default(),
adult: Default::default(),
swarm: Default::default(),
+ render: Default::default(),
+ text: Default::default(),
}
}
}
@@ -941,6 +972,22 @@ pub struct GatewayConfig {
/// (`gateway_announce`). Opt out: false, or `C0MPUTE_GATEWAY_AUTO=0`.
#[serde(default = "default_true")]
pub auto_announce: bool,
+ /// Give every hosted site its own web origin: with
+ /// `site_host_suffix = "sites.example.com"`, `.sites.example.com`
+ /// serves that site at `/` and nothing else, and a browser opening
+ /// `/s//...` is sent there. Needs wildcard DNS and TLS for
+ /// `*.` (docs/hosting.md). Unset (the default): sites are served
+ /// under `/s/` only. `C0MPUTE_SITE_HOST_SUFFIX` overrides it.
+ #[serde(default)]
+ pub site_host_suffix: Option,
+ /// Pages served under the shared `/s//` path are sandboxed
+ /// (`Content-Security-Policy: sandbox ...`), so one site's scripts cannot
+ /// read another site's storage or call this gateway's APIs as the same
+ /// origin. Pages that need localStorage, cookies or a service worker
+ /// work on a custom domain or a per-site host instead. On by default;
+ /// `C0MPUTE_SANDBOX_SHARED_SITES=0` also turns it off.
+ #[serde(default = "default_true")]
+ pub sandbox_shared_sites: bool,
}
impl Default for GatewayConfig {
@@ -949,6 +996,8 @@ impl Default for GatewayConfig {
bind: "0.0.0.0:7777".into(),
announce: None,
auto_announce: true,
+ site_host_suffix: None,
+ sandbox_shared_sites: true,
}
}
}
diff --git a/node/crates/c0mpute-core/src/container_run.rs b/node/crates/c0mpute-core/src/container_run.rs
index cf32af1f..6d41d3c7 100644
--- a/node/crates/c0mpute-core/src/container_run.rs
+++ b/node/crates/c0mpute-core/src/container_run.rs
@@ -1320,7 +1320,7 @@ pub fn guard_rules(bridge: &str) -> Vec> {
rules
}
-fn is_root() -> bool {
+pub(crate) fn is_root() -> bool {
#[cfg(unix)]
{
// SAFETY: geteuid has no preconditions.
@@ -1333,11 +1333,17 @@ fn is_root() -> bool {
}
/// Insert the guard rules for `bridge` (idempotent). True when they are in.
-async fn ensure_guard(bridge: &str) -> bool {
+pub(crate) async fn ensure_guard(bridge: &str) -> bool {
+ ensure_rules(guard_rules(bridge)).await
+}
+
+/// Insert iptables `rules` (each as [`guard_rules`] gives them) that are not
+/// in yet. True when all of them are in; always false for a non-root worker.
+pub(crate) async fn ensure_rules(rules: Vec>) -> bool {
if !is_root() {
return false;
}
- for rule in guard_rules(bridge) {
+ for rule in rules {
let (chain, rest) = rule.split_first().expect("non-empty rule");
let mut check = vec!["-C".to_string(), chain.clone()];
check.extend(rest.iter().cloned());
@@ -1370,8 +1376,17 @@ async fn ensure_guard(bridge: &str) -> bool {
/// Make sure the jobs network exists (containers on it cannot reach each
/// other) and, when configured, guard it. Returns whether egress is guarded.
async fn ensure_jobs_network(cfg: &ContainersConfig) -> Result {
+ ensure_bridge_network(JOBS_NETWORK, JOBS_BRIDGE, "c0mpute.jobs=1")
+ .await
+ .context("create the jobs network")?;
+ Ok(cfg.egress_guard && ensure_guard(JOBS_BRIDGE).await)
+}
+
+/// Make sure the docker bridge network `name` exists on the Linux interface
+/// `bridge`, with its containers unable to reach each other.
+pub(crate) async fn ensure_bridge_network(name: &str, bridge: &str, label: &str) -> Result<()> {
let t = Duration::from_secs(30);
- if docker_out(&s(&["network", "inspect", JOBS_NETWORK]), t)
+ if docker_out(&s(&["network", "inspect", name]), t)
.await
.is_err()
{
@@ -1379,20 +1394,20 @@ async fn ensure_jobs_network(cfg: &ContainersConfig) -> Result {
"network",
"create",
"--driver=bridge",
- &format!("--opt=com.docker.network.bridge.name={JOBS_BRIDGE}"),
+ &format!("--opt=com.docker.network.bridge.name={bridge}"),
"--opt=com.docker.network.bridge.enable_icc=false",
- "--label=c0mpute.jobs=1",
- JOBS_NETWORK,
+ &format!("--label={label}"),
+ name,
]);
if let Err(e) = docker_out(&create, t).await
- && docker_out(&s(&["network", "inspect", JOBS_NETWORK]), t)
+ && docker_out(&s(&["network", "inspect", name]), t)
.await
.is_err()
{
- return Err(e).context("create the jobs network");
+ return Err(e);
}
}
- Ok(cfg.egress_guard && ensure_guard(JOBS_BRIDGE).await)
+ Ok(())
}
/// The N-of-M hook: a buyer that ran the same build on several builders can
diff --git a/node/crates/c0mpute-core/src/hosting/edge_api.rs b/node/crates/c0mpute-core/src/hosting/edge_api.rs
index ceb92d47..821a78e3 100644
--- a/node/crates/c0mpute-core/src/hosting/edge_api.rs
+++ b/node/crates/c0mpute-core/src/hosting/edge_api.rs
@@ -178,6 +178,9 @@ pub(super) async fn chunk(
let mut resp = (StatusCode::OK, bytes).into_response();
let hdr = resp.headers_mut();
hdr.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream"));
+ // A chunk of a hosted page is that page's HTML: never let a
+ // browser guess a type for it on this origin.
+ hdr.insert("x-content-type-options", HeaderValue::from_static("nosniff"));
hdr.insert(
header::CACHE_CONTROL,
HeaderValue::from_static("public, max-age=31536000, immutable"),
diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs
index 14215e14..1f187209 100644
--- a/node/crates/c0mpute-core/src/hosting/mod.rs
+++ b/node/crates/c0mpute-core/src/hosting/mod.rs
@@ -26,7 +26,9 @@ pub mod git;
mod git_browse;
pub mod market;
pub mod meta;
+pub mod origin;
pub mod proof;
+pub mod relay;
pub mod replicas;
pub mod seed;
pub mod settle;
@@ -85,6 +87,32 @@ pub const MAX_BYTES_PER_OWNER: u64 = 16 * GIB_BYTES;
const SITES_PER_MESSAGE: usize = 200;
/// Other holders asked for a chunk before this node lets a surplus copy go.
const DROP_PROBES: usize = 12;
+/// How long a passed challenge keeps a holder counted toward the copies a
+/// site needs, in holder TTLs: 108 x 200 s = 6 h. Long enough that the nodes
+/// next in line for a site re-ask each holder a few times a day, not every
+/// repair round; short enough that a holder that deleted its copy and kept
+/// announcing stops counting the same day.
+const PROOF_TTL_HOLDER_TTLS: u32 = 108;
+/// Background challenges a repair round may start: in all, of one peer, and
+/// for one site. A node that just started knows no proofs and may be next in
+/// line for thousands of sites; it works through them at this rate, and until
+/// a site's turn comes its announced holders are taken at their word.
+const PROOF_ASKS_PER_ROUND: usize = 32;
+const PROOF_ASKS_PER_PEER: usize = 4;
+const PROOF_ASKS_PER_SITE: usize = MAX_REPLICAS;
+/// Background challenges in flight at once.
+const PROOF_SLOTS: usize = 8;
+/// Before this node gives up a copy because other holders have the site, it
+/// must have seen at least [`TARGET_REPLICAS`] of them pass its challenges at
+/// this many separate times, over at least this long. Filling asks for one
+/// recent pass only, so a new site still settles in minutes; letting go of
+/// data is the step that cannot be undone. Keys that only announce and relay
+/// then have to stay up and relay for hours before any honest holder trims,
+/// and since a relay through the challenger itself is seen (see
+/// [`Hosting::holds_now`]), the last honest holder is never talked into
+/// dropping by peers that hold nothing.
+pub const DROP_STANDING_PASSES: u32 = 3;
+pub const DROP_STANDING_SPAN: Duration = Duration::from_secs(6 * 3600);
/// Unfinished multi-part holding lists kept at once, and for how long. Each
/// is a peer's word, so neither the number nor the size is open-ended.
const MAX_PARTIAL_HOLDINGS: usize = 1024;
@@ -252,7 +280,23 @@ struct Inner {
/// When each site ([`WHOLE`]) or group was first seen over the target,
/// for trimming (see [`Hosting::trim_due`]).
over_since: std::sync::Mutex>,
+ /// When this node first asked the announced holders of a site to prove
+ /// their copies because it would otherwise take one itself (see
+ /// [`Hosting::fill_due`]).
+ unproven_since: std::sync::Mutex>,
+ /// Background challenges in flight, as (site id, holder).
+ proving: std::sync::Mutex>,
+ prove_slots: Semaphore,
+ /// Which held sites use a chunk (see [`relay`]). Filled only once
+ /// [`Hosting::refuses_fetch`] is in use.
+ chunk_index: std::sync::Mutex,
+ guards_fetches: std::sync::atomic::AtomicBool,
dns: dns::Verifier,
+ /// `[gateway] site_host_suffix`, normalized: `.` is
+ /// that site's own origin (see [`origin`]).
+ site_host_suffix: Option,
+ /// `[gateway] sandbox_shared_sites`: pages under `/s/` are sandboxed.
+ sandbox_shared: bool,
/// Optional handler for requests with a query string (podcast ad
/// insertion installs one); it may decline and let the plain serve run.
hook: std::sync::OnceLock>,
@@ -330,6 +374,13 @@ pub struct Options {
pub catalog_path: Option,
pub timing: Timing,
pub dns: dns::Verifier,
+ /// `[hosting] require_proofs`: count a holder toward a site's copies only
+ /// after this node saw it pass a challenge, and rely on it for a drop
+ /// only once it has standing. Off by default: announcements count, as
+ /// before 0.3.65, until the stricter rule has been run on real networks
+ /// (while a small site is still being copied its chunks are read
+ /// constantly, challenges are voided, and every host takes a copy).
+ pub require_proofs: bool,
/// Key that signs proof-of-storage receipts; no challenges without it.
pub challenger: Option,
pub challenge: ChallengeScope,
@@ -343,6 +394,13 @@ pub struct Options {
/// Pretend the disk has (free, total) bytes instead of asking the OS.
/// For tests on machines whose disk is below the free-disk floor.
pub disk_override: Option<(u64, u64)>,
+ /// `[gateway] site_host_suffix`: serve each site on its own host under
+ /// this suffix. `None`: as this machine is configured
+ /// ([`origin::configured`]); `Some("")`: off whatever is configured.
+ pub site_host_suffix: Option,
+ /// `[gateway] sandbox_shared_sites`. `None`: as this machine is
+ /// configured (on unless turned off).
+ pub sandbox_shared_sites: Option,
}
impl Default for Options {
@@ -353,6 +411,7 @@ impl Default for Options {
catalog_path: None,
timing: Timing::default(),
dns: dns::Verifier::default(),
+ require_proofs: false,
challenger: None,
challenge: ChallengeScope::Own,
challenge_every: Duration::from_secs(600),
@@ -360,6 +419,8 @@ impl Default for Options {
payout: None,
receipts_path: None,
disk_override: None,
+ site_host_suffix: None,
+ sandbox_shared_sites: None,
}
}
}
@@ -392,6 +453,25 @@ impl Hosting {
.unwrap_or_default();
let challenger_hex = opts.challenger.as_ref().and_then(site::owner_hex);
let records_path = opts.catalog_path.as_ref().map(|p| p.with_file_name("records.json"));
+ // Read the machine's configuration only for what the caller left open.
+ let (site_host_suffix, sandbox_shared) = match (opts.site_host_suffix, opts.sandbox_shared_sites) {
+ (Some(suffix), Some(sandbox)) => origin::resolve(Some(suffix), sandbox, None, None),
+ (suffix, sandbox) => {
+ let (cfg_suffix, cfg_sandbox) = origin::configured();
+ (
+ match suffix {
+ Some(s) => origin::resolve(Some(s), true, None, None).0,
+ None => cfg_suffix,
+ },
+ sandbox.unwrap_or(cfg_sandbox),
+ )
+ }
+ };
+ if let Some(suffix) = &site_host_suffix {
+ // The gateway's listener keeps every other route off those hosts.
+ c0mpute_gateway::site_host::configure(suffix);
+ info!(%suffix, "hosting: each site is served on its own host, .");
+ }
let known: Vec = records_path
.as_ref()
.and_then(|p| std::fs::read(p).ok())
@@ -404,7 +484,10 @@ impl Hosting {
hosts: opts.hosts,
max_bytes: opts.max_bytes,
timing: opts.timing,
- map: RwLock::new(ReplicaMap::default()),
+ map: RwLock::new(ReplicaMap {
+ count_claims: !(opts.require_proofs || std::env::var("C0MPUTE_HOSTING_REQUIRE_PROOFS").is_ok_and(|v| v == "1")),
+ ..Default::default()
+ }),
catalog: RwLock::new(Catalog::default()),
catalog_path: opts.catalog_path,
records_path,
@@ -421,7 +504,14 @@ impl Hosting {
layouts: RwLock::new(HashMap::new()),
short_since: std::sync::Mutex::new(HashMap::new()),
over_since: std::sync::Mutex::new(HashMap::new()),
+ unproven_since: std::sync::Mutex::new(HashMap::new()),
+ proving: std::sync::Mutex::new(HashSet::new()),
+ prove_slots: Semaphore::new(PROOF_SLOTS),
+ chunk_index: std::sync::Mutex::new(relay::ChunkIndex::default()),
+ guards_fetches: std::sync::atomic::AtomicBool::new(false),
dns: opts.dns,
+ site_host_suffix,
+ sandbox_shared,
hook: std::sync::OnceLock::new(),
host_hook: std::sync::OnceLock::new(),
challenger: opts.challenger,
@@ -913,6 +1003,7 @@ impl Hosting {
return;
}
h.announce_holding().await;
+ h.refresh_chunk_index().await;
h.connect_hosts().await;
if last_records.elapsed() >= h.0.timing.records_every {
h.announce_records().await;
@@ -945,24 +1036,42 @@ impl Hosting {
// A full node does not put itself forward for new copies (it would be
// chosen, fail, and hold the slot from the next ranked node).
let can_take = self.0.hosts && !self.is_full().await;
+ let mut asks = Asks::new(self.0.proving.lock().expect("proving").clone());
let (whole, striped): (Vec<(SiteRecord, Action)>, Vec) = {
let mut map = self.0.map.write().await;
let now = Instant::now();
+ let now_secs = crate::payments::now_secs();
+ let ttl = self.proof_ttl();
+ let gap = self.standing_gap();
map.prune(now, self.0.timing.holder_ttl);
+ map.prune_proofs(now, ttl.max(self.standing_span()));
self.0
.over_since
.lock()
.expect("over_since")
.retain(|(s, _), _| map.records.contains_key(s));
- let mut whole = Vec::new();
+ self.0
+ .unproven_since
+ .lock()
+ .expect("unproven_since")
+ .retain(|s, _| map.records.contains_key(s));
+ let mut whole: Vec<(SiteRecord, Action)> = Vec::new();
let mut striped = Vec::new();
+ // Sites (by place in `whole`) whose unproven holders are to be
+ // asked, and sites this node is building standing for.
+ let mut doubted: Vec<(usize, Vec)> = Vec::new();
+ let mut standing: Vec<(SiteRecord, Vec)> = Vec::new();
for r in map.records.values() {
if stripe::is_striped(r.size) {
striped.push(r.clone());
continue;
}
- let trim = self.trim_due(&r.site_id, WHOLE, map.holders(&r.site_id).len(), now);
- let a = replicas::decide(
+ let holders = map.holders(&r.site_id);
+ let trim = self.trim_due(&r.site_id, WHOLE, holders.len(), now);
+ // An announced holder fills a slot once it has proven its
+ // copy; announcing alone is free.
+ let proven = |p: &str| p == self.0.me || map.proven(&r.site_id, p, now, now_secs, ttl);
+ let a = replicas::decide_counting(
&map,
&r.site_id,
&self.0.me,
@@ -970,11 +1079,63 @@ impl Hosting {
TARGET_REPLICAS,
MAX_REPLICAS,
trim,
+ &proven,
);
+ let short_on_claims = || {
+ replicas::decide(&map, &r.site_id, &self.0.me, can_take, TARGET_REPLICAS, MAX_REPLICAS, trim)
+ == Action::Replicate
+ };
+ if a == Action::Replicate && !short_on_claims() {
+ // Only unproven announcements stand between this node
+ // and a copy: they are asked below, and the copy waits
+ // out the grace.
+ let unproven: Vec = holders.iter().filter(|p| !proven(p)).cloned().collect();
+ doubted.push((whole.len(), unproven));
+ } else {
+ self.0.unproven_since.lock().expect("unproven_since").remove(&r.site_id);
+ }
+ // A free copy of a site that is over the target may be let
+ // go later, and that takes standing (see
+ // [`DROP_STANDING_SPAN`]): start counting the others' passes
+ // now, whichever holder ends up being the one to let go.
+ if a == Action::Nothing
+ && holders.len() > TARGET_REPLICAS
+ && holders.iter().any(|p| *p == self.0.me)
+ && !map.effective_preferred(&r.site_id).contains(&self.0.me)
+ {
+ let due: Vec = holders
+ .iter()
+ .filter(|p| **p != self.0.me && map.pass_due(&r.site_id, p, now, gap))
+ .cloned()
+ .collect();
+ standing.push((r.clone(), due));
+ }
whole.push((r.clone(), a));
}
+ // The round's challenges go first to sites already waiting on
+ // their holders (longest first), so a holder that was slow once
+ // is asked again before its site's grace runs out; then to
+ // sites not asked yet; and last to building standing.
+ {
+ let since = self.0.unproven_since.lock().expect("unproven_since");
+ doubted.sort_by_key(|(i, _)| match since.get(&whole[*i].0.site_id) {
+ Some(t) => (0, Some(*t)),
+ None => (1, None),
+ });
+ }
+ for (i, unproven) in doubted {
+ let most = unproven.len().min(PROOF_ASKS_PER_SITE);
+ let asked = asks.take(&whole[i].0, unproven, most);
+ if !self.fill_due(&whole[i].0.site_id, asked >= most, now) {
+ whole[i].1 = Action::Nothing;
+ }
+ }
+ for (rec, due) in standing {
+ asks.take(&rec, due, PROOF_ASKS_PER_SITE);
+ }
(whole, striped)
};
+ self.spawn_proofs(std::mem::take(&mut asks.list));
for (rec, action) in whole {
// A site we hold whose record moved on: fetch the new version.
let stale = self
@@ -1069,17 +1230,41 @@ impl Hosting {
.map(|(g, hs)| self.trim_due(&rec.site_id, g as u32, hs.len(), now))
.collect();
let trim = |g: u32| trims.get(g as usize).copied().unwrap_or(false);
- stripe::plan(
- &map,
- &rec.site_id,
- &layout,
- &self.0.me,
- self.0.hosts,
- TARGET_REPLICAS,
- MAX_REPLICAS,
- &widen,
- &trim,
- )
+ // As for a whole site: a holder fills a group's slot once it
+ // has proven a copy, and while only unproven announcements
+ // stand between this node and a group they are asked first.
+ let now_secs = crate::payments::now_secs();
+ let ttl = self.proof_ttl();
+ let proven = |p: &str| p == self.0.me || map.proven(&rec.site_id, p, now, now_secs, ttl);
+ let plan_with = |counts: &dyn Fn(&str) -> bool| {
+ stripe::plan_counting(
+ &map,
+ &rec.site_id,
+ &layout,
+ &self.0.me,
+ self.0.hosts,
+ TARGET_REPLICAS,
+ MAX_REPLICAS,
+ &widen,
+ &trim,
+ counts,
+ )
+ };
+ let strict = plan_with(&proven);
+ let lenient = plan_with(&|_| true);
+ if strict.fetch == lenient.fetch {
+ self.0.unproven_since.lock().expect("unproven_since").remove(&rec.site_id);
+ strict
+ } else {
+ let mut unproven: Vec = raw.iter().flatten().filter(|p| !proven(p)).cloned().collect();
+ unproven.sort();
+ unproven.dedup();
+ let mut asks = Asks::new(self.0.proving.lock().expect("proving").clone());
+ let most = unproven.len().min(PROOF_ASKS_PER_SITE);
+ let asked = asks.take(&rec, unproven, most);
+ self.spawn_proofs(asks.list);
+ if self.fill_due(&rec.site_id, asked >= most, now) { strict } else { lenient }
+ }
};
let stale = held.as_ref().is_some_and(|h| h.seq < rec.seq);
if !plan.drop.is_empty() && !stale {
@@ -1127,6 +1312,79 @@ impl Hosting {
now.duration_since(since) >= self.0.timing.holder_ttl
}
+ /// How long a passed challenge keeps a holder counted (see
+ /// [`PROOF_TTL_HOLDER_TTLS`]).
+ fn proof_ttl(&self) -> Duration {
+ self.0.timing.holder_ttl * PROOF_TTL_HOLDER_TTLS
+ }
+
+ /// [`DROP_STANDING_SPAN`] on a node with the normal timing. Tests shrink
+ /// [`Timing`] to seconds and cannot wait six hours for a surplus copy to
+ /// go; there the span shrinks with the holder TTL.
+ fn standing_span(&self) -> Duration {
+ if self.0.timing.holder_ttl >= Timing::default().holder_ttl {
+ DROP_STANDING_SPAN
+ } else {
+ self.0.timing.holder_ttl * 2
+ }
+ }
+
+ /// The least time between two passes that both count toward standing.
+ fn standing_gap(&self) -> Duration {
+ self.standing_span() / DROP_STANDING_PASSES
+ }
+
+ /// Whether this node may now take a copy of a site whose announced
+ /// holders it has not seen prove theirs. `asked`: a challenge of each of
+ /// them (of [`PROOF_ASKS_PER_SITE`] of them, when there are more) was
+ /// queued this round or is already running.
+ ///
+ /// Not before they have had the grace ([`Self::proof_grace`], one holder
+ /// TTL) to answer, counted from the first round they were asked: honest holders that are slow or
+ /// unreachable for a moment are asked again every round meanwhile, so a
+ /// healthy site does not gain a copy it then has to trim. The clock is
+ /// the site's, not the holders': announcing under new keys does not
+ /// restart it, and it only stops when the site has its proven copies (or
+ /// the unproven announcements go away, which lets the fill run at once).
+ /// A site whose turn to be asked has not come (the round's challenges
+ /// were used up) has no clock yet and is left as it is.
+ fn fill_due(&self, site_id: &str, asked: bool, now: Instant) -> bool {
+ let mut since = self.0.unproven_since.lock().expect("unproven_since");
+ fill_clock(&mut since, site_id, asked, now, self.proof_grace())
+ }
+
+ /// How long announced holders have to prove their copies before a node
+ /// next in line takes one anyway: one holder TTL (what the network
+ /// already waits before it replaces a holder that went quiet), and
+ /// never less than two challenges at their slowest, so an answer that
+ /// is still on its way is not read as no answer (tests set the TTL
+ /// below the fetch timeout).
+ fn proof_grace(&self) -> Duration {
+ self.0.timing.holder_ttl.max(self.0.timing.fetch_timeout * 2)
+ }
+
+ /// Challenge each (site, holder) in the background, a few at a time.
+ /// Passes land in the map ([`Self::holds_now`]); the next repair round
+ /// reads them.
+ fn spawn_proofs(&self, list: Vec<(SiteRecord, String)>) {
+ for (rec, holder) in list {
+ let key = (rec.site_id.clone(), holder.clone());
+ if !self.0.proving.lock().expect("proving").insert(key.clone()) {
+ continue;
+ }
+ let h = self.clone();
+ tokio::spawn(async move {
+ if let Ok(_slot) = h.0.prove_slots.acquire().await
+ && !h.is_stopped()
+ {
+ let ok = h.holds_now(&rec, &holder, None).await;
+ debug!(site = %rec.site_id, %holder, ok, "hosting: asked a holder to prove its copy");
+ }
+ h.0.proving.lock().expect("proving").remove(&key);
+ });
+ }
+ }
+
fn spawn_replicate(&self, rec: SiteRecord) {
let h = self.clone();
tokio::spawn(async move {
@@ -2101,7 +2359,16 @@ impl Hosting {
/// Whether `holder` hands over a random chunk of `rec` right now, asked
/// directly. A holding announcement is only a claim (any key can gossip
- /// one); this is the check a node makes before it lets its own copy go.
+ /// one); this is the check a node makes before it counts the copy or
+ /// lets its own go. A pass is remembered ([`ReplicaMap::note_pass`]).
+ ///
+ /// A peer with no copy can still pass by fetching the chunk from a real
+ /// holder while it is being asked. When the real holder it asks is this
+ /// node, that shows: the chunk is read from this node's store between
+ /// the question and the answer. Such an answer is not a pass. The read
+ /// is matched by chunk, not by peer (a relay would ask under another
+ /// key), so now and then an unrelated read of the same chunk voids an
+ /// honest answer; the holder is simply asked again later.
async fn holds_now(&self, rec: &SiteRecord, holder: &str, only: Option<&[u32]>) -> bool {
let Ok(chunk) = self.challenge_chunk(rec, holder, only).await else {
return false;
@@ -2113,12 +2380,85 @@ impl Hosting {
chunk_hash: hash,
shard_index: None,
};
+ let watch = self.0.store.watch_reads(&hash);
let got = tokio::time::timeout(self.0.timing.fetch_timeout, self.0.net.fetch_from_peer(pid, &req)).await;
- matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash)
+ let ok = answer_passes(matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash), watch.seen());
+ if ok {
+ self.note_pass(rec, holder).await;
+ }
+ ok
+ }
+
+ /// The announced holders of `rec` whose copy this node has seen proven,
+ /// after asking those it has not (at most [`DROP_PROBES`] of them, at
+ /// once). What a publisher counts before it stops seeding: three keys
+ /// that only announce would otherwise read as three copies, the
+ /// publisher would leave, and the site would have no source left.
+ pub async fn verified_holders(&self, rec: &SiteRecord) -> Vec {
+ let (now_secs, ttl) = (crate::payments::now_secs(), self.proof_ttl());
+ let holders = self.0.map.read().await.holders(&rec.site_id);
+ let unproven: Vec = {
+ let map = self.0.map.read().await;
+ let now = Instant::now();
+ holders
+ .iter()
+ .filter(|p| **p != self.0.me && !map.proven(&rec.site_id, p, now, now_secs, ttl))
+ .take(DROP_PROBES)
+ .cloned()
+ .collect()
+ };
+ futures::future::join_all(unproven.iter().map(|p| self.holds_now(rec, p, None))).await;
+ let map = self.0.map.read().await;
+ let now = Instant::now();
+ holders
+ .into_iter()
+ .filter(|p| *p == self.0.me || map.proven(&rec.site_id, p, now, now_secs, ttl))
+ .collect()
+ }
+
+ /// Remember that `holder` just passed a challenge for `rec`.
+ async fn note_pass(&self, rec: &SiteRecord, holder: &str) {
+ self.0.map.write().await.note_pass(
+ &rec.site_id,
+ holder,
+ &rec.manifest,
+ Instant::now(),
+ self.standing_gap(),
+ self.standing_span(),
+ );
+ }
+
+ /// The holders among `others` worth asking before a copy is let go:
+ /// those that would have standing with a pass now, and those whose pass
+ /// would add to their count. Asking the rest proves nothing new.
+ async fn drop_asks(&self, rec: &SiteRecord, others: Vec) -> Vec {
+ let (now, gap, span) = (Instant::now(), self.standing_gap(), self.standing_span());
+ let map = self.0.map.read().await;
+ others
+ .into_iter()
+ .filter(|p| {
+ map.has_standing(&rec.site_id, p, now, DROP_STANDING_PASSES, span)
+ || map.pass_due(&rec.site_id, p, now, gap)
+ })
+ .take(DROP_PROBES)
+ .collect()
+ }
+
+ /// Whether `holder` (which just passed) has the standing a drop needs.
+ async fn has_standing(&self, rec: &SiteRecord, holder: &str) -> bool {
+ self.0.map.read().await.has_standing(
+ &rec.site_id,
+ holder,
+ Instant::now(),
+ DROP_STANDING_PASSES,
+ self.standing_span(),
+ )
}
/// Whether at least `need` other announced holders of `rec` prove their
- /// copy (see [`Self::holds_now`]). At most [`DROP_PROBES`] are asked.
+ /// copy now (see [`Self::holds_now`]) and have proven it before, often
+ /// and long enough to have standing (see [`DROP_STANDING_SPAN`]). At
+ /// most [`DROP_PROBES`] are asked.
async fn others_prove_whole(&self, rec: &SiteRecord, need: usize) -> bool {
let mut others: Vec = self.0.map.read().await.holders(&rec.site_id);
others.retain(|p| *p != self.0.me);
@@ -2126,9 +2466,10 @@ impl Hosting {
use rand::seq::SliceRandom;
others.shuffle(&mut rand::thread_rng());
}
+ let others = self.drop_asks(rec, others).await;
let mut proven = 0;
- for holder in others.iter().take(DROP_PROBES) {
- if self.holds_now(rec, holder, None).await {
+ for holder in others.iter() {
+ if self.holds_now(rec, holder, None).await && self.has_standing(rec, holder).await {
proven += 1;
if proven >= need {
return true;
@@ -2139,8 +2480,10 @@ impl Hosting {
}
/// Of `groups` (ones this node is about to let go), those that at least
- /// `need` other announced holders prove they hold. Each holder is asked
- /// once, for a chunk of one of the groups in question that it claims.
+ /// `need` other announced holders prove they hold, with the standing a
+ /// whole site asks for ([`Self::others_prove_whole`]). Each holder is
+ /// asked once, for a chunk of one of the groups in question that it
+ /// claims.
async fn groups_others_prove(&self, rec: &SiteRecord, groups: &[u32], need: usize) -> Vec {
let Ok(layout) = self.layout(rec).await else {
return Vec::new();
@@ -2159,9 +2502,10 @@ impl Hosting {
use rand::seq::SliceRandom;
claimants.shuffle(&mut rand::thread_rng());
}
+ let claimants = self.drop_asks(rec, claimants).await;
let mut proven: HashSet = HashSet::new();
- for holder in claimants.iter().take(DROP_PROBES) {
- if self.holds_now(rec, holder, Some(groups)).await {
+ for holder in claimants.iter() {
+ if self.holds_now(rec, holder, Some(groups)).await && self.has_standing(rec, holder).await {
proven.insert(holder.clone());
}
}
@@ -2191,8 +2535,19 @@ impl Hosting {
shard_index: None,
};
let started = Instant::now();
+ let watch = self.0.store.watch_reads(&hash);
let got = tokio::time::timeout(self.0.timing.fetch_timeout, self.0.net.fetch_from_peer(pid, &req)).await;
- let ok = matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash);
+ let right = matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash);
+ if right && watch.seen() {
+ // The chunk was read from this node while the holder was being
+ // asked for it (see [`Self::holds_now`]): neither a pass nor,
+ // since the read may have been someone else's, a signed failure.
+ bail!("{holder} answered while chunk {chunk} was being read from this node");
+ }
+ let ok = right;
+ if ok {
+ self.note_pass(rec, holder).await;
+ }
if !ok && !self.0.map.read().await.holders(&rec.site_id).iter().any(|p| p == holder) {
// The holder announced it let the copy go while we were asking:
// not a failed proof, just a stale target. No receipt.
@@ -2351,6 +2706,65 @@ impl Hosting {
(dues, outcomes)
}
+ // ── relay guard ──────────────────────────────────────────────────────
+
+ /// Whether a chunk fetch from `peer` should be answered as if the chunk
+ /// were not here: `peer` announces, whole, every site held here that
+ /// uses the chunk (see [`relay`]). For whatever serves libp2p chunk
+ /// fetches; gateway HTTP reads never come this way.
+ ///
+ /// The first call turns the chunk index on. Until the next announce
+ /// round has filled it, and for chunks of sites too big for it, the
+ /// answer is false (serve).
+ pub async fn refuses_fetch(&self, peer: &str, chunk: &Hash) -> bool {
+ self.0.guards_fetches.store(true, Ordering::Relaxed);
+ let map = self.0.map.read().await;
+ let index = self.0.chunk_index.lock().expect("chunk_index");
+ relay::refuses(&map, &index, peer, chunk)
+ }
+
+ /// Bring the chunk index in line with the catalog: sites let go are
+ /// forgotten, new or changed ones (another version, other groups)
+ /// indexed. Runs with the announcer, so it is at most one round behind;
+ /// behind means a fetch is served that could have been refused, or (a
+ /// site just let go) refused to a peer that claims the site anyway.
+ async fn refresh_chunk_index(&self) {
+ if !self.0.guards_fetches.load(Ordering::Relaxed) {
+ return;
+ }
+ let (held, stripes): (Vec, HashMap>) = {
+ let cat = self.0.catalog.read().await;
+ (cat.sites.values().cloned().collect(), cat.stripes.clone())
+ };
+ let ids: HashSet<&str> = held.iter().map(|r| r.site_id.as_str()).collect();
+ self.0
+ .chunk_index
+ .lock()
+ .expect("chunk_index")
+ .retain_sites(&|s| ids.contains(s));
+ for r in &held {
+ let groups = stripes.get(&r.site_id);
+ let stamp = format!("{}:{}", r.manifest, groups.map(stripe::encode_groups).unwrap_or_default());
+ let room = {
+ let index = self.0.chunk_index.lock().expect("chunk_index");
+ if index.stamp(&r.site_id) == Some(stamp.as_str()) {
+ continue;
+ }
+ index.room()
+ };
+ // A site plainly too big is not listed chunk by chunk to find
+ // that out (a whole one; a striped one is judged by its groups).
+ if groups.is_none() && r.size / site::FILE_CHUNK_BYTES as u64 > room as u64 {
+ self.0.chunk_index.lock().expect("chunk_index").skip(&r.site_id, &stamp);
+ continue;
+ }
+ let mut live = HashSet::new();
+ self.live_chunks_of(r, groups, &mut live).await;
+ let chunks: Vec = live.iter().filter_map(|h| Hash::from_hex(h).ok()).collect();
+ self.0.chunk_index.lock().expect("chunk_index").set(&r.site_id, &stamp, chunks);
+ }
+ }
+
// ── fetching ─────────────────────────────────────────────────────────
/// One chunk: local (store, or a file this process is publishing), else
@@ -2897,6 +3311,8 @@ impl Hosting {
.route("/p/{site}/", get(private_loader))
.merge(git::router())
.fallback(serve_by_host)
+ // Last, so it is in front of every route above and the fallback.
+ .layer(axum::middleware::from_fn_with_state(self.clone(), serve_site_host))
.with_state(self.clone())
}
}
@@ -2969,6 +3385,79 @@ fn site_root_redirect(site: &str, query: Option<&str>) -> Response {
}
}
+/// Whether an answer to a challenge is a pass: the right bytes came back,
+/// and the chunk was not read from this node while the holder was being
+/// asked (see [`Hosting::holds_now`]).
+fn answer_passes(right_bytes: bool, read_from_us_meanwhile: bool) -> bool {
+ right_bytes && !read_from_us_meanwhile
+}
+
+/// [`Hosting::fill_due`] on its table of clocks: true once `site_id`'s clock
+/// has run for `grace`. The clock starts the first round the site's unproven
+/// holders were all asked (`asked`): a site only some of whose holders could
+/// be asked has not had its chance yet.
+fn fill_clock(since: &mut HashMap, site_id: &str, asked: bool, now: Instant, grace: Duration) -> bool {
+ match since.get(site_id) {
+ Some(t) => now.duration_since(*t) >= grace,
+ None => {
+ if asked {
+ since.insert(site_id.to_string(), now);
+ }
+ false
+ }
+ }
+}
+
+/// The background challenges one repair round starts, within its limits
+/// ([`PROOF_ASKS_PER_ROUND`], [`PROOF_ASKS_PER_PEER`]).
+struct Asks {
+ /// Challenges already running, as (site id, holder).
+ busy: HashSet<(String, String)>,
+ left: usize,
+ per_peer: HashMap,
+ list: Vec<(SiteRecord, String)>,
+}
+
+impl Asks {
+ fn new(busy: HashSet<(String, String)>) -> Self {
+ Self {
+ busy,
+ left: PROOF_ASKS_PER_ROUND,
+ per_peer: HashMap::new(),
+ list: Vec::new(),
+ }
+ }
+
+ /// Queue challenges of up to `most` of `holders` of `rec`, picked at
+ /// random (so which of many announcers are asked cannot be arranged),
+ /// as far as the round's limits allow. Returns how many are queued or
+ /// already running.
+ fn take(&mut self, rec: &SiteRecord, mut holders: Vec, most: usize) -> usize {
+ use rand::seq::SliceRandom;
+ holders.shuffle(&mut rand::thread_rng());
+ let mut n = 0;
+ for holder in holders {
+ if n >= most {
+ break;
+ }
+ if self.busy.contains(&(rec.site_id.clone(), holder.clone())) {
+ n += 1;
+ continue;
+ }
+ let of_peer = self.per_peer.entry(holder.clone()).or_insert(0);
+ if self.left == 0 || *of_peer >= PROOF_ASKS_PER_PEER {
+ continue;
+ }
+ self.left -= 1;
+ *of_peer += 1;
+ self.busy.insert((rec.site_id.clone(), holder.clone()));
+ self.list.push((rec.clone(), holder));
+ n += 1;
+ }
+ n
+ }
+}
+
const GIB_BYTES: u64 = 1024 * 1024 * 1024;
/// Free space a hosting copy must leave on the disk: 10 GiB or 5% of it,
@@ -3187,7 +3676,7 @@ async fn serve_root(
};
return axum::response::Redirect::permanent(&to).into_response();
}
- serve_site(&h, &site, "", uri.query(), &headers).await
+ serve_site(&h, &site, "", Some(""), uri.query(), &headers).await
}
async fn serve_path(
@@ -3196,18 +3685,92 @@ async fn serve_path(
headers: HeaderMap,
uri: axum::http::Uri,
) -> Response {
- serve_site(&h, &site, &path, uri.query(), &headers).await
+ // The path as it was sent (still percent-encoded), for a redirect.
+ let prefix = format!("/s/{site}/");
+ serve_site(&h, &site, &path, uri.path().strip_prefix(&prefix), uri.query(), &headers).await
}
-async fn serve_site(h: &Hosting, site: &str, path: &str, query: Option<&str>, headers: &HeaderMap) -> Response {
- match h.record_for(site).await {
+/// Serve `path` of a site under the shared `/s//` path. `rest`: the
+/// path after `/s//` as the visitor sent it, when it could be told.
+async fn serve_site(
+ h: &Hosting,
+ site: &str,
+ path: &str,
+ rest: Option<&str>,
+ query: Option<&str>,
+ headers: &HeaderMap,
+) -> Response {
+ // With per-site hosts on, a browser opening a page here is sent to the
+ // site's own host, where the page has an origin of its own. Fetches
+ // that are not navigations (podcast apps, feed readers, the CLI, the
+ // `/p/` loader reading its ciphertext from this same origin) are
+ // answered here as before: they run no page on this origin, and a
+ // redirect to another host would break the ones that cannot follow it.
+ if let (Some(suffix), Some(rest)) = (h.0.site_host_suffix.as_deref(), rest)
+ && origin::wants_document(headers)
+ && let Some(url) = c0mpute_gateway::site_host::site_url(site, suffix, rest, query)
+ {
+ return origin::redirect_to_site_host(&url);
+ }
+ let mut resp = match h.record_for(site).await {
Some(rec) => {
let resp = h.serve_with_hook(&rec, path, query, headers).await;
let page = meta::public_origin(headers).map(|o| format!("{o}/s/{site}/{path}"));
with_absolute_meta(resp, page.as_deref()).await
}
None => (StatusCode::NOT_FOUND, "no such site on the network (yet)").into_response(),
+ };
+ origin::apply(resp.headers_mut(), origin::Served::Shared, h.0.sandbox_shared);
+ if h.0.site_host_suffix.is_some() {
+ origin::vary_on_navigation(resp.headers_mut());
}
+ resp
+}
+
+/// In front of every hosting route: a request on a per-site host
+/// (`.`) is that site and nothing else, so it never reaches
+/// the routes behind (`/hosting/v1/...`, `/p/`, git, another site's `/s/`).
+/// Any other request passes through.
+///
+/// The gateway's listener rewrites such a request to `/s/` and
+/// marks it before routing (`c0mpute_gateway::site_host::isolate`), which is
+/// what keeps the other services' routes off these hosts; a router used on
+/// its own (tests, the podcast listener) gets the request as it was sent.
+/// Both are handled here.
+async fn serve_site_host(State(h): State, req: axum::extract::Request, next: axum::middleware::Next) -> Response {
+ use c0mpute_gateway::site_host::{self, SiteHost};
+ let Some(suffix) = h.0.site_host_suffix.as_deref() else {
+ return next.run(req).await;
+ };
+ let (site, path) = match req.extensions().get::() {
+ Some(mark) => {
+ let prefix = format!("/s/{}", mark.site);
+ let path = req.uri().path().strip_prefix(&prefix).unwrap_or("/");
+ (mark.site.clone(), path.to_string())
+ }
+ None => {
+ let Some(site) =
+ site_host::request_host(req.headers(), req.uri()).and_then(|x| site_host::site_of_host(&x, suffix))
+ else {
+ return next.run(req).await;
+ };
+ if req.method() != axum::http::Method::GET && req.method() != axum::http::Method::HEAD {
+ return (StatusCode::NOT_FOUND, "not found").into_response();
+ }
+ (site, req.uri().path().to_string())
+ }
+ };
+ let (headers, query) = (req.headers(), req.uri().query());
+ let mut resp = match h.record_for(&site).await {
+ Some(rec) => {
+ let resp = h.serve_with_hook(&rec, &path, query, headers).await;
+ let page = meta::public_origin(headers).map(|o| format!("{o}{path}"));
+ with_absolute_meta(resp, page.as_deref()).await
+ }
+ None => (StatusCode::NOT_FOUND, "no such site on the network (yet)").into_response(),
+ };
+ origin::apply(resp.headers_mut(), origin::Served::SiteHost, h.0.sandbox_shared);
+ resp
}
/// Largest HTML page whose social meta tags get rewritten.
@@ -3319,7 +3882,9 @@ async fn serve_by_host(State(h): State, req: axum::extract::Request) ->
if h.0.dns.confirms(&host, &rec.site_id).await {
let resp = h.serve_with_hook(&rec, uri.path(), uri.query(), &headers).await;
let page = meta::public_origin(&headers).map(|o| format!("{o}{}", uri.path()));
- return with_absolute_meta(resp, page.as_deref()).await;
+ let mut resp = with_absolute_meta(resp, page.as_deref()).await;
+ origin::apply(resp.headers_mut(), origin::Served::Domain, h.0.sandbox_shared);
+ return resp;
}
}
(StatusCode::NOT_FOUND, "not found").into_response()
@@ -3329,6 +3894,83 @@ async fn serve_by_host(State(h): State, req: axum::extract::Request) ->
mod tests {
use super::*;
+ #[test]
+ fn an_answer_relayed_through_the_challenger_is_not_a_pass() {
+ assert!(answer_passes(true, false));
+ assert!(!answer_passes(true, true), "the chunk was read from us while we asked for it");
+ assert!(!answer_passes(false, false));
+ assert!(!answer_passes(false, true));
+ }
+
+ #[tokio::test]
+ async fn a_challenge_sees_the_chunk_read_from_this_node_meanwhile() {
+ // What Hosting::holds_now does around its fetch: a relay that has
+ // no copy can only answer by reading the chunk from a holder, and
+ // when that holder is the challenger the read is seen.
+ let dir = std::env::temp_dir().join(format!("c0mpute-relay-test-{}", rand::random::()));
+ let store = ChunkStore::open(&dir).await.unwrap().with_min_free_bytes(0);
+ let hash = store.put(b"the challenged chunk").await.unwrap();
+ let serving = store.clone();
+ // An honest holder answers from its own disk: nothing read here.
+ let watch = store.watch_reads(&hash);
+ assert!(answer_passes(true, watch.seen()));
+ // A relay asks this node (through the chunk server's clone of the
+ // store) for the very chunk it was challenged on.
+ let relayed = serving.get(&hash).await.unwrap();
+ assert!(!answer_passes(Hash::of(&relayed) == hash, watch.seen()));
+ drop(watch);
+ // The next challenge starts clean.
+ assert!(answer_passes(true, store.watch_reads(&hash).seen()));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn unproven_holders_hold_their_slots_for_one_grace_and_no_longer() {
+ let grace = Duration::from_secs(200);
+ let t0 = Instant::now();
+ let mut since = HashMap::new();
+ // Not asked yet (the round's challenges were used up): no clock, no copy.
+ assert!(!fill_clock(&mut since, "s", false, t0, grace));
+ assert!(since.is_empty());
+ // Asked: the clock starts, and the holders have the grace to answer.
+ assert!(!fill_clock(&mut since, "s", true, t0, grace));
+ assert!(!fill_clock(&mut since, "s", true, t0 + Duration::from_secs(199), grace));
+ // A round that could not ask again (or new keys still to be asked)
+ // does not restart it.
+ assert!(!fill_clock(&mut since, "s", false, t0 + Duration::from_secs(150), grace));
+ assert!(fill_clock(&mut since, "s", true, t0 + grace, grace));
+ // And it stays due while the site is short, asked this round or not.
+ assert!(fill_clock(&mut since, "s", false, t0 + grace * 2, grace));
+ // Another site has its own clock.
+ assert!(!fill_clock(&mut since, "t", true, t0 + grace * 2, grace));
+ }
+
+ #[test]
+ fn a_round_of_proof_asks_is_bounded_per_round_per_peer_and_per_site() {
+ let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let rec = |name: &str| SiteRecord::sign(&kp, name, &Hash::of(name.as_bytes()), 1, 10, vec![]).unwrap();
+ let peers = |n: usize| -> Vec { (0..n).map(|i| format!("p{i}")).collect() };
+ // Per site: at most `most`, however many announce it.
+ let mut asks = Asks::new(HashSet::new());
+ assert_eq!(asks.take(&rec("a"), peers(40), PROOF_ASKS_PER_SITE), PROOF_ASKS_PER_SITE);
+ assert_eq!(asks.list.len(), PROOF_ASKS_PER_SITE);
+ // Per peer: one peer announcing many sites is asked about a few.
+ let mut asks = Asks::new(HashSet::new());
+ let asked: usize = (0..10).map(|i| asks.take(&rec(&format!("s{i}")), peers(1), 5)).sum();
+ assert_eq!(asked, PROOF_ASKS_PER_PEER);
+ // Per round.
+ let mut asks = Asks::new(HashSet::new());
+ for i in 0..100 {
+ asks.take(&rec(&format!("s{i}")), vec![format!("q{i}")], 5);
+ }
+ assert_eq!(asks.list.len(), PROOF_ASKS_PER_ROUND);
+ // A challenge still running counts as asked and is not started twice.
+ let a = rec("a");
+ let mut asks = Asks::new([(a.site_id.clone(), "p0".to_string())].into());
+ assert_eq!(asks.take(&a, peers(1), 5), 1);
+ assert!(asks.list.is_empty());
+ }
+
#[test]
fn a_holding_part_past_its_own_count_does_not_grow_the_list() {
assert!(holding_part_fits(0, 1, 0));
@@ -4372,7 +5014,9 @@ async fn wait_whole(hosting: &Hosting, record: &SiteRecord, wait: Duration) -> u
while started.elapsed() < wait {
let left = wait.saturating_sub(started.elapsed());
tokio::time::sleep(COPY_POLL.min(left)).await;
- let holders = hosting.holders(&record.site_id).await;
+ // Copies that proved themselves, not announcements (anyone can
+ // announce): this is what decides when the publisher may leave.
+ let holders = hosting.verified_holders(record).await;
let others = holders.len();
if others != copies {
if others > copies {
diff --git a/node/crates/c0mpute-core/src/hosting/origin.rs b/node/crates/c0mpute-core/src/hosting/origin.rs
new file mode 100644
index 00000000..fb8ced51
--- /dev/null
+++ b/node/crates/c0mpute-core/src/hosting/origin.rs
@@ -0,0 +1,318 @@
+//! Which web origin a hosted site is served on, and the headers that follow
+//! from it.
+//!
+//! A site is reached three ways:
+//!
+//! * under the shared path `/s//` ([`Served::Shared`]).
+//! Every site there, and the gateway's own APIs, are one origin to a
+//! browser, so pages are sandboxed: each document gets an origin of its
+//! own that no other document shares;
+//! * on a per-site host `.` ([`Served::SiteHost`]), when
+//! the operator set `[gateway] site_host_suffix`. The host is the site's
+//! origin, and nothing else is served on it;
+//! * on a custom domain the owner signed into the record and DNS confirms
+//! ([`Served::Domain`]). The domain is the site's origin.
+//!
+//! On the last two a page is an ordinary page: localStorage, cookies and
+//! service workers are its own.
+
+use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+
+/// How a response reaches the visitor.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Served {
+ /// `/s//...`
+ Shared,
+ /// `./...`
+ SiteHost,
+ /// A custom domain of the site.
+ Domain,
+}
+
+/// The policy on pages under the shared path. `sandbox` without
+/// `allow-same-origin` gives the document an opaque origin: it runs its
+/// scripts and forms, but has no localStorage, cookies or service worker,
+/// and its requests to this gateway are cross-origin ones.
+pub const SHARED_SANDBOX: &str = "sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads \
+allow-pointer-lock allow-presentation";
+
+/// Env overrides of `[gateway] site_host_suffix` / `sandbox_shared_sites`.
+pub const SITE_HOST_SUFFIX_ENV: &str = "C0MPUTE_SITE_HOST_SUFFIX";
+pub const SANDBOX_SHARED_SITES_ENV: &str = "C0MPUTE_SANDBOX_SHARED_SITES";
+
+/// Whether a response of this type is a document a browser runs scripts
+/// from when it is opened: HTML, and the XML types (an SVG opened on its own
+/// is a document too).
+pub fn active_document(content_type: &str) -> bool {
+ let ct = content_type.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
+ matches!(
+ ct.as_str(),
+ "text/html" | "application/xhtml+xml" | "image/svg+xml" | "text/xml" | "application/xml"
+ ) || ct.ends_with("+xml")
+}
+
+/// Add the headers a hosted response needs for where it is served.
+/// `sandbox`: `[gateway] sandbox_shared_sites`.
+///
+/// Everywhere: `X-Content-Type-Options: nosniff`, when absent. Under the
+/// shared path, documents also get `Referrer-Policy: no-referrer` (the URL
+/// names the gateway and the site) and, with `sandbox`, the sandbox policy;
+/// and every response then allows cross-origin reads, since a sandboxed
+/// page is cross-origin to its own assets and could not otherwise `fetch`
+/// them, load its fonts or import its modules. Everything under `/s/` is
+/// public and served without credentials, so that gives nothing away.
+pub fn apply(headers: &mut HeaderMap, served: Served, sandbox: bool) {
+ let absent = |headers: &mut HeaderMap, name: &'static str, value: &'static str| {
+ if !headers.contains_key(name) {
+ headers.insert(name, HeaderValue::from_static(value));
+ }
+ };
+ absent(headers, "x-content-type-options", "nosniff");
+ if served != Served::Shared {
+ return;
+ }
+ let document = headers
+ .get(header::CONTENT_TYPE)
+ .and_then(|v| v.to_str().ok())
+ .is_some_and(active_document);
+ if document {
+ absent(headers, "referrer-policy", "no-referrer");
+ }
+ if !sandbox {
+ return;
+ }
+ if document {
+ // Appended, not replaced: a policy already on the response stays
+ // in force next to this one.
+ headers.append(header::CONTENT_SECURITY_POLICY, HeaderValue::from_static(SHARED_SANDBOX));
+ }
+ absent(headers, "access-control-allow-origin", "*");
+ absent(headers, "cross-origin-resource-policy", "cross-origin");
+}
+
+/// Whether a request is a browser opening a page (a navigation, in a tab or
+/// a frame) rather than a fetch of a file: `Sec-Fetch-Mode: navigate`, or
+/// for a browser too old to send that, an `Accept` that asks for HTML.
+/// Scripts, feed readers, podcast apps, `curl` and the `/p/` loader's fetch
+/// of its ciphertext are not navigations.
+pub fn wants_document(headers: &HeaderMap) -> bool {
+ let get = |name: &str| headers.get(name).and_then(|v| v.to_str().ok()).map(|v| v.trim().to_ascii_lowercase());
+ match get("sec-fetch-mode") {
+ Some(mode) => mode == "navigate",
+ None => get("accept").is_some_and(|a| a.contains("text/html")),
+ }
+}
+
+/// Send a page opened under `/s//` to the site's own host.
+///
+/// 307: the method is kept, and unlike a 301 or 308 a browser does not
+/// remember it for good, so an operator who takes the suffix away again is
+/// not left with visitors stuck on a dead name. Caches may keep it for five
+/// minutes, per kind of request (only navigations are redirected).
+pub fn redirect_to_site_host(url: &str) -> Response {
+ let Ok(location) = HeaderValue::from_str(url) else {
+ return (StatusCode::NOT_FOUND, "not found").into_response();
+ };
+ let mut resp = StatusCode::TEMPORARY_REDIRECT.into_response();
+ let h = resp.headers_mut();
+ h.insert(header::LOCATION, location);
+ h.insert(header::CACHE_CONTROL, HeaderValue::from_static("public, max-age=300"));
+ vary_on_navigation(h);
+ h.insert("x-content-type-options", HeaderValue::from_static("nosniff"));
+ resp
+}
+
+/// With per-site hosts on, what `/s/...` answers depends on whether the
+/// request is a navigation: say so to caches.
+pub fn vary_on_navigation(headers: &mut HeaderMap) {
+ headers.append(header::VARY, HeaderValue::from_static("Sec-Fetch-Mode, Accept"));
+}
+
+/// `(site_host_suffix, sandbox_shared_sites)` as this machine is configured:
+/// the env overrides, else the `[gateway]` section of the config file in its
+/// usual place, else the defaults (no suffix, sandbox on).
+pub fn configured() -> (Option, bool) {
+ let file = crate::config::Config::load_raw(&crate::config::default_config_path())
+ .map(|c| c.gateway)
+ .unwrap_or_default();
+ let env = |name: &str| std::env::var(name).ok().filter(|v| !v.trim().is_empty());
+ resolve(
+ file.site_host_suffix,
+ file.sandbox_shared_sites,
+ env(SITE_HOST_SUFFIX_ENV),
+ env(SANDBOX_SHARED_SITES_ENV),
+ )
+}
+
+/// [`configured`] from its inputs. A suffix that is not a DNS name is
+/// dropped (sites stay under `/s/`) rather than half-applied.
+pub fn resolve(
+ file_suffix: Option,
+ file_sandbox: bool,
+ env_suffix: Option,
+ env_sandbox: Option,
+) -> (Option, bool) {
+ let suffix = env_suffix.or(file_suffix).filter(|s| !s.trim().is_empty()).and_then(|raw| {
+ let ok = c0mpute_gateway::site_host::normalize_suffix(&raw);
+ if ok.is_none() {
+ tracing::warn!(suffix = %raw, "hosting: site_host_suffix is not a domain name, ignored");
+ }
+ ok
+ });
+ (suffix, crate::config::auto_install_enabled_with(file_sandbox, env_sandbox))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn headers_for(content_type: Option<&str>, served: Served, sandbox: bool) -> HeaderMap {
+ let mut h = HeaderMap::new();
+ if let Some(ct) = content_type {
+ h.insert(header::CONTENT_TYPE, HeaderValue::from_str(ct).unwrap());
+ }
+ apply(&mut h, served, sandbox);
+ h
+ }
+
+ fn get<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> {
+ h.get(name).and_then(|v| v.to_str().ok())
+ }
+
+ #[test]
+ fn pages_under_the_shared_path_are_sandboxed_and_their_assets_readable() {
+ let h = headers_for(Some("text/html; charset=utf-8"), Served::Shared, true);
+ assert_eq!(get(&h, "content-security-policy"), Some(SHARED_SANDBOX));
+ assert_eq!(get(&h, "referrer-policy"), Some("no-referrer"));
+ assert_eq!(get(&h, "access-control-allow-origin"), Some("*"));
+ assert_eq!(get(&h, "cross-origin-resource-policy"), Some("cross-origin"));
+ assert_eq!(get(&h, "x-content-type-options"), Some("nosniff"));
+ // The policy gives the page an opaque origin: never allow-same-origin.
+ assert!(SHARED_SANDBOX.starts_with("sandbox ") && !SHARED_SANDBOX.contains("allow-same-origin"));
+ assert!(!SHARED_SANDBOX.contains("allow-top-navigation"));
+ // An asset: readable by the sandboxed page, and not itself sandboxed
+ // (the policy would break a PDF opened in the browser's viewer).
+ for ct in ["text/css", "application/javascript", "image/png", "application/pdf", "audio/mpeg"] {
+ let h = headers_for(Some(ct), Served::Shared, true);
+ assert_eq!(get(&h, "content-security-policy"), None, "{ct}");
+ assert_eq!(get(&h, "referrer-policy"), None, "{ct}");
+ assert_eq!(get(&h, "access-control-allow-origin"), Some("*"), "{ct}");
+ assert_eq!(get(&h, "cross-origin-resource-policy"), Some("cross-origin"), "{ct}");
+ }
+ // A response with no type (a 304, a plain 404) still allows the read.
+ let h = headers_for(None, Served::Shared, true);
+ assert_eq!(get(&h, "content-security-policy"), None);
+ assert_eq!(get(&h, "access-control-allow-origin"), Some("*"));
+ assert_eq!(get(&h, "x-content-type-options"), Some("nosniff"));
+ }
+
+ #[test]
+ fn an_svg_or_xml_opened_as_a_page_is_a_document_too() {
+ for ct in ["image/svg+xml", "application/xhtml+xml", "text/xml", "application/xml", "TEXT/HTML", "application/rss+xml"] {
+ assert!(active_document(ct), "{ct}");
+ let h = headers_for(Some(ct), Served::Shared, true);
+ assert_eq!(get(&h, "content-security-policy"), Some(SHARED_SANDBOX), "{ct}");
+ }
+ for ct in ["text/plain", "application/json", "text/htmlx", "video/mp4", ""] {
+ assert!(!active_document(ct), "{ct}");
+ }
+ }
+
+ #[test]
+ fn the_operator_can_turn_the_sandbox_off() {
+ let h = headers_for(Some("text/html"), Served::Shared, false);
+ assert_eq!(get(&h, "content-security-policy"), None);
+ assert_eq!(get(&h, "access-control-allow-origin"), None);
+ assert_eq!(get(&h, "cross-origin-resource-policy"), None);
+ // The baseline stays.
+ assert_eq!(get(&h, "referrer-policy"), Some("no-referrer"));
+ assert_eq!(get(&h, "x-content-type-options"), Some("nosniff"));
+ }
+
+ #[test]
+ fn a_site_on_its_own_origin_is_an_ordinary_page() {
+ for served in [Served::SiteHost, Served::Domain] {
+ for sandbox in [true, false] {
+ let h = headers_for(Some("text/html; charset=utf-8"), served, sandbox);
+ assert_eq!(get(&h, "content-security-policy"), None, "{served:?}");
+ assert_eq!(get(&h, "access-control-allow-origin"), None, "{served:?}");
+ assert_eq!(get(&h, "cross-origin-resource-policy"), None, "{served:?}");
+ assert_eq!(get(&h, "referrer-policy"), None, "{served:?}");
+ assert_eq!(get(&h, "x-content-type-options"), Some("nosniff"), "{served:?}");
+ }
+ }
+ }
+
+ #[test]
+ fn headers_already_set_are_kept_and_a_policy_is_added_to_not_replaced() {
+ let mut h = HeaderMap::new();
+ h.insert(header::CONTENT_TYPE, HeaderValue::from_static("text/html"));
+ h.insert("x-content-type-options", HeaderValue::from_static("nosniff"));
+ h.insert("referrer-policy", HeaderValue::from_static("same-origin"));
+ h.insert("access-control-allow-origin", HeaderValue::from_static("https://ads.example"));
+ h.insert(header::CONTENT_SECURITY_POLICY, HeaderValue::from_static("default-src 'self'"));
+ apply(&mut h, Served::Shared, true);
+ assert_eq!(get(&h, "referrer-policy"), Some("same-origin"));
+ assert_eq!(get(&h, "access-control-allow-origin"), Some("https://ads.example"));
+ let policies: Vec<&str> = h
+ .get_all(header::CONTENT_SECURITY_POLICY)
+ .iter()
+ .filter_map(|v| v.to_str().ok())
+ .collect();
+ assert_eq!(policies, vec!["default-src 'self'", SHARED_SANDBOX]);
+ assert_eq!(h.get_all("x-content-type-options").iter().count(), 1);
+ }
+
+ #[test]
+ fn only_a_browser_opening_a_page_is_a_navigation() {
+ let with = |pairs: &[(&'static str, &'static str)]| {
+ let mut h = HeaderMap::new();
+ for (k, v) in pairs {
+ h.insert(*k, HeaderValue::from_static(*v));
+ }
+ h
+ };
+ assert!(wants_document(&with(&[("sec-fetch-mode", "navigate"), ("accept", "text/html,*/*")])));
+ assert!(wants_document(&with(&[("sec-fetch-mode", "Navigate")])));
+ // fetch() from a page (the /p/ loader reading its ciphertext), an
+ // , a script: served where they asked.
+ assert!(!wants_document(&with(&[("sec-fetch-mode", "cors"), ("accept", "text/html")])));
+ assert!(!wants_document(&with(&[("sec-fetch-mode", "same-origin")])));
+ assert!(!wants_document(&with(&[("sec-fetch-mode", "no-cors"), ("accept", "image/avif,*/*")])));
+ // An old browser sends no Sec-Fetch-Mode but asks for HTML.
+ assert!(wants_document(&with(&[("accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8")])));
+ // curl, reqwest, a feed reader.
+ assert!(!wants_document(&with(&[("accept", "*/*")])));
+ assert!(!wants_document(&with(&[("accept", "application/rss+xml")])));
+ assert!(!wants_document(&HeaderMap::new()));
+ }
+
+ #[test]
+ fn the_redirect_keeps_the_method_and_is_cached_briefly() {
+ let r = redirect_to_site_host("https://0123456789abcdef0123.sites.example.com/a%20b?x=1");
+ assert_eq!(r.status(), StatusCode::TEMPORARY_REDIRECT);
+ assert_eq!(
+ get(r.headers(), "location"),
+ Some("https://0123456789abcdef0123.sites.example.com/a%20b?x=1")
+ );
+ assert_eq!(get(r.headers(), "cache-control"), Some("public, max-age=300"));
+ assert_eq!(get(r.headers(), "vary"), Some("Sec-Fetch-Mode, Accept"));
+ assert_eq!(redirect_to_site_host("bad\r\nset-cookie: x").status(), StatusCode::NOT_FOUND);
+ }
+
+ #[test]
+ fn the_env_overrides_the_file_and_a_bad_suffix_is_dropped() {
+ let s = |v: &str| Some(v.to_string());
+ assert_eq!(resolve(None, true, None, None), (None, true), "the default: /s/ only, sandboxed");
+ assert_eq!(resolve(s("*.Sites.Example.com"), true, None, None), (s("sites.example.com"), true));
+ assert_eq!(resolve(s("a.example.com"), true, s("b.example.com"), None).0, s("b.example.com"));
+ assert_eq!(resolve(s("not a domain"), true, None, None).0, None);
+ assert_eq!(resolve(s("localhost"), true, None, None).0, None);
+ assert_eq!(resolve(s(" "), true, None, None).0, None);
+ assert!(!resolve(None, false, None, None).1);
+ assert!(!resolve(None, true, None, s("0")).1);
+ assert!(!resolve(None, true, None, s("off")).1);
+ assert!(resolve(None, false, None, s("1")).1);
+ }
+}
diff --git a/node/crates/c0mpute-core/src/hosting/proof.rs b/node/crates/c0mpute-core/src/hosting/proof.rs
index 2fabaa28..5ddb817f 100644
--- a/node/crates/c0mpute-core/src/hosting/proof.rs
+++ b/node/crates/c0mpute-core/src/hosting/proof.rs
@@ -16,6 +16,25 @@
//! own copy rather than relaying from another holder; relaying costs the
//! relayer bandwidth and latency, and the rest of the network still has the
//! data, so the site stays up either way.
+//!
+//! What narrows the relay, without a new message:
+//!
+//! * a relay through the challenger itself is seen (the challenged chunk is
+//! read from the challenger's store while it waits) and is not a pass
+//! (`Hosting::holds_now`);
+//! * a holder refuses chunk fetches from peers that announce the same site
+//! whole ([`super::relay`]);
+//! * a copy is let go only on the strength of holders that passed at
+//! several separate times over hours (`DROP_STANDING_SPAN`).
+//!
+//! What was weighed and left out: a response deadline tight enough to miss a
+//! relay's extra round trip also fails honest holders behind a relayed or
+//! slow link, which is the churn the fill must not cause; asking for several
+//! chunks, or a keyed digest over ranges of them, multiplies an honest
+//! holder's cost by the same factor as the relay's (the relay still passes,
+//! it only pays more) and the digest needs a message old nodes do not have.
+//! A relay from a holder other than the challenger, under a second key,
+//! still passes.
use anyhow::Result;
use c0mpute_proto::Hash;
@@ -31,6 +50,9 @@ pub const KEEP_PER_HOLDER: usize = 16;
pub const RECENT: usize = 5;
/// …and loses preference when at least this many of them failed.
pub const FAIL_LIMIT: usize = 3;
+/// How far ahead of this node's clock a receipt may be dated and still be
+/// read as a recent proof (see `ReplicaMap::proven`).
+pub const CLOCK_SKEW_SECS: u64 = 300;
/// "I asked `holder` for chunk `chunk` of `site_id` at `at`; it passed or not."
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
diff --git a/node/crates/c0mpute-core/src/hosting/relay.rs b/node/crates/c0mpute-core/src/hosting/relay.rs
new file mode 100644
index 00000000..e35dfabc
--- /dev/null
+++ b/node/crates/c0mpute-core/src/hosting/relay.rs
@@ -0,0 +1,243 @@
+//! A holder does not serve the peers that claim to hold the same site.
+//!
+//! A storage challenge ([`super::proof`]) cannot tell a peer that keeps its
+//! copy from one that fetches the challenged chunk from a real holder and
+//! passes it on. The cheapest place to fetch it is the real holders
+//! themselves, so they refuse: a peer that announces the current version of a
+//! site whole has every chunk of it and no reason to ask for one, and a node
+//! that holds the site answers such a fetch as if the chunk were absent
+//! ([`refuses`]). A relay is left with holders that do not run this rule and
+//! with identities that announce nothing.
+//!
+//! Only whole claims are refused. A peer that announces another version, or
+//! only some groups of a striped site, or nothing, is served as before: that
+//! is every node still fetching (a copy counts, and is announced, only once
+//! it is complete), so two nodes that each hold some groups always serve
+//! each other the rest.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use c0mpute_proto::Hash;
+
+use super::replicas::ReplicaMap;
+
+/// Chunks the index covers. Sites that do not fit are left out, and fetches
+/// of their chunks are served to everyone as before.
+pub const MAX_INDEXED_CHUNKS: usize = 131_072;
+
+/// Which held sites use a chunk, for the chunks of the sites this node
+/// holds. Kept up from the catalog, bounded by [`MAX_INDEXED_CHUNKS`].
+#[derive(Default)]
+pub struct ChunkIndex {
+ chunks: HashMap>>,
+ /// Per indexed site: what was indexed (version and groups) and its
+ /// chunks. A site too big for the index keeps its stamp and no chunks,
+ /// so it is not listed again until it changes.
+ sites: HashMap)>,
+ indexed: usize,
+}
+
+impl ChunkIndex {
+ /// What `site` was last indexed as (see [`Self::set`]).
+ pub fn stamp(&self, site: &str) -> Option<&str> {
+ self.sites.get(site).map(|(s, _)| s.as_str())
+ }
+
+ /// Chunks the index can still take.
+ pub fn room(&self) -> usize {
+ MAX_INDEXED_CHUNKS.saturating_sub(self.indexed)
+ }
+
+ /// Index `chunks` as the chunks of `site` held here, replacing what was
+ /// indexed for it. `stamp` names the version and groups, so the caller
+ /// can tell when to index again. Returns false (and indexes none of
+ /// them) when they do not fit.
+ pub fn set(&mut self, site: &str, stamp: &str, chunks: Vec) -> bool {
+ self.set_capped(site, stamp, chunks, MAX_INDEXED_CHUNKS)
+ }
+
+ /// [`Self::set`] with the cap given (tests use a small one).
+ pub(crate) fn set_capped(&mut self, site: &str, stamp: &str, mut chunks: Vec, max: usize) -> bool {
+ self.remove(site);
+ chunks.sort_by(|a, b| a.0.cmp(&b.0));
+ chunks.dedup();
+ if self.indexed + chunks.len() > max {
+ self.sites.insert(site.to_string(), (stamp.to_string(), Vec::new()));
+ return false;
+ }
+ let name: Arc = Arc::from(site);
+ for h in &chunks {
+ self.chunks.entry(*h).or_default().push(name.clone());
+ }
+ self.indexed += chunks.len();
+ self.sites.insert(site.to_string(), (stamp.to_string(), chunks));
+ true
+ }
+
+ /// Remember `site` by its stamp without indexing any of its chunks (it
+ /// is too big for the index).
+ pub fn skip(&mut self, site: &str, stamp: &str) {
+ self.remove(site);
+ self.sites.insert(site.to_string(), (stamp.to_string(), Vec::new()));
+ }
+
+ /// Forget `site`.
+ pub fn remove(&mut self, site: &str) {
+ let Some((_, chunks)) = self.sites.remove(site) else {
+ return;
+ };
+ self.indexed -= chunks.len();
+ for h in chunks {
+ if let Some(v) = self.chunks.get_mut(&h) {
+ v.retain(|s| &**s != site);
+ if v.is_empty() {
+ self.chunks.remove(&h);
+ }
+ }
+ }
+ }
+
+ /// Forget every site `keep` does not name.
+ pub fn retain_sites(&mut self, keep: &dyn Fn(&str) -> bool) {
+ let gone: Vec = self.sites.keys().filter(|s| !keep(s)).cloned().collect();
+ for s in gone {
+ self.remove(&s);
+ }
+ }
+
+ /// The held sites that use `chunk`.
+ pub fn sites_of(&self, chunk: &Hash) -> &[Arc] {
+ self.chunks.get(chunk).map(|v| v.as_slice()).unwrap_or(&[])
+ }
+}
+
+/// Whether `peer` announces the current version of `site_id` whole: the
+/// record's manifest, with no list of groups.
+pub fn claims_whole(map: &ReplicaMap, site_id: &str, peer: &str) -> bool {
+ let (Some(rec), Some(h)) = (map.records.get(site_id), map.peers.get(peer)) else {
+ return false;
+ };
+ h.sites.get(site_id).is_some_and(|(m, _)| *m == rec.manifest) && !h.groups.contains_key(site_id)
+}
+
+/// Whether a fetch of `chunk` by `peer` is refused: the chunk belongs to
+/// sites held here, and `peer` claims every one of them whole. A chunk two
+/// sites share is served unless the peer claims both, since it may be
+/// fetching the one it does not have.
+pub fn refuses(map: &ReplicaMap, index: &ChunkIndex, peer: &str, chunk: &Hash) -> bool {
+ let sites = index.sites_of(chunk);
+ !sites.is_empty() && sites.iter().all(|s| claims_whole(map, s, peer))
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{BTreeSet, HashMap};
+ use std::time::Instant;
+
+ use super::*;
+ use crate::hosting::site::SiteRecord;
+
+ fn rec(name: &str) -> SiteRecord {
+ let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ SiteRecord::sign(&kp, name, &Hash::of(name.as_bytes()), 1, 10, vec![]).unwrap()
+ }
+
+ fn announce(m: &mut ReplicaMap, peer: &str, held: &[(&SiteRecord, Option<&str>, Option<&[u32]>)]) {
+ let mut sites = HashMap::new();
+ let mut groups = HashMap::new();
+ for (r, manifest, gs) in held {
+ sites.insert(r.site_id.clone(), (manifest.unwrap_or(&r.manifest).to_string(), r.seq));
+ if let Some(gs) = gs {
+ groups.insert(r.site_id.clone(), gs.iter().copied().collect::>());
+ }
+ }
+ m.apply_holding_full(peer, sites, groups, Instant::now(), true, None, None, None, vec![]);
+ }
+
+ #[test]
+ fn a_peer_claiming_the_site_whole_is_refused_and_every_other_peer_is_served() {
+ let a = rec("a");
+ let mut m = ReplicaMap::default();
+ m.apply_record(a.clone());
+ let chunk = Hash::of(b"chunk of a");
+ let mut idx = ChunkIndex::default();
+ assert!(idx.set(&a.site_id, "v1", vec![chunk]));
+
+ announce(&mut m, "whole", &[(&a, None, None)]);
+ announce(&mut m, "nothing", &[]);
+ announce(&mut m, "old-version", &[(&a, Some("an-older-manifest"), None)]);
+ announce(&mut m, "some-groups", &[(&a, None, Some(&[0u32, 2][..]))]);
+
+ assert!(refuses(&m, &idx, "whole", &chunk), "claims whole: has no reason to fetch");
+ assert!(!refuses(&m, &idx, "nothing", &chunk), "a node taking its first copy");
+ assert!(!refuses(&m, &idx, "old-version", &chunk), "a holder moving to the new version");
+ assert!(!refuses(&m, &idx, "some-groups", &chunk), "a holder of other groups");
+ assert!(!refuses(&m, &idx, "never-heard-of", &chunk));
+ // A chunk of no site held here is nobody's business.
+ assert!(!refuses(&m, &idx, "whole", &Hash::of(b"something else")));
+ // It stops claiming (a repairing node must): served again.
+ announce(&mut m, "whole", &[]);
+ assert!(!refuses(&m, &idx, "whole", &chunk));
+ }
+
+ #[test]
+ fn a_chunk_two_sites_share_is_refused_only_to_a_peer_claiming_both() {
+ let (a, b) = (rec("a"), rec("b"));
+ let mut m = ReplicaMap::default();
+ m.apply_record(a.clone());
+ m.apply_record(b.clone());
+ let shared = Hash::of(b"shared");
+ let only_a = Hash::of(b"only a");
+ let mut idx = ChunkIndex::default();
+ idx.set(&a.site_id, "v1", vec![shared, only_a]);
+ idx.set(&b.site_id, "v1", vec![shared]);
+ announce(&mut m, "has-a", &[(&a, None, None)]);
+ announce(&mut m, "has-both", &[(&a, None, None), (&b, None, None)]);
+ assert!(refuses(&m, &idx, "has-a", &only_a));
+ assert!(!refuses(&m, &idx, "has-a", &shared), "it may be fetching b");
+ assert!(refuses(&m, &idx, "has-both", &shared));
+ // Letting b go leaves the chunk as a's alone.
+ idx.remove(&b.site_id);
+ assert!(refuses(&m, &idx, "has-a", &shared));
+ }
+
+ #[test]
+ fn two_nodes_holding_different_groups_serve_each_other() {
+ // Each announces the groups it has; neither claims the site whole,
+ // so each can fetch from the other the groups it lacks.
+ let a = rec("a");
+ let mut m = ReplicaMap::default();
+ m.apply_record(a.clone());
+ announce(&mut m, "left", &[(&a, None, Some(&[0u32, 1][..]))]);
+ announce(&mut m, "right", &[(&a, None, Some(&[2u32, 3][..]))]);
+ let (g0, g2) = (Hash::of(b"group 0"), Hash::of(b"group 2"));
+ // What "left" and "right" each index: the chunks of their groups.
+ let mut left = ChunkIndex::default();
+ left.set(&a.site_id, "v1:0,1", vec![g0]);
+ let mut right = ChunkIndex::default();
+ right.set(&a.site_id, "v1:2,3", vec![g2]);
+ assert!(!refuses(&m, &left, "right", &g0));
+ assert!(!refuses(&m, &right, "left", &g2));
+ }
+
+ #[test]
+ fn the_index_is_bounded_and_forgets_sites_let_go() {
+ let mut idx = ChunkIndex::default();
+ let h = |i: u32| Hash::of(&i.to_le_bytes());
+ assert!(idx.set_capped("a", "v1", vec![h(1), h(2), h(2)], 3));
+ assert_eq!(idx.indexed, 2, "a chunk listed twice is one chunk");
+ // Too big for what is left: remembered by its stamp, not indexed.
+ assert!(!idx.set_capped("b", "v1", vec![h(3), h(4)], 3));
+ assert_eq!(idx.stamp("b"), Some("v1"));
+ assert!(idx.sites_of(&h(3)).is_empty());
+ idx.skip("c", "v1");
+ assert_eq!((idx.stamp("c"), idx.indexed), (Some("v1"), 2));
+ // A new version of a replaces the old one's chunks.
+ assert!(idx.set_capped("a", "v2", vec![h(5)], 3));
+ assert!(idx.sites_of(&h(1)).is_empty());
+ assert_eq!(idx.sites_of(&h(5)).len(), 1);
+ idx.retain_sites(&|s| s == "b");
+ assert_eq!((idx.indexed, idx.chunks.len(), idx.stamp("a")), (0, 0, None));
+ }
+}
diff --git a/node/crates/c0mpute-core/src/hosting/replicas.rs b/node/crates/c0mpute-core/src/hosting/replicas.rs
index 92d326c3..b56e7705 100644
--- a/node/crates/c0mpute-core/src/hosting/replicas.rs
+++ b/node/crates/c0mpute-core/src/hosting/replicas.rs
@@ -63,6 +63,27 @@ pub struct Holding {
/// nobody announces go first, and then new ones are refused.
pub const MAX_RECORDS: usize = 200_000;
pub const MAX_RECEIPT_LISTS: usize = 200_000;
+/// Proofs (see [`Proof`]) a node remembers. Past the cap the one passed
+/// longest ago makes room.
+pub const MAX_PROOFS: usize = 200_000;
+
+/// What this node has itself seen one holder of one site prove: it asked the
+/// holder for chunks it picked at random and got the right bytes back. A
+/// holding announcement is only a claim (any key can gossip one); this is
+/// what makes a claimed copy count.
+#[derive(Clone, Debug)]
+pub struct Proof {
+ /// The version the passes were for. A new version starts over.
+ pub manifest: String,
+ /// The first pass of this run of passes, and the latest.
+ pub first: Instant,
+ pub last: Instant,
+ /// Passes counted in the run. At most one is counted per gap (see
+ /// [`ReplicaMap::note_pass`]), so three passes are three separate times.
+ pub passes: u32,
+ /// When a pass was last counted.
+ counted: Instant,
+}
#[derive(Debug, Default)]
pub struct ReplicaMap {
@@ -74,6 +95,13 @@ pub struct ReplicaMap {
pub rtt_ms: HashMap,
/// Recent availability receipts per (site id, holder), newest last.
pub receipts: HashMap<(String, String), Vec>,
+ /// Challenges this node saw pass, per (site id, holder).
+ pub proofs: HashMap<(String, String), Proof>,
+ /// Count a holder on its announcement alone, as every version up to
+ /// 0.3.64 did (`[hosting] require_proofs` off, the default). With it
+ /// false a holder counts only once proven ([`Self::proven`]) and may only
+ /// be relied on for a drop once it has standing ([`Self::has_standing`]).
+ pub count_claims: bool,
}
impl ReplicaMap {
@@ -336,6 +364,128 @@ impl ReplicaMap {
.collect()
}
+ /// `holder` just passed a challenge by this node for `manifest` of
+ /// `site_id`. A pass less than `gap` after the last counted one renews
+ /// the proof without adding to the count; a run with no pass for `lapse`
+ /// (or for another version) starts over.
+ pub fn note_pass(&mut self, site_id: &str, holder: &str, manifest: &str, now: Instant, gap: Duration, lapse: Duration) {
+ self.note_pass_capped(site_id, holder, manifest, now, gap, lapse, MAX_PROOFS);
+ }
+
+ /// [`Self::note_pass`] with the table's cap given (tests use a small one).
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) fn note_pass_capped(
+ &mut self,
+ site_id: &str,
+ holder: &str,
+ manifest: &str,
+ now: Instant,
+ gap: Duration,
+ lapse: Duration,
+ max: usize,
+ ) {
+ let key = (site_id.to_string(), holder.to_string());
+ if let Some(p) = self.proofs.get_mut(&key)
+ && p.manifest == manifest
+ && now.duration_since(p.last) <= lapse
+ {
+ p.last = now;
+ if now.duration_since(p.counted) >= gap {
+ p.passes += 1;
+ p.counted = now;
+ }
+ return;
+ }
+ if !self.proofs.contains_key(&key) && self.proofs.len() >= max {
+ let oldest = self.proofs.iter().min_by_key(|(_, p)| p.last).map(|(k, _)| k.clone());
+ if let Some(k) = oldest {
+ self.proofs.remove(&k);
+ }
+ }
+ self.proofs.insert(
+ key,
+ Proof {
+ manifest: manifest.to_string(),
+ first: now,
+ last: now,
+ passes: 1,
+ counted: now,
+ },
+ );
+ }
+
+ /// Forget proofs with no pass for `lapse`, and those of sites no longer
+ /// known.
+ pub fn prune_proofs(&mut self, now: Instant, lapse: Duration) {
+ let records = &self.records;
+ self.proofs
+ .retain(|(site, _), p| now.duration_since(p.last) <= lapse && records.contains_key(site));
+ }
+
+ /// Whether `holder`'s copy of the current version of `site_id` counts
+ /// toward the copies the site needs: this node saw it pass a challenge
+ /// within `ttl`, or the latest receipt for it signed by the site's owner
+ /// or payer (the keys [`proof::demoted`] trusts) is a pass no older than
+ /// `ttl`. `now_secs` is the wall clock receipts are dated with.
+ pub fn proven(&self, site_id: &str, holder: &str, now: Instant, now_secs: u64, ttl: Duration) -> bool {
+ if self.count_claims {
+ return true;
+ }
+ let Some(rec) = self.records.get(site_id) else {
+ return false;
+ };
+ if self
+ .proofs
+ .get(&(site_id.to_string(), holder.to_string()))
+ .is_some_and(|p| p.manifest == rec.manifest && now.duration_since(p.last) <= ttl)
+ {
+ return true;
+ }
+ let trusted = [rec.owner.as_str(), rec.payer_key()];
+ self.receipts_for(site_id, holder)
+ .iter()
+ .filter(|r| r.manifest == rec.manifest && trusted.contains(&r.challenger.as_str()))
+ // A receipt dated ahead of this node's clock is not fresher
+ // than one dated now.
+ .filter(|r| r.at <= now_secs.saturating_add(proof::CLOCK_SKEW_SECS))
+ .max_by(|a, b| a.at.cmp(&b.at).then(a.sig.cmp(&b.sig)))
+ .is_some_and(|r| r.ok && now_secs.saturating_sub(r.at) <= ttl.as_secs())
+ }
+
+ /// Whether `holder` has the standing a copy needs before this node gives
+ /// up its own on the strength of it: this node saw it pass at least
+ /// `passes` challenges at separate times, the first at least `span` ago,
+ /// with no gap of `span` between passes since. Receipts from other keys
+ /// do not count here: only what this node asked for itself.
+ pub fn has_standing(&self, site_id: &str, holder: &str, now: Instant, passes: u32, span: Duration) -> bool {
+ if self.count_claims {
+ return true;
+ }
+ let Some(rec) = self.records.get(site_id) else {
+ return false;
+ };
+ self.proofs
+ .get(&(site_id.to_string(), holder.to_string()))
+ .is_some_and(|p| {
+ p.manifest == rec.manifest
+ && p.passes >= passes
+ && now.duration_since(p.first) >= span
+ && now.duration_since(p.last) <= span
+ })
+ }
+
+ /// Whether a pass by `holder` now would be counted (see
+ /// [`Self::note_pass`]): nothing counted for the current version within
+ /// `gap`. Asking sooner proves nothing new.
+ pub fn pass_due(&self, site_id: &str, holder: &str, now: Instant, gap: Duration) -> bool {
+ let Some(rec) = self.records.get(site_id) else {
+ return false;
+ };
+ self.proofs
+ .get(&(site_id.to_string(), holder.to_string()))
+ .is_none_or(|p| p.manifest != rec.manifest || now.duration_since(p.counted) >= gap)
+ }
+
/// Forget peers not heard from within `ttl`.
pub fn prune(&mut self, now: Instant, ttl: Duration) {
self.peers
@@ -636,8 +786,9 @@ pub enum Action {
Drop,
}
-/// Decide for `me` (which may or may not host). `trim`: the site has had more
-/// than `target` copies for the whole grace period (see [`releases`]).
+/// Decide for `me` (which may or may not host), taking every announced
+/// holder at its word. `trim`: the site has had more than `target` copies for
+/// the whole grace period (see [`releases`]).
pub fn decide(
map: &ReplicaMap,
site_id: &str,
@@ -646,6 +797,29 @@ pub fn decide(
target: usize,
max: usize,
trim: bool,
+) -> Action {
+ decide_counting(map, site_id, me, i_host, target, max, trim, &|_| true)
+}
+
+/// [`decide`], where an announced holder fills one of the `target` copies
+/// only when `counts` says so (it proved its copy, see
+/// [`ReplicaMap::proven`]). Announcing is free, so three keys that only
+/// announce would otherwise keep a site from ever being copied.
+///
+/// A holder that does not count is still left out of the candidates and is
+/// still a source to fetch from. The fill takes the `target` minus counted
+/// top-ranked candidates, so however many nodes doubt the announced holders,
+/// the same few (never more than `target`) are the ones to act.
+#[allow(clippy::too_many_arguments)]
+pub fn decide_counting(
+ map: &ReplicaMap,
+ site_id: &str,
+ me: &str,
+ i_host: bool,
+ target: usize,
+ max: usize,
+ trim: bool,
+ counts: &dyn Fn(&str) -> bool,
) -> Action {
let holders = map.holders(site_id);
let i_hold = holders.iter().any(|p| p == me);
@@ -669,7 +843,8 @@ pub fn decide(
if preferred.iter().any(|p| p == me) {
return Action::Replicate;
}
- if holders.len() >= target {
+ let counted = holders.iter().filter(|p| counts(p)).count();
+ if counted >= target {
return Action::Nothing;
}
// The rendezvous fill. Preferred hosts that do not hold yet are left out:
@@ -693,7 +868,7 @@ pub fn decide(
}
candidates.push(me.to_string());
}
- if chosen_by_space(map, site_id, &candidates, target - holders.len())
+ if chosen_by_space(map, site_id, &candidates, target - counted)
.iter()
.any(|p| p == me)
{
@@ -1027,6 +1202,204 @@ mod tests {
assert!(m.live_peers().is_empty());
}
+ /// A site with `claimants` announcing it and `spare` hosting nodes that
+ /// do not hold it. Returns (map, site id, manifest, claimants, spares).
+ fn claimed(claimants: usize, spare: usize) -> (ReplicaMap, String, String, Vec, Vec) {
+ let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let r = rec(&kp, 1);
+ let mut m = ReplicaMap::default();
+ m.apply_record(r.clone());
+ let fake: Vec = (0..claimants).map(|i| format!("claim{i}")).collect();
+ holding(&mut m, &r.site_id, &r.manifest, &fake);
+ let spares: Vec = (0..spare).map(|i| format!("spare{i:02}")).collect();
+ hosting_peers(&mut m, &spares);
+ (m, r.site_id, r.manifest, fake, spares)
+ }
+
+ const TTL: Duration = Duration::from_secs(1800);
+ const GAP: Duration = Duration::from_secs(7200);
+ const SPAN: Duration = Duration::from_secs(6 * 3600);
+
+ /// The `peers` that would take a copy when only proven holders count.
+ fn takers(m: &ReplicaMap, site: &str, peers: &[String], now: Instant) -> Vec {
+ peers
+ .iter()
+ .filter(|p| {
+ let counts = |h: &str| h == p.as_str() || m.proven(site, h, now, 1_000_000, TTL);
+ decide_counting(m, site, p, true, 3, 5, false, &counts) == Action::Replicate
+ })
+ .cloned()
+ .collect()
+ }
+
+ #[test]
+ fn holders_that_only_announce_do_not_fill_the_target() {
+ let (m, site, _, _, spares) = claimed(3, 6);
+ let now = Instant::now();
+ // Taken at their word, three announcements are three copies.
+ assert!(spares.iter().all(|p| decide(&m, &site, p, true, 3, 5, false) == Action::Nothing));
+ // Counted only once proven, the site still needs all three, and
+ // exactly the three top-ranked spare nodes take them: not everyone
+ // who doubts the claims.
+ let t = takers(&m, &site, &spares, now);
+ assert_eq!(t.len(), 3, "{t:?}");
+ let mut want = chosen(&site, &spares, 3);
+ want.sort();
+ assert_eq!(t, want);
+ }
+
+ #[test]
+ fn proven_holders_fill_the_target_and_a_stale_proof_expires() {
+ let (mut m, site, manifest, fake, spares) = claimed(3, 6);
+ let t0 = Instant::now();
+ for p in &fake[..2] {
+ m.note_pass(&site, p, &manifest, t0, GAP, SPAN);
+ }
+ // Two proven: one more copy, from the top-ranked spare node alone.
+ assert_eq!(takers(&m, &site, &spares, t0), chosen(&site, &spares, 1));
+ m.note_pass(&site, &fake[2], &manifest, t0, GAP, SPAN);
+ assert!(takers(&m, &site, &spares, t0).is_empty(), "three proven copies: nothing to do");
+ assert!(takers(&m, &site, &spares, t0 + TTL).is_empty());
+ // Past the TTL the proofs no longer count.
+ let late = t0 + TTL + Duration::from_secs(1);
+ assert!(!m.proven(&site, &fake[0], late, 1_000_000, TTL));
+ assert_eq!(takers(&m, &site, &spares, late).len(), 3);
+ // A proof is for one version of the site.
+ m.note_pass(&site, &fake[0], "another-manifest", t0, GAP, SPAN);
+ assert!(!m.proven(&site, &fake[0], t0, 1_000_000, TTL));
+ }
+
+ #[test]
+ fn the_ceiling_still_holds_when_claims_do_not_count() {
+ // Eight keys announce a site none of them holds. The fill is still
+ // three copies from the three top-ranked nodes, however many nodes
+ // look, and once those three are proven nobody else takes one.
+ let (mut m, site, manifest, fake, spares) = claimed(8, 12);
+ let now = Instant::now();
+ let first = takers(&m, &site, &spares, now);
+ assert_eq!(first.len(), 3, "{first:?}");
+ holding(&mut m, &site, &manifest, &first);
+ for p in &first {
+ m.note_pass(&site, p, &manifest, now, GAP, SPAN);
+ }
+ let rest: Vec = spares.iter().filter(|p| !first.contains(p)).cloned().collect();
+ assert!(takers(&m, &site, &rest, now).is_empty(), "three proven copies stop the fill");
+ // With eleven announced copies the site is far over the ceiling, and
+ // the rule above it is unchanged: one holder a round is named to let
+ // go (whether it does is decided on proofs of the others, see
+ // `Hosting::others_prove_whole`).
+ let all: Vec = fake.iter().chain(first.iter()).cloned().collect();
+ assert_eq!(droppers(&m, &site, &all, false).len(), 1);
+ }
+
+ #[test]
+ fn a_trusted_receipt_proves_a_holder_until_it_is_stale() {
+ use super::super::proof::{Challenge, Receipt};
+ let owner = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let stranger = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let r = rec(&owner, 1);
+ let mut m = ReplicaMap::default();
+ m.apply_record(r.clone());
+ holding(&mut m, &r.site_id, &r.manifest, &["h".to_string()]);
+ let chunk = Hash::of(b"c").to_hex();
+ let receipt = |kp: &c0mpute_net::libp2p::identity::Keypair, ok: bool, at: u64| {
+ Receipt::sign(
+ kp,
+ Challenge {
+ site_id: &r.site_id,
+ manifest: &r.manifest,
+ holder: "h",
+ chunk: &chunk,
+ ok,
+ rtt_ms: 1,
+ at,
+ },
+ )
+ .unwrap()
+ };
+ let now = Instant::now();
+ let proven = |m: &ReplicaMap, secs: u64| m.proven(&r.site_id, "h", now, secs, TTL);
+ // Anyone can sign a receipt: a stranger's proves nothing.
+ m.apply_receipt(receipt(&stranger, true, 1_000));
+ assert!(!proven(&m, 1_000));
+ m.apply_receipt(receipt(&owner, true, 1_000));
+ assert!(proven(&m, 1_000) && proven(&m, 1_000 + TTL.as_secs()));
+ assert!(!proven(&m, 1_001 + TTL.as_secs()), "an old pass is not a copy today");
+ // The owner's latest word wins, and a date from the future is not one.
+ m.apply_receipt(receipt(&owner, false, 1_100));
+ assert!(!proven(&m, 1_200));
+ m.apply_receipt(receipt(&owner, true, 900_000));
+ assert!(!proven(&m, 1_200));
+ }
+
+ #[test]
+ fn one_fresh_pass_counts_for_the_fill_but_dropping_needs_standing() {
+ let (mut m, site, manifest, fake, _) = claimed(1, 0);
+ let h = &fake[0];
+ let t0 = Instant::now();
+ let standing = |m: &ReplicaMap, at: Instant| m.has_standing(&site, h, at, 3, SPAN);
+ assert!(m.pass_due(&site, h, t0, GAP), "never asked: a pass would count");
+ m.note_pass(&site, h, &manifest, t0, GAP, SPAN);
+ assert!(m.proven(&site, h, t0, 0, TTL), "one pass fills a slot");
+ assert!(!standing(&m, t0));
+ // Passes close together are one time, however many there are.
+ for s in 1..50 {
+ m.note_pass(&site, h, &manifest, t0 + Duration::from_secs(s), GAP, SPAN);
+ }
+ assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 1);
+ assert!(!m.pass_due(&site, h, t0 + Duration::from_secs(60), GAP));
+ assert!(!standing(&m, t0 + SPAN), "six hours since one pass is not three passes");
+ // Three passes two hours apart, but only four hours in: not yet.
+ m.note_pass(&site, h, &manifest, t0 + GAP, GAP, SPAN);
+ m.note_pass(&site, h, &manifest, t0 + GAP * 2, GAP, SPAN);
+ assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 3);
+ assert!(!standing(&m, t0 + GAP * 2));
+ // A pass at six hours: three or more separate times spanning six hours.
+ m.note_pass(&site, h, &manifest, t0 + SPAN, GAP, SPAN);
+ assert!(standing(&m, t0 + SPAN));
+ }
+
+ #[test]
+ fn standing_lapses_and_the_count_starts_over() {
+ let (mut m, site, manifest, fake, _) = claimed(1, 0);
+ let h = &fake[0];
+ let t0 = Instant::now();
+ for i in 0..4 {
+ m.note_pass(&site, h, &manifest, t0 + GAP * i, GAP, SPAN);
+ }
+ let end = t0 + GAP * 3;
+ assert!(m.has_standing(&site, h, end, 3, SPAN));
+ // No pass for a whole span: the standing is gone...
+ let late = end + SPAN + Duration::from_secs(1);
+ assert!(!m.has_standing(&site, h, late, 3, SPAN));
+ // ...and the next pass is the first of a new run, not the fifth.
+ m.note_pass(&site, h, &manifest, late, GAP, SPAN);
+ let p = &m.proofs[&(site.clone(), h.clone())];
+ assert_eq!((p.passes, p.first), (1, late));
+ // A new version of the site starts over too.
+ m.note_pass(&site, h, "next-version", late + GAP, GAP, SPAN);
+ assert_eq!(m.proofs[&(site.clone(), h.clone())].passes, 1);
+ // Pruning drops what lapsed, and proofs of sites no longer known.
+ m.note_pass("gone-site", h, &manifest, late + GAP, GAP, SPAN);
+ m.prune_proofs(late + GAP, SPAN);
+ assert_eq!(m.proofs.len(), 1);
+ m.prune_proofs(late + GAP + SPAN + Duration::from_secs(1), SPAN);
+ assert!(m.proofs.is_empty());
+ }
+
+ #[test]
+ fn the_proof_table_is_capped_and_the_oldest_pass_makes_room() {
+ let mut m = ReplicaMap::default();
+ let t0 = Instant::now();
+ for i in 0..3u64 {
+ m.note_pass_capped("s", &format!("h{i}"), "m", t0 + Duration::from_secs(i), GAP, SPAN, 3);
+ }
+ m.note_pass_capped("s", "h9", "m", t0 + Duration::from_secs(9), GAP, SPAN, 3);
+ assert_eq!(m.proofs.len(), 3);
+ assert!(!m.proofs.contains_key(&("s".to_string(), "h0".to_string())));
+ assert!(m.proofs.contains_key(&("s".to_string(), "h9".to_string())));
+ }
+
fn pid() -> String {
c0mpute_net::libp2p::identity::Keypair::generate_ed25519()
.public()
diff --git a/node/crates/c0mpute-core/src/hosting/stripe.rs b/node/crates/c0mpute-core/src/hosting/stripe.rs
index 730b7e8f..bcba49b8 100644
--- a/node/crates/c0mpute-core/src/hosting/stripe.rs
+++ b/node/crates/c0mpute-core/src/hosting/stripe.rs
@@ -239,6 +239,27 @@ pub fn plan(
max: usize,
widen: &dyn Fn(u32) -> usize,
trim: &dyn Fn(u32) -> bool,
+) -> Plan {
+ plan_counting(map, site_id, layout, me, i_host, target, max, widen, trim, &|_| true)
+}
+
+/// [`plan`], where an announced holder of a group fills one of its `target`
+/// copies only when `counts` says so (it proved a copy of the site, see
+/// `ReplicaMap::proven`), as in `replicas::decide_counting`. Releasing is
+/// unchanged: it is decided on every announced holder and then checked
+/// against proofs by the caller.
+#[allow(clippy::too_many_arguments)]
+pub fn plan_counting(
+ map: &ReplicaMap,
+ site_id: &str,
+ layout: &Layout,
+ me: &str,
+ i_host: bool,
+ target: usize,
+ max: usize,
+ widen: &dyn Fn(u32) -> usize,
+ trim: &dyn Fn(u32) -> bool,
+ counts: &dyn Fn(&str) -> bool,
) -> Plan {
let holders = map.group_holders(site_id, layout.len(), true);
let preferred = map.effective_preferred(site_id);
@@ -278,7 +299,8 @@ pub fn plan(
wanted.push((0, score(&key, me), g));
continue;
}
- if hs.len() >= target {
+ let counted = hs.iter().filter(|p| counts(p)).count();
+ if counted >= target {
continue;
}
if hs.iter().any(|h| same_machine(&machines, h, me)) {
@@ -308,9 +330,9 @@ pub fn plan(
}
candidates.push(me.to_string());
}
- let n = target - hs.len() + widen(g);
+ let n = target - counted + widen(g);
if chosen(&key, &candidates, n).iter().any(|p| p == me) {
- wanted.push((1 + hs.len(), score(&key, me), g));
+ wanted.push((1 + counted, score(&key, me), g));
}
}
// Paid groups, then groups with fewest copies, then a stable order.
@@ -469,6 +491,39 @@ mod tests {
map.set_machine(p, machine);
}
+ #[test]
+ fn groups_whose_holders_only_announce_are_still_filled() {
+ let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
+ let m = manifest(64 * 8);
+ let rec = big_record(&kp, &m);
+ let layout = Layout::of(&m);
+ let n = layout.len() as u32;
+ let peers: Vec = (0..9).map(|i| format!("peer{i}")).collect();
+ let mut map = map_with(&peers, &rec);
+ // Three keys announce every group.
+ for p in &peers[..3] {
+ hold_all(&mut map, &rec, p, 0..n);
+ }
+ let takers = |counts: &dyn Fn(&str) -> bool| -> Vec {
+ let mut t = vec![0usize; layout.len()];
+ for p in &peers[3..] {
+ let mine = |h: &str| h == p.as_str() || counts(h);
+ let plan = plan_counting(&map, &rec.site_id, &layout, p, true, 3, 5, &|_| 0, &|_| false, &mine);
+ assert!(plan.drop.is_empty());
+ for g in plan.fetch {
+ t[g as usize] += 1;
+ }
+ }
+ t
+ };
+ // Taken at their word (or all proven) every group is full.
+ assert!(takers(&|_| true).iter().all(|t| *t == 0));
+ // Unproven, they fill nothing: each group gets three real takers.
+ assert!(takers(&|_| false).iter().all(|t| *t == 3));
+ // One of them proven: two more takers per group.
+ assert!(takers(&|h| h == "peer0").iter().all(|t| *t == 2));
+ }
+
#[test]
fn two_nodes_on_one_machine_are_one_copy() {
let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519();
diff --git a/node/crates/c0mpute-core/src/render.rs b/node/crates/c0mpute-core/src/render.rs
index 75e44edf..bde08c5c 100644
--- a/node/crates/c0mpute-core/src/render.rs
+++ b/node/crates/c0mpute-core/src/render.rs
@@ -21,7 +21,11 @@
//! Safety: Blender always runs with `--disable-autoexec` before the file is
//! loaded, so Python embedded in the `.blend` (drivers, registered text
//! blocks, app handlers) never runs. The only Python is the one-line settings
-//! expression this module builds from validated integers.
+//! expression this module builds from validated integers, and the constant
+//! [`SANITIZE_EXPR`], which cuts every file reference that leaves the job
+//! directory before the render starts. Where bubblewrap works (Linux),
+//! Blender also runs inside it, with no network and nothing of the host but
+//! the system directories and the job directory ([`Sandbox`]).
use std::collections::HashMap;
use std::ffi::OsString;
@@ -74,6 +78,65 @@ pub const MAX_SAMPLES: u32 = 16_384;
const CACHE_MAX_BYTES: u64 = 4 * 1024 * 1024 * 1024;
const PROBE_TIMEOUT: Duration = Duration::from_secs(90);
+// ─── config ──────────────────────────────────────────────────────────────
+
+/// `[render]` in config.toml.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(default)]
+pub struct RenderConfig {
+ /// `"auto"` (the default): run Blender inside bubblewrap where it works,
+ /// otherwise with scene sanitisation only. `"off"`: never use
+ /// bubblewrap. `"require"`: serve no render that cannot be sandboxed.
+ /// `C0MPUTE_RENDER_SANDBOX` wins over the file.
+ pub sandbox: String,
+}
+
+impl Default for RenderConfig {
+ fn default() -> Self {
+ Self { sandbox: "auto".into() }
+ }
+}
+
+impl RenderConfig {
+ /// The mode `sandbox` (or `env`, when set) names. A value that is not
+ /// one of the three is read as `auto`, so a typo never stops a node.
+ pub fn mode_with(&self, env: Option) -> SandboxMode {
+ let value = env.filter(|v| !v.trim().is_empty()).unwrap_or_else(|| self.sandbox.clone());
+ SandboxMode::parse(&value).unwrap_or_else(|| {
+ warn!(value = %value, "render: sandbox must be auto, off or require; using auto");
+ SandboxMode::Auto
+ })
+ }
+}
+
+/// Whether Blender runs inside bubblewrap (`[render] sandbox`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum SandboxMode {
+ #[default]
+ Auto,
+ Off,
+ Require,
+}
+
+impl SandboxMode {
+ pub fn parse(s: &str) -> Option {
+ match s.trim().to_ascii_lowercase().as_str() {
+ "auto" => Some(Self::Auto),
+ "off" => Some(Self::Off),
+ "require" => Some(Self::Require),
+ _ => None,
+ }
+ }
+}
+
+/// The node's `[render] sandbox`, from its config file and the environment.
+/// The render role is started without a config handle, so this reads the
+/// file itself, as the speech gateway does for `[adult]`.
+fn sandbox_mode() -> SandboxMode {
+ let file = crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default();
+ file.render.mode_with(std::env::var("C0MPUTE_RENDER_SANDBOX").ok())
+}
+
// ─── spec ────────────────────────────────────────────────────────────────
/// Render engine. `EEVEE` is mapped to the identifier the worker's Blender
@@ -634,25 +697,36 @@ pub struct Runtime {
pub version: (u32, u32, u32),
/// Cycles GPU backends with at least one device, best first.
pub gpu_backends: Vec,
+ /// The bubblewrap sandbox Blender runs in, where one works.
+ pub sandbox: Option,
+ pub sandbox_mode: SandboxMode,
}
/// Preference order when several backends see devices.
const GPU_BACKENDS: &[&str] = &["OPTIX", "CUDA", "HIP", "METAL", "ONEAPI"];
impl Runtime {
+ /// Blender answers and, with `sandbox = "require"`, the sandbox works.
pub fn ready(&self) -> bool {
self.version_line.is_some()
+ && (self.sandbox_mode != SandboxMode::Require || self.sandbox.is_some())
}
pub fn gpu(&self) -> Option<&str> {
self.gpu_backends.first().map(String::as_str)
}
+ /// Outside `require` every job may run; with it, only what the sandbox
+ /// was seen to do (`can`).
+ fn sandbox_allows(&self, can: impl Fn(&Sandbox) -> bool) -> bool {
+ self.sandbox_mode != SandboxMode::Require || self.sandbox.as_ref().is_some_and(can)
+ }
pub fn can_serve(&self, parsed: &Parsed) -> bool {
self.can_serve_engine(parsed.engine, parsed.device)
}
pub fn can_serve_engine(&self, engine: Engine, device: Device) -> bool {
self.ready()
&& match (engine.needs_gpu(), device) {
- (true, _) | (false, Device::Gpu) => self.gpu().is_some(),
+ (true, _) => self.gpu().is_some() && self.sandbox_allows(|s| s.raster),
+ (false, Device::Gpu) => self.gpu().is_some() && self.sandbox_allows(|s| s.cycles_gpu),
(false, _) => true,
}
}
@@ -661,22 +735,47 @@ impl Runtime {
if parsed.engine != Engine::Cycles || parsed.device == Device::Cpu {
return None;
}
+ // `require` with a sandbox that cannot reach the GPU: AUTO means CPU.
+ if !self.sandbox_allows(|s| s.cycles_gpu) {
+ return None;
+ }
self.gpu().map(str::to_string)
}
+ /// The sandbox this job's Blender runs in. `None` runs it directly:
+ /// there is no sandbox, or (outside `require`) the job needs a GPU the
+ /// sandbox was not seen to reach.
+ pub fn sandbox_for(&self, parsed: &Parsed) -> Option<&Sandbox> {
+ let sandbox = self.sandbox.as_ref()?;
+ let usable = if parsed.engine.needs_gpu() {
+ sandbox.raster
+ } else if self.cycles_device(parsed).is_some() {
+ sandbox.cycles_gpu
+ } else {
+ true
+ };
+ usable.then_some(sandbox)
+ }
pub fn tags(&self) -> Vec {
if !self.ready() {
return Vec::new();
}
let mut tags = vec![ROLE_TAG.to_string(), Engine::Cycles.tag()];
if self.gpu().is_some() {
- tags.push(GPU_TAG.to_string());
- tags.push(Engine::Eevee.tag());
- tags.push(Engine::Workbench.tag());
- tags.extend(
- self.gpu_backends
- .iter()
- .map(|b| format!("c0mpute:render:device:{}", b.to_ascii_lowercase())),
- );
+ let cycles = self.sandbox_allows(|s| s.cycles_gpu);
+ if cycles {
+ tags.push(GPU_TAG.to_string());
+ }
+ if self.sandbox_allows(|s| s.raster) {
+ tags.push(Engine::Eevee.tag());
+ tags.push(Engine::Workbench.tag());
+ }
+ if cycles {
+ tags.extend(
+ self.gpu_backends
+ .iter()
+ .map(|b| format!("c0mpute:render:device:{}", b.to_ascii_lowercase())),
+ );
+ }
}
tags
}
@@ -762,12 +861,29 @@ fn probe() -> Runtime {
.map(|out| parse_gpu_probe(&out))
.unwrap_or_default()
};
- let rt = Runtime {
+ let mut rt = Runtime {
version_line: Some(line),
version,
gpu_backends,
+ sandbox: None,
+ sandbox_mode: sandbox_mode(),
};
info!(blender = ?rt.version_line, gpu = ?rt.gpu_backends, "render: blender found");
+ if rt.sandbox_mode != SandboxMode::Off {
+ rt.sandbox = probe_sandbox(&rt.gpu_backends);
+ }
+ // Said once: the probe runs once per process.
+ match (&rt.sandbox, rt.sandbox_mode) {
+ (Some(sb), _) => info!(
+ bwrap = %sb.bwrap.display(),
+ cycles_gpu = sb.cycles_gpu,
+ raster = sb.raster,
+ "render: blender runs inside bubblewrap (no network, only the job directory writable)"
+ ),
+ (None, SandboxMode::Off) => info!("render: [render] sandbox is off; renders run with scene sanitisation only"),
+ (None, SandboxMode::Auto) => info!("render: bubblewrap does not work on this machine; renders run with scene sanitisation only"),
+ (None, SandboxMode::Require) => warn!("render: [render] sandbox is \"require\" but bubblewrap does not work on this machine; not serving renders"),
+ }
rt
}
@@ -803,6 +919,269 @@ pub fn parse_gpu_probe(text: &str) -> Vec {
.collect()
}
+// ─── the sandbox ─────────────────────────────────────────────────────────
+
+/// One thing of the host the sandbox shows.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Mount {
+ /// A file or directory, read-only, at its own path.
+ RoBind(PathBuf),
+ /// A device node (or a directory of them), usable, at its own path.
+ DevBind(PathBuf),
+ /// A symlink the host has (`/bin` to `usr/bin`), made again.
+ Symlink { target: PathBuf, link: PathBuf },
+}
+
+/// How Blender is confined on this machine, found once by the probe: a
+/// bubblewrap (`bwrap`) command line that was seen to start this Blender.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Sandbox {
+ pub bwrap: PathBuf,
+ /// The Blender binary with symlinks resolved; run by this path inside.
+ pub blender: PathBuf,
+ pub mounts: Vec,
+ /// The operator's home directory: an empty tmpfs inside.
+ pub home: Option,
+ /// The worker is root, so bubblewrap is told to drop every capability
+ /// (it keeps them for uid 0 otherwise).
+ pub as_root: bool,
+ /// Cycles found the same GPU backends inside the sandbox as outside.
+ pub cycles_gpu: bool,
+ /// A Workbench frame rendered inside the sandbox (EEVEE and Workbench
+ /// need a GPU context, which a missing display socket can take away).
+ pub raster: bool,
+}
+
+/// The `bwrap` arguments that come before the Blender binary: no network,
+/// new pid, ipc and hostname namespaces, an empty root with a private
+/// `/tmp` and home, the host's `mounts`, and `job_dir` (absolute) as the
+/// only writable part of the host. `blend`, the job's input inside it, is
+/// read-only again: it can be a hard link to the node's cached copy. Pure,
+/// so it is tested without bwrap.
+pub fn bwrap_args(sandbox: &Sandbox, job_dir: &Path, blend: Option<&Path>) -> Vec {
+ let mut a: Vec = [
+ "--unshare-net",
+ "--unshare-pid",
+ "--unshare-ipc",
+ "--unshare-uts",
+ "--hostname",
+ "c0mpute-render",
+ "--die-with-parent",
+ "--new-session",
+ ]
+ .into_iter()
+ .map(OsString::from)
+ .collect();
+ if sandbox.as_root {
+ a.extend(["--cap-drop", "ALL"].map(OsString::from));
+ }
+ a.extend(["--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp"].map(OsString::from));
+ // The tmpfs comes before the binds: a Blender or a job directory under
+ // the home (or under /tmp) is then mounted on top of it.
+ if let Some(home) = &sandbox.home {
+ a.push("--tmpfs".into());
+ a.push(home.into());
+ }
+ for mount in &sandbox.mounts {
+ let (flag, from, to) = match mount {
+ Mount::RoBind(p) => ("--ro-bind", p, p),
+ Mount::DevBind(p) => ("--dev-bind", p, p),
+ Mount::Symlink { target, link } => ("--symlink", target, link),
+ };
+ a.push(flag.into());
+ a.push(from.into());
+ a.push(to.into());
+ }
+ a.push("--bind".into());
+ a.push(job_dir.into());
+ a.push(job_dir.into());
+ if let Some(blend) = blend {
+ a.push("--ro-bind".into());
+ a.push(blend.into());
+ a.push(blend.into());
+ }
+ a.push("--setenv".into());
+ a.push("HOME".into());
+ a.push(sandbox.home.clone().unwrap_or_else(|| PathBuf::from("/tmp")).into());
+ a.push("--chdir".into());
+ a.push(job_dir.into());
+ a.push("--".into());
+ a
+}
+
+/// Is `path` under something already bound read-only?
+#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))]
+fn covered(mounts: &[Mount], path: &Path) -> bool {
+ mounts.iter().any(|m| matches!(m, Mount::RoBind(root) if path.starts_with(root)))
+}
+
+/// May a directory outside the system directories be shown read-only? Not
+/// the root, not a top-level directory, and nothing that holds the home.
+#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))]
+fn shareable(dir: &Path, home: Option<&Path>) -> bool {
+ dir.is_absolute()
+ && dir.components().filter(|c| matches!(c, std::path::Component::Normal(_))).count() >= 2
+ && !home.is_some_and(|h| h.starts_with(dir))
+}
+
+/// Sets a 16x16 frame for the sandbox's Workbench test render.
+#[cfg(target_os = "linux")]
+const RASTER_PROBE_EXPR: &str = "import bpy\n\
+s = bpy.context.scene\n\
+s.render.resolution_x = 16\n\
+s.render.resolution_y = 16\n\
+s.render.resolution_percentage = 100\n";
+
+/// What `/etc` holds that a dynamically linked, GPU-using Blender reads.
+/// Never the whole directory: a root worker can read all of it.
+#[cfg(target_os = "linux")]
+const ETC_ENTRIES: &[&str] = &["ld.so.cache", "ld.so.conf", "ld.so.conf.d", "alternatives", "OpenCL", "vulkan", "glvnd", "egl"];
+
+#[cfg(target_os = "linux")]
+fn find_on_path(name: &str) -> Option {
+ std::env::split_paths(&std::env::var_os("PATH")?).map(|dir| dir.join(name)).find(|p| p.is_file())
+}
+
+/// Show `path` read-only where it really lives, keeping the symlink when it
+/// is one (`/bin`, `/opt/rocm`).
+#[cfg(target_os = "linux")]
+fn add_ro(mounts: &mut Vec, path: &Path) {
+ let (Ok(meta), Ok(real)) = (std::fs::symlink_metadata(path), path.canonicalize()) else {
+ return;
+ };
+ if meta.file_type().is_symlink()
+ && let Ok(target) = std::fs::read_link(path)
+ {
+ mounts.push(Mount::Symlink { target, link: path.to_path_buf() });
+ }
+ if !covered(mounts, &real) {
+ mounts.push(Mount::RoBind(real));
+ }
+}
+
+/// What of this host Blender needs: the system directories, the loader and
+/// GPU vendor files under `/etc`, `/sys` and the GPU device nodes, and the
+/// directory Blender is installed in when that is not a system directory (a
+/// blender.org tarball under `/opt` or the operator's home). `None` when
+/// showing that directory would show the home itself.
+#[cfg(target_os = "linux")]
+fn host_mounts(blender: &Path, home: Option<&Path>) -> Option> {
+ let mut mounts = Vec::new();
+ for dir in ["/usr", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/libx32", "/sys", "/opt/rocm"] {
+ add_ro(&mut mounts, Path::new(dir));
+ }
+ for name in ETC_ENTRIES {
+ add_ro(&mut mounts, &Path::new("/etc").join(name));
+ }
+ // Driver libraries the operator points the loader at (a CUDA or ROCm
+ // install outside the system directories).
+ if let Some(paths) = std::env::var_os("LD_LIBRARY_PATH") {
+ for dir in std::env::split_paths(&paths) {
+ if let Ok(real) = dir.canonicalize()
+ && real.is_dir()
+ && !covered(&mounts, &real)
+ && shareable(&real, home)
+ {
+ mounts.push(Mount::RoBind(real));
+ }
+ }
+ }
+ let install = blender.parent()?;
+ if !covered(&mounts, install) {
+ if !shareable(install, home) {
+ return None;
+ }
+ mounts.push(Mount::RoBind(install.to_path_buf()));
+ }
+ for dev in ["/dev/dri", "/dev/kfd"] {
+ if Path::new(dev).exists() {
+ mounts.push(Mount::DevBind(dev.into()));
+ }
+ }
+ let mut nvidia: Vec = std::fs::read_dir("/dev")
+ .into_iter()
+ .flatten()
+ .flatten()
+ .filter(|e| e.file_name().to_string_lossy().starts_with("nvidia"))
+ .map(|e| e.path())
+ .collect();
+ nvidia.sort();
+ mounts.extend(nvidia.into_iter().map(Mount::DevBind));
+ Some(mounts)
+}
+
+/// Find out, once, whether bubblewrap can run this machine's Blender: build
+/// the sandbox and start Blender inside it. A machine without `bwrap`, with
+/// unprivileged user namespaces switched off (Ubuntu 24.04's AppArmor
+/// default without a bwrap profile), or with a Blender the sandbox cannot
+/// start (a snap, a wrapper script around an install elsewhere) gets `None`.
+/// With a GPU, also checks that Cycles and Workbench still reach it inside.
+#[cfg(target_os = "linux")]
+fn probe_sandbox(gpu_backends: &[String]) -> Option {
+ let bwrap = find_on_path("bwrap")?;
+ let blender = find_on_path("blender")?.canonicalize().ok()?;
+ let home = std::env::var_os("HOME")
+ .map(PathBuf::from)
+ .filter(|h| h.is_absolute() && h.parent().is_some());
+ let mounts = host_mounts(&blender, home.as_deref())?;
+ let mut sandbox = Sandbox {
+ bwrap,
+ blender,
+ mounts,
+ home,
+ as_root: crate::container_run::is_root(),
+ cycles_gpu: false,
+ raster: false,
+ };
+ let dir = std::env::temp_dir().join(format!("c0mpute-render-probe-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).ok()?;
+ let run = |sandbox: &Sandbox, args: &[&std::ffi::OsStr]| {
+ let mut cmd = std::process::Command::new(&sandbox.bwrap);
+ cmd.args(bwrap_args(sandbox, &dir, None)).arg(&sandbox.blender).args(args);
+ run_with_deadline(cmd, PROBE_TIMEOUT)
+ };
+ let os = |s: &'static str| std::ffi::OsStr::new(s);
+ let works = run(&sandbox, &[os("--factory-startup"), os("--version")])
+ .is_some_and(|out| parse_version(&out).is_some());
+ if works && !gpu_backends.is_empty() {
+ sandbox.cycles_gpu = run(
+ &sandbox,
+ &[os("--background"), os("--factory-startup"), os("--disable-autoexec"), os("--python-expr"), os(GPU_PROBE_EXPR)],
+ )
+ .is_some_and(|out| parse_gpu_probe(&out) == gpu_backends);
+ let frame = dir.join("probe_####");
+ sandbox.raster = run(
+ &sandbox,
+ &[
+ os("--background"),
+ os("--factory-startup"),
+ os("--disable-autoexec"),
+ os("--python-expr"),
+ os(RASTER_PROBE_EXPR),
+ os("--engine"),
+ os("BLENDER_WORKBENCH"),
+ os("--render-output"),
+ frame.as_os_str(),
+ os("--render-format"),
+ os("PNG"),
+ os("--render-frame"),
+ os("1"),
+ ],
+ )
+ .is_some()
+ && dir.join("probe_0001.png").is_file();
+ }
+ let _ = std::fs::remove_dir_all(&dir);
+ works.then_some(sandbox)
+}
+
+/// Bubblewrap is Linux only: everywhere else renders run with scene
+/// sanitisation only.
+#[cfg(not(target_os = "linux"))]
+fn probe_sandbox(_gpu_backends: &[String]) -> Option {
+ None
+}
+
/// Bid filter: skip offers this box cannot render (a GPU engine or device
/// without a GPU, or a spec it cannot read).
pub fn offer_filter() -> crate::dispatch::OfferFilter {
@@ -863,6 +1242,176 @@ pub fn settings_expr(parsed: &Parsed, cycles_device: Option<&str>) -> String {
lines.join("\n")
}
+/// The job directory and its output directory, handed to [`SANITIZE_EXPR`]
+/// in the environment so that no path is ever written into the script.
+pub const JOB_DIR_ENV: &str = "C0MPUTE_RENDER_JOB_DIR";
+pub const OUT_DIR_ENV: &str = "C0MPUTE_RENDER_OUT_DIR";
+/// Blender's exit code when [`SANITIZE_EXPR`] could not finish: the scene is
+/// not rendered at all rather than rendered as the buyer saved it.
+pub const SANITIZE_EXIT: i32 = 97;
+
+/// Python run after the file loads and before the render, on every OS: it
+/// leaves the scene able to read and write inside the job directory only.
+/// A constant: nothing from the job is formatted into it (the two
+/// directories come from the environment), so a buyer cannot reach it.
+///
+/// A `.blend` names files by path: image textures, movie clips, sounds,
+/// fonts, linked libraries, caches, sequencer strips. Blender reads them
+/// from the worker's disk when it renders, so a scene that names
+/// `/home//Pictures/x.png` would return that picture inside a frame.
+/// Every such path that is absolute or leaves the job directory is emptied
+/// (the scene then renders with Blender's pink "missing" texture, the
+/// built-in font, or without the strip or cache); data packed into the
+/// `.blend` is untouched. Linked libraries are loaded by Blender while the
+/// file opens, before this runs, so what came from them is deleted instead.
+/// Compositor File Output nodes are removed, Open Shading Language is turned
+/// off (a compiled shader may open any image by name), and every scene's
+/// output path is set to the node's own output directory.
+///
+/// Written for Blender 3.x to 5.x: attributes a version lacks are skipped
+/// (`getattr`), and any other failure exits with [`SANITIZE_EXIT`].
+pub const SANITIZE_EXPR: &str = r#"import bpy, os, sys, traceback
+
+def _c0_sanitize():
+ D = bpy.data
+ job_dir = os.environ.get('C0MPUTE_RENDER_JOB_DIR')
+ out_dir = os.environ.get('C0MPUTE_RENDER_OUT_DIR')
+ if not job_dir or not out_dir:
+ raise RuntimeError('the job and output directories are not set')
+ root = os.path.normcase(os.path.realpath(job_dir))
+ cut = [0]
+
+ def inside(path, lib=None):
+ try:
+ full = os.path.normcase(os.path.realpath(bpy.path.abspath(path, library=lib)))
+ return os.path.commonpath([root, full]) == root
+ except Exception:
+ return False
+
+ def drop(owner, attr, lib=None, to=''):
+ path = getattr(owner, attr, None)
+ if not isinstance(path, str) or path in ('', to) or inside(path, lib):
+ return False
+ setattr(owner, attr, to)
+ cut[0] += 1
+ return True
+
+ def outside_libs():
+ return [lib for lib in D.libraries if not inside(lib.filepath)]
+
+ bad = outside_libs()
+ if bad:
+ def from_bad(i):
+ if i.library in bad:
+ return True
+ ref = getattr(getattr(i, 'override_library', None), 'reference', None)
+ return ref is not None and ref.library in bad
+ D.batch_remove([i for i in D.user_map() if from_bad(i)])
+ cut[0] += len(bad)
+ for lib in outside_libs():
+ D.libraries.remove(lib)
+ if outside_libs():
+ raise RuntimeError('a linked library could not be removed')
+
+ for im in D.images:
+ packed = im.packed_file is not None or len(getattr(im, 'packed_files', ())) > 0
+ if packed and im.source in ('FILE', 'TILED'):
+ continue
+ drop(im, 'filepath', im.library)
+ for name, keep_packed in (('movieclips', False), ('sounds', True), ('volumes', True), ('cache_files', False)):
+ for item in getattr(D, name, ()):
+ if keep_packed and getattr(item, 'packed_file', None) is not None:
+ continue
+ drop(item, 'filepath', item.library)
+ for font in D.fonts:
+ if font.packed_file is None:
+ drop(font, 'filepath', font.library, '')
+
+ for ob in D.objects:
+ for mod in ob.modifiers:
+ domain = getattr(mod, 'domain_settings', None)
+ if domain is not None:
+ drop(domain, 'cache_directory', ob.library, os.path.join(job_dir, 'fluid_cache'))
+ drop(mod, 'filepath', ob.library)
+ drop(mod, 'bake_directory', ob.library)
+ for bake in getattr(mod, 'bakes', ()):
+ drop(bake, 'directory', ob.library)
+ caches = [getattr(mod, 'point_cache', None)]
+ for surface in getattr(getattr(mod, 'canvas_settings', None), 'canvas_surfaces', ()):
+ caches.append(getattr(surface, 'point_cache', None))
+ for cache in caches:
+ if cache is not None:
+ drop(cache, 'filepath', ob.library)
+ for psys in ob.particle_systems:
+ cache = getattr(psys, 'point_cache', None)
+ if cache is not None:
+ drop(cache, 'filepath', ob.library)
+
+ trees = list(D.node_groups)
+ for name in ('materials', 'worlds', 'lights', 'textures', 'linestyles', 'scenes'):
+ for item in getattr(D, name, ()):
+ tree = getattr(item, 'node_tree', None)
+ if tree is not None:
+ trees.append(tree)
+ for tree in trees:
+ for node in list(tree.nodes):
+ if node.type == 'OUTPUT_FILE':
+ try:
+ tree.nodes.remove(node)
+ except Exception:
+ node.mute = True
+ for attr in ('base_path', 'directory'):
+ if hasattr(node, attr):
+ setattr(node, attr, out_dir)
+ cut[0] += 1
+ else:
+ drop(node, 'filepath', tree.library)
+
+ for sc in D.scenes:
+ r = sc.render
+ r.filepath = os.path.join(out_dir, 'frame_####')
+ r.use_file_extension = True
+ for attr in ('use_render_cache', 'use_stamp_hostname', 'use_stamp_filename'):
+ if hasattr(r, attr):
+ setattr(r, attr, False)
+ cycles = getattr(sc, 'cycles', None)
+ if cycles is not None and hasattr(cycles, 'shading_system'):
+ cycles.shading_system = False
+ world = getattr(sc, 'rigidbody_world', None)
+ if world is not None and getattr(world, 'point_cache', None) is not None:
+ drop(world.point_cache, 'filepath', sc.library)
+ ed = sc.sequence_editor
+ if ed is None:
+ continue
+ if drop(ed, 'proxy_dir', sc.library) and hasattr(ed, 'proxy_storage'):
+ ed.proxy_storage = 'PER_STRIP'
+ strips = getattr(ed, 'strips_all', None)
+ if strips is None:
+ strips = getattr(ed, 'sequences_all', ())
+ for strip in strips:
+ hit = drop(strip, 'filepath', sc.library)
+ hit = drop(strip, 'directory', sc.library) or hit
+ if hit:
+ strip.mute = True
+ proxy = getattr(strip, 'proxy', None)
+ if proxy is not None:
+ hit = drop(proxy, 'filepath', sc.library)
+ hit = drop(proxy, 'directory', sc.library) or hit
+ if hit and hasattr(strip, 'use_proxy'):
+ strip.use_proxy = False
+
+ left = [p for p in bpy.utils.blend_paths(absolute=True, packed=False, local=False) if not inside(p)]
+ print('c0mpute: scene sanitised: %d outside references cut, %d path fields a render does not read left as they were' % (cut[0], len(left)))
+
+try:
+ _c0_sanitize()
+except BaseException:
+ traceback.print_exc()
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(97)
+"#;
+
/// The full `blender` argument list for one job. `out_dir` must not contain
/// `#` (Blender's frame-number placeholder). The `.blend` path is one
/// argument (no shell is involved), and `--disable-autoexec` comes before it
@@ -885,6 +1434,8 @@ pub fn blender_args(
args.push(blend.as_os_str().to_owned());
args.push("--python-expr".into());
args.push(settings_expr(parsed, cycles_device).into());
+ args.push("--python-expr".into());
+ args.push(SANITIZE_EXPR.into());
args.push("--engine".into());
args.push(parsed.engine.blender_id(version).into());
args.push("--render-output".into());
@@ -931,9 +1482,12 @@ pub struct Rendered {
/// output. Needs a ready [`runtime`].
pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result> {
let rt = runtime();
- if !rt.ready() {
+ if rt.version_line.is_none() {
bail!("blender is not installed on this node");
}
+ if !rt.ready() {
+ bail!("this node renders only inside a sandbox, and bubblewrap does not work here");
+ }
if !rt.can_serve(parsed) {
bail!(
"this node cannot render {} on {}",
@@ -941,11 +1495,29 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result tokio::fs::canonicalize(job_dir).await?,
+ None => std::path::absolute(job_dir)?,
+ };
let out_dir = job_dir.join("out");
- tokio::fs::create_dir_all(&out_dir).await?;
+ let blend = &stage_input(blend, job_dir).await?;
let cycles_device = rt.cycles_device(parsed);
- let mut command = Command::new("blender");
+ let mut command = match sandbox {
+ Some(sb) => {
+ let mut command = Command::new(&sb.bwrap);
+ command.args(bwrap_args(sb, job_dir, Some(blend.as_path()))).arg(&sb.blender);
+ command
+ }
+ None => Command::new("blender"),
+ };
+ // Environment only, so the limits reach Blender through bubblewrap too.
let threads = crate::resources::apply_job_limits(&mut command);
+ command.env(JOB_DIR_ENV, job_dir).env(OUT_DIR_ENV, &out_dir);
command.args(blender_args(
blend,
&out_dir,
@@ -972,6 +1544,10 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result = HashMap::new();
let mut entries = tokio::fs::read_dir(&out_dir).await?;
@@ -1011,6 +1587,27 @@ pub async fn render(blend: &Path, parsed: &Parsed, job_dir: &Path) -> Result/input.blend`: a hard link
+/// to the cached copy (a copy where the filesystem cannot link). Blender
+/// resolves `//relative` paths against the file's directory, and the cache
+/// directory holds other buyers' scenes; the job directory holds only this
+/// job's.
+async fn stage_input(blend: &Path, job_dir: &Path) -> Result {
+ // Already the job's own (an opened sealed input), however it is spelled.
+ let real = tokio::fs::canonicalize(blend).await.context("find the .blend")?;
+ if real.parent() == tokio::fs::canonicalize(job_dir).await.ok().as_deref()
+ && let Some(name) = blend.file_name()
+ {
+ return Ok(job_dir.join(name));
+ }
+ let staged = job_dir.join("input.blend");
+ let _ = tokio::fs::remove_file(&staged).await;
+ if tokio::fs::hard_link(blend, &staged).await.is_err() {
+ tokio::fs::copy(blend, &staged).await.context("copy the .blend into the job directory")?;
+ }
+ Ok(staged)
+}
+
async fn log_tail(path: &Path) -> String {
let text = tokio::fs::read_to_string(path).await.unwrap_or_default();
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
@@ -1934,6 +2531,8 @@ mod tests {
version_line: Some("Blender 3.4.1".into()),
version: (3, 4, 1),
gpu_backends: vec![],
+ sandbox: None,
+ sandbox_mode: SandboxMode::Auto,
};
let gpu = Runtime {
gpu_backends: vec!["CUDA".into()],
@@ -1957,6 +2556,213 @@ mod tests {
assert!(Runtime::default().tags().is_empty());
}
+ #[test]
+ fn sandbox_setting_parses_and_a_typo_means_auto() {
+ use crate::config::Config;
+ // A config written before the section existed, and the default.
+ let cfg: Config = toml::from_str("roles = []\n").unwrap();
+ assert_eq!(cfg.render.mode_with(None), SandboxMode::Auto);
+ assert_eq!(Config::default().render.sandbox, "auto");
+ let mode = |text: &str| toml::from_str::(text).unwrap().render.mode_with(None);
+ assert_eq!(mode("[render]\nsandbox = \"off\"\n"), SandboxMode::Off);
+ assert_eq!(mode("[render]\nsandbox = \"require\"\n"), SandboxMode::Require);
+ assert_eq!(mode("[render]\nsandbox = \" Require \"\n"), SandboxMode::Require);
+ // A value that is none of the three never stops the node: it is auto.
+ assert_eq!(mode("[render]\nsandbox = \"always\"\n"), SandboxMode::Auto);
+ // The environment wins when it is set to something.
+ let cfg: Config = toml::from_str("[render]\nsandbox = \"require\"\n").unwrap();
+ assert_eq!(cfg.render.mode_with(Some("off".into())), SandboxMode::Off);
+ assert_eq!(cfg.render.mode_with(Some(" ".into())), SandboxMode::Require);
+ }
+
+ fn sandbox(cycles_gpu: bool, raster: bool) -> Sandbox {
+ Sandbox {
+ bwrap: "/usr/bin/bwrap".into(),
+ blender: "/home/op/blender-4.2/blender".into(),
+ mounts: vec![
+ Mount::RoBind("/usr".into()),
+ Mount::Symlink { target: "usr/bin".into(), link: "/bin".into() },
+ Mount::RoBind("/etc/ld.so.cache".into()),
+ Mount::RoBind("/sys".into()),
+ Mount::RoBind("/home/op/blender-4.2".into()),
+ Mount::DevBind("/dev/dri".into()),
+ Mount::DevBind("/dev/nvidia0".into()),
+ ],
+ home: Some("/home/op".into()),
+ as_root: false,
+ cycles_gpu,
+ raster,
+ }
+ }
+
+ #[test]
+ fn bwrap_argv_shows_the_system_read_only_and_only_the_job_writable() {
+ let job = Path::new("/home/op/.local/share/c0mpute/cache/jobs/j1");
+ let blend = job.join("input.blend");
+ let a: Vec = bwrap_args(&sandbox(true, true), job, Some(&blend)).iter().map(|x| x.to_string_lossy().into_owned()).collect();
+ let at = |w: &[&str]| a.windows(w.len()).position(|x| x == w).unwrap_or_else(|| panic!("{w:?} not in {a:?}"));
+ for flag in ["--unshare-net", "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--die-with-parent", "--new-session"] {
+ assert!(a.iter().any(|x| x == flag), "{flag}");
+ }
+ at(&["--proc", "/proc"]);
+ at(&["--ro-bind", "/usr", "/usr"]);
+ at(&["--symlink", "usr/bin", "/bin"]);
+ at(&["--ro-bind", "/sys", "/sys"]);
+ at(&["--dev-bind", "/dev/nvidia0", "/dev/nvidia0"]);
+ // /dev is made fresh before the GPU nodes are put back in it.
+ assert!(at(&["--dev", "/dev"]) < at(&["--dev-bind", "/dev/dri", "/dev/dri"]));
+ // The home is an empty tmpfs; Blender's own directory and the job
+ // directory, both under it, are mounted on top afterwards.
+ let home = at(&["--tmpfs", "/home/op"]);
+ assert!(at(&["--tmpfs", "/tmp"]) < home);
+ assert!(home < at(&["--ro-bind", "/home/op/blender-4.2", "/home/op/blender-4.2"]));
+ let job_s = job.to_str().unwrap();
+ assert!(home < at(&["--bind", job_s, job_s]));
+ // The input (a hard link to the cached copy) is read-only inside it.
+ let blend_s = blend.to_str().unwrap();
+ assert!(at(&["--bind", job_s, job_s]) < at(&["--ro-bind", blend_s, blend_s]));
+ at(&["--setenv", "HOME", "/home/op"]);
+ at(&["--chdir", job_s]);
+ assert_eq!(a.last().unwrap(), "--");
+ // Exactly one writable bind of the host, and it is the job directory.
+ assert_eq!(a.iter().filter(|x| *x == "--bind").count(), 1);
+ assert!(!a.iter().any(|x| x == "--share-net" || x == "--dev-bind-try" || x == "/home" || x == "/root" || x == "/etc"));
+ // An unprivileged worker gets no capabilities from bwrap; root must drop them.
+ assert!(!a.iter().any(|x| x == "--cap-drop"));
+ let root = Sandbox { as_root: true, home: None, ..sandbox(false, false) };
+ let r: Vec = bwrap_args(&root, Path::new("/var/lib/c0mpute/jobs/j1"), None).iter().map(|x| x.to_string_lossy().into_owned()).collect();
+ assert!(r.windows(2).any(|w| w == ["--cap-drop", "ALL"]));
+ assert!(r.windows(3).any(|w| w == ["--setenv", "HOME", "/tmp"]), "no home: HOME is the private /tmp");
+ }
+
+ #[test]
+ fn only_install_directories_that_do_not_hold_the_home_are_shown() {
+ let home = Some(Path::new("/home/op"));
+ assert!(shareable(Path::new("/opt/blender-4.2"), home));
+ assert!(shareable(Path::new("/home/op/blender-4.2"), home));
+ assert!(!shareable(Path::new("/home/op"), home), "the home itself");
+ assert!(!shareable(Path::new("/home"), home));
+ assert!(!shareable(Path::new("/opt"), home), "a top-level directory");
+ assert!(!shareable(Path::new("/"), None));
+ assert!(!shareable(Path::new("blender"), None));
+ let mounts = vec![Mount::RoBind("/usr".into()), Mount::DevBind("/dev/dri".into())];
+ assert!(covered(&mounts, Path::new("/usr/bin")));
+ assert!(!covered(&mounts, Path::new("/usrlocal")));
+ assert!(!covered(&mounts, Path::new("/dev/dri/card0")), "a device bind is not a read-only view");
+ }
+
+ #[test]
+ fn require_serves_only_what_the_sandbox_was_seen_to_do() {
+ let p = |extra: Value| parse_spec(&spec(extra)).unwrap();
+ let base = Runtime {
+ version_line: Some("Blender 4.2.3 LTS".into()),
+ version: (4, 2, 3),
+ gpu_backends: vec!["OPTIX".into()],
+ sandbox: None,
+ sandbox_mode: SandboxMode::Require,
+ };
+ // No working sandbox: no role, no tags, no bids.
+ assert!(!base.ready());
+ assert!(base.tags().is_empty());
+ assert!(!base.can_serve(&p(json!({"device":"CPU"}))));
+ // A sandbox that starts Blender but cannot reach the GPU: CPU Cycles only.
+ let cpu_only = Runtime { sandbox: Some(sandbox(false, false)), ..base.clone() };
+ assert!(cpu_only.ready());
+ assert!(cpu_only.can_serve(&p(json!({}))));
+ assert_eq!(cpu_only.cycles_device(&p(json!({}))), None, "AUTO renders on the CPU inside the sandbox");
+ assert!(!cpu_only.can_serve(&p(json!({"device":"GPU"}))));
+ assert!(!cpu_only.can_serve(&p(json!({"engine":"EEVEE"}))));
+ assert_eq!(cpu_only.tags(), vec![ROLE_TAG, "c0mpute:render:engine:cycles"]);
+ assert!(cpu_only.sandbox_for(&p(json!({}))).is_some());
+ // A sandbox that reaches the GPU serves everything, inside it.
+ let full = Runtime { sandbox: Some(sandbox(true, true)), ..base.clone() };
+ assert!(full.can_serve(&p(json!({"device":"GPU"}))) && full.can_serve(&p(json!({"engine":"EEVEE"}))));
+ assert_eq!(full.cycles_device(&p(json!({}))).as_deref(), Some("OPTIX"));
+ assert!(full.sandbox_for(&p(json!({"engine":"EEVEE"}))).is_some());
+ assert!(full.tags().contains(&GPU_TAG.to_string()));
+
+ // "auto" keeps today's behaviour: every job runs, and a GPU job the
+ // sandbox cannot serve runs outside it.
+ let auto = Runtime { sandbox_mode: SandboxMode::Auto, ..cpu_only.clone() };
+ assert!(auto.can_serve(&p(json!({"device":"GPU"}))) && auto.can_serve(&p(json!({"engine":"EEVEE"}))));
+ assert_eq!(auto.cycles_device(&p(json!({}))).as_deref(), Some("OPTIX"));
+ assert!(auto.sandbox_for(&p(json!({}))).is_none(), "GPU Cycles, sandbox without the GPU");
+ assert!(auto.sandbox_for(&p(json!({"engine":"EEVEE"}))).is_none());
+ assert!(auto.sandbox_for(&p(json!({"device":"CPU"}))).is_some());
+ let none = Runtime { sandbox: None, ..auto.clone() };
+ assert!(none.ready() && none.sandbox_for(&p(json!({"device":"CPU"}))).is_none());
+ }
+
+ #[test]
+ fn no_buyer_text_reaches_the_python_blender_runs() {
+ // Everything a buyer chooses, at its most hostile: the file's path
+ // (its hash names it in practice) and every spec field.
+ let p = parse_spec(&spec(json!({
+ "frame_start": MAX_FRAME - 1, "frame_end": MAX_FRAME, "resolution_pct": MAX_RESOLUTION_PCT,
+ "samples": MAX_SAMPLES, "format": "jpg", "engine": "cycles", "device": "gpu", "frame_timeout_secs": 10,
+ })))
+ .unwrap();
+ let blend = Path::new("/cache/'); import os; os.system('id') #.blend");
+ let out = Path::new("/jobs/\"]); __import__('os')/out");
+ let args: Vec = blender_args(blend, out, &p, (4, 2, 0), Some("CUDA"), 4)
+ .unwrap()
+ .iter()
+ .map(|a| a.to_string_lossy().into_owned())
+ .collect();
+ let exprs: Vec<&String> = args.iter().zip(args.iter().skip(1)).filter(|(flag, _)| *flag == "--python-expr").map(|(_, e)| e).collect();
+ assert_eq!(exprs.len(), 2, "the settings expression and the sanitiser");
+ // The sanitiser is the constant, byte for byte, whatever the job is.
+ assert_eq!(exprs[1], SANITIZE_EXPR);
+ let other = parse_spec(&spec(json!({"engine": "EEVEE", "samples": 1}))).unwrap();
+ let other_args = blender_args(Path::new("/x.blend"), Path::new("/o"), &other, (3, 4, 1), None, 1).unwrap();
+ assert!(other_args.iter().any(|a| a.to_str() == Some(SANITIZE_EXPR)));
+ // The settings expression holds the validated numbers and fixed words only.
+ for needle in ["os.system", "__import__", "/cache", "/jobs", ".blend", "CUDA"] {
+ assert!(!exprs[0].contains(needle), "{needle} in {}", exprs[0]);
+ }
+ assert!(exprs[0].contains(&format!("s.cycles.samples = {MAX_SAMPLES}")));
+ // Both run before the render and after the file is loaded.
+ let at = |needle: &str| args.iter().position(|a| a == needle).unwrap();
+ assert!(at(SANITIZE_EXPR) > at(blend.to_str().unwrap()) && at(SANITIZE_EXPR) < at("--render-anim"));
+ // The directories reach the script through the environment, and a
+ // script that cannot finish stops Blender instead of rendering.
+ assert!(SANITIZE_EXPR.contains(&format!("os.environ.get('{JOB_DIR_ENV}')")));
+ assert!(SANITIZE_EXPR.contains(&format!("os.environ.get('{OUT_DIR_ENV}')")));
+ assert!(SANITIZE_EXPR.contains(&format!("os._exit({SANITIZE_EXIT})")));
+ // Python needs its indentation (a `\` line continuation in a Rust
+ // string would strip it), and the script has no format placeholders.
+ assert!(SANITIZE_EXPR.contains("\n D = bpy.data\n") && SANITIZE_EXPR.contains("\n try:\n"));
+ assert!(!SANITIZE_EXPR.contains("{}") && !SANITIZE_EXPR.contains("{0}"));
+ for covered in ["D.libraries", "D.images", "'movieclips'", "'sounds'", "'volumes'", "'cache_files'", "D.fonts", "OUTPUT_FILE", "sequence_editor", "shading_system", "blend_paths"] {
+ assert!(SANITIZE_EXPR.contains(covered), "{covered}");
+ }
+ }
+
+ #[tokio::test]
+ async fn the_blend_is_staged_into_the_job_directory() {
+ let dir = tempfile::tempdir().unwrap();
+ let cache = dir.path().join("render-inputs");
+ let job = dir.path().join("jobs").join("j1");
+ std::fs::create_dir_all(&cache).unwrap();
+ std::fs::create_dir_all(&job).unwrap();
+ std::fs::write(cache.join("abc.blend"), b"BLENDER").unwrap();
+ std::fs::write(cache.join("other-buyer.blend"), b"BLENDER2").unwrap();
+ let staged = stage_input(&cache.join("abc.blend"), &job).await.unwrap();
+ assert_eq!(staged, job.join("input.blend"));
+ assert_eq!(std::fs::read(&staged).unwrap(), b"BLENDER");
+ // Its directory holds this job's file only, not the other scenes.
+ assert_eq!(std::fs::read_dir(&job).unwrap().count(), 1);
+ // The cached copy outlives the job directory.
+ std::fs::remove_dir_all(&job).unwrap();
+ assert!(cache.join("abc.blend").exists());
+ // An input already in the job directory (an opened sealed .blend) stays where it is.
+ std::fs::create_dir_all(&job).unwrap();
+ std::fs::write(job.join("input.blend"), b"PLAIN").unwrap();
+ let same = stage_input(&job.join("input.blend"), &job).await.unwrap();
+ assert_eq!(same, job.join("input.blend"));
+ assert_eq!(std::fs::read(&same).unwrap(), b"PLAIN");
+ }
+
/// Runs only where `blender` is installed; elsewhere it says so and
/// passes. Builds a tiny scene with Blender itself and renders two frames.
#[tokio::test]
@@ -2000,6 +2806,58 @@ mod tests {
);
}
+ /// Runs only where `blender` is installed. A scene that names an image
+ /// outside the job directory by absolute path still renders, with that
+ /// reference cut before the render starts.
+ #[tokio::test]
+ async fn a_scene_naming_a_host_file_renders_without_it() {
+ let rt = tokio::task::spawn_blocking(runtime).await.unwrap();
+ if !rt.ready() {
+ eprintln!("skipping: blender is not installed");
+ return;
+ }
+ let dir = tempfile::tempdir().unwrap();
+ let outside = dir.path().join("outside");
+ std::fs::create_dir_all(&outside).unwrap();
+ let (blend, secret) = (dir.path().join("scene.blend"), outside.join("secret.png"));
+ // Saved once before the image is added: Blender turns absolute paths
+ // into relative ones on a file's first save, and this scene must
+ // keep naming the image by its absolute path.
+ let expr = format!(
+ "import bpy\ns = bpy.context.scene\ns.render.engine = 'CYCLES'\n\
+ s.render.resolution_x = 64\ns.render.resolution_y = 48\n\
+ s.render.resolution_percentage = 100\ns.cycles.samples = 2\n\
+ bpy.ops.wm.save_as_mainfile(filepath={blend:?})\n\
+ img = bpy.data.images.new('secret', 8, 8)\nimg.filepath_raw = {secret:?}\n\
+ img.file_format = 'PNG'\nimg.save()\nbpy.data.images.remove(img)\n\
+ ext = bpy.data.images.load({secret:?})\next.filepath = {secret:?}\n\
+ ext.use_fake_user = True\ns.render.filepath = {out:?}\n\
+ bpy.ops.wm.save_mainfile()\n",
+ secret = secret.to_string_lossy(),
+ out = outside.join("render_").to_string_lossy(),
+ blend = blend.to_string_lossy(),
+ );
+ let status = std::process::Command::new("blender")
+ .args(["--background", "--factory-startup", "--python-expr", &expr])
+ .stdout(Stdio::null())
+ .status()
+ .unwrap();
+ assert!(status.success() && blend.exists() && secret.exists(), "could not build the test scene");
+ let size = std::fs::metadata(&blend).unwrap().len();
+ let p = parse_spec(&spec(json!({
+ "input_size": size, "input_pieces": vec![hash(); piece_count(size)],
+ "frame_start": 1, "frame_end": 1, "samples": 2, "device": "CPU",
+ })))
+ .unwrap();
+ let job = dir.path().join("job");
+ let frames = render(&blend, &p, &job).await.unwrap();
+ assert_eq!(frames.len(), 1);
+ let log = std::fs::read_to_string(job.join("blender.log")).unwrap();
+ assert!(log.contains("scene sanitised: 1 outside references cut"), "{log}");
+ // Nothing was written beside the file the scene pointed its output at.
+ assert_eq!(std::fs::read_dir(&outside).unwrap().count(), 1);
+ }
+
/// Save the default scene (cube, camera, light) at 64x48 to `path`.
pub(crate) fn make_test_scene(path: &Path) {
let expr = format!(
diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs
index c79bec10..b419ad18 100644
--- a/node/crates/c0mpute-core/src/supervisor.rs
+++ b/node/crates/c0mpute-core/src/supervisor.rs
@@ -915,6 +915,11 @@ impl Supervisor {
dir: Some(self.config.storage.root.join("apps")),
hosting,
registry: Some(self.registry.clone()),
+ // From the config this node was started with, not the
+ // default path (a node run with --config elsewhere).
+ egress_guard: Some(cfg.egress_guard),
+ require_egress_guard: Some(cfg.require_egress_guard),
+ max_disk_mb: Some(cfg.max_disk_mb),
..Default::default()
},
)
@@ -941,6 +946,7 @@ impl Supervisor {
crate::hosting::Options {
hosts,
max_bytes: cfg.max_bytes,
+ require_proofs: cfg.require_proofs,
catalog_path,
challenger,
challenge,
diff --git a/node/crates/c0mpute-core/src/text.rs b/node/crates/c0mpute-core/src/text.rs
index 7f2b103a..97b6c1b5 100644
--- a/node/crates/c0mpute-core/src/text.rs
+++ b/node/crates/c0mpute-core/src/text.rs
@@ -78,6 +78,159 @@ pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
/// the worker up (three missed beats plus gossip slack) and re-auctions.
pub const HEARTBEAT_SILENCE: Duration = Duration::from_secs(120);
+/// The base models a worker serves when its operator lists none: the ones
+/// this codebase's own clients name (`c0mpute train lora --base`'s example
+/// and the job file on the /train page). A base model is downloaded from
+/// Hugging Face and loaded by the trainer, so a worker runs only the ones
+/// its operator has agreed to.
+pub const DEFAULT_BASE_MODELS: [&str; 2] = ["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"];
+
+/// `[text]` in config.toml: what `c0mpute infernet text serve` accepts.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(default)]
+pub struct TextConfig {
+ /// Base models buyers may train on and chat with here: exact Hugging
+ /// Face ids (`org/name`, case as on the Hub), `org/*` for every model of
+ /// one organisation, or `*` for any. Jobs naming another get no bid.
+ pub base_models: Vec,
+ /// Free space kept on the disk the text cache is on, GiB; a job that
+ /// starts below it fails at once (0 = min(20 GiB, 10% of the disk), the
+ /// floor container jobs keep).
+ pub min_free_gb: u64,
+}
+
+impl Default for TextConfig {
+ fn default() -> Self {
+ Self { base_models: DEFAULT_BASE_MODELS.iter().map(|m| m.to_string()).collect(), min_free_gb: 0 }
+ }
+}
+
+impl TextConfig {
+ /// The node's `[text]` section. `text serve` is started without a config
+ /// handle, so this reads the file itself, as the speech gateway does.
+ pub fn load() -> Self {
+ crate::config::Config::load_raw(&crate::config::default_config_path()).unwrap_or_default().text
+ }
+
+ /// May a buyer's job use `base_model` on this node?
+ pub fn admits_model(&self, base_model: &str) -> bool {
+ model_allowed(&self.base_models, base_model)
+ }
+
+ /// Fail unless the disk holding `dir` has its free-space floor left.
+ pub fn check_free(&self, dir: &Path) -> Result<()> {
+ let (free, total) = c0mpute_store::free_space_at(dir);
+ let floor = crate::container_run::min_free_bytes(
+ &crate::container_run::ContainersConfig { min_free_gb: self.min_free_gb, ..Default::default() },
+ total,
+ );
+ if free < floor {
+ bail!("E_DISK: {} has {} GiB free; this node keeps {} GiB", dir.display(), free >> 30, floor >> 30);
+ }
+ Ok(())
+ }
+}
+
+/// A Hugging Face repo id, strictly: `org/name`, each part 1 to 96 of
+/// `A-Z a-z 0-9 . _ -`, not starting or ending with `.` or `-`, no `..` or
+/// `--`. Anything else could be read by the trainer as a path on the worker
+/// (`models/x`, `./x`, a bare directory name) or as a URL.
+pub fn valid_repo_id(id: &str) -> bool {
+ let part = |p: &str| {
+ !p.is_empty()
+ && p.len() <= 96
+ && p.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))
+ && !p.starts_with(['.', '-'])
+ && !p.ends_with(['.', '-'])
+ && !p.contains("..")
+ && !p.contains("--")
+ };
+ matches!(id.split_once('/'), Some((org, name)) if part(org) && part(name))
+}
+
+/// Is `id` a valid repo id that `allow` names: exactly, by `org/*`, or by
+/// `*`? Case matters: the Hub answers a differently-cased id with the same
+/// model, and each spelling would be a second copy in the worker's cache.
+pub fn model_allowed(allow: &[String], id: &str) -> bool {
+ let Some((org, _)) = id.split_once('/').filter(|_| valid_repo_id(id)) else {
+ return false;
+ };
+ allow.iter().map(|e| e.trim()).any(|entry| entry == "*" || entry == id || entry.strip_suffix("/*") == Some(org))
+}
+
+/// Top-level keys a job config may carry: what `c0mpute train lora`
+/// ([`crate::train::LoraOptions::job`]) and a hand-written job file for
+/// `c0mpute infernet text train` send, plus `limits.timeout_s`, which the
+/// worker reads. The trainer is handed the job as its arguments, so a key it
+/// does not know is refused here, not passed on.
+pub const JOB_KEYS: [&str; 8] = ["name", "kind", "base_model", "system_prompt", "hyperparams", "limits", "dataset", "output"];
+/// Hyperparameters a job may set, with the ranges `c0mpute train lora` uses.
+pub const HYPERPARAM_KEYS: [&str; 7] = ["lr", "batch_size", "epochs", "seq_len", "lora_r", "max_steps", "seed"];
+const MAX_SYSTEM_PROMPT_BYTES: usize = 8 * 1024;
+
+fn whole_in(v: &Value, range: std::ops::RangeInclusive) -> bool {
+ v.as_u64().is_some_and(|n| range.contains(&n))
+}
+
+/// Check a job config before it reaches the trainer: only [`JOB_KEYS`], only
+/// [`HYPERPARAM_KEYS`], every value of the right type and in range.
+pub fn validate_job(job: &Value) -> Result<()> {
+ let obj = job.as_object().context("job must be an object")?;
+ if let Some(k) = obj.keys().find(|k| !JOB_KEYS.contains(&k.as_str())) {
+ bail!("job.{k} is not accepted; a text training job may set {JOB_KEYS:?}");
+ }
+ let kind = obj.get("kind").and_then(Value::as_str).unwrap_or("");
+ if !KINDS.contains(&kind) {
+ bail!("job.kind must be one of {KINDS:?}, got {kind:?}");
+ }
+ // A Hugging Face repo id (org/name): never a path or a URL on the worker.
+ if !valid_repo_id(obj.get("base_model").and_then(Value::as_str).unwrap_or("")) {
+ bail!("job.base_model must be a model repo id like org/name");
+ }
+ match obj.get("name") {
+ None => {}
+ Some(Value::String(n)) if !n.is_empty() && n.len() <= 64 && n.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) => {}
+ Some(_) => bail!("job.name must be 1-64 characters of letters, digits, '-', '_' or '.'"),
+ }
+ match obj.get("system_prompt") {
+ None | Some(Value::Null) => {}
+ Some(Value::String(p)) if p.len() <= MAX_SYSTEM_PROMPT_BYTES => {}
+ Some(_) => bail!("job.system_prompt must be a string of at most {MAX_SYSTEM_PROMPT_BYTES} bytes"),
+ }
+ // The worker replaces both with its own local paths; they only have to
+ // be what the clients send, an object.
+ for k in ["dataset", "output"] {
+ if obj.get(k).is_some_and(|v| !v.is_object()) {
+ bail!("job.{k} must be an object");
+ }
+ }
+ if let Some(limits) = obj.get("limits") {
+ let l = limits.as_object().context("job.limits must be an object")?;
+ if l.keys().any(|k| k != "timeout_s") || l.get("timeout_s").is_some_and(|t| !t.is_u64()) {
+ bail!("job.limits may set only timeout_s, a whole number of seconds");
+ }
+ }
+ if let Some(hp) = obj.get("hyperparams") {
+ let hp = hp.as_object().context("job.hyperparams must be an object")?;
+ for (k, v) in hp {
+ let ok = match k.as_str() {
+ "lr" => v.as_f64().is_some_and(|x| x.is_finite() && x > 0.0 && x <= 1e-2),
+ "batch_size" => whole_in(v, 1..=64),
+ "epochs" => whole_in(v, 1..=20),
+ "seq_len" => whole_in(v, 32..=4096),
+ "lora_r" => whole_in(v, 1..=128),
+ "max_steps" => whole_in(v, 1..=1_000_000),
+ "seed" => whole_in(v, 0..=u32::MAX as u64),
+ _ => bail!("job.hyperparams.{k} is not accepted; a text training job may set {HYPERPARAM_KEYS:?}"),
+ };
+ if !ok {
+ bail!("job.hyperparams.{k} is out of range: lr in (0, 0.01], batch_size 1-64, epochs 1-20, seq_len 32-4096, lora_r 1-128, max_steps 1-1000000, seed 0-4294967295, whole numbers except lr");
+ }
+ }
+ }
+ Ok(())
+}
+
/// Jobs a text worker is running now, by job id, with each one's run id, so a
/// shutdown can fail them out loud instead of leaving the buyer waiting.
pub type InFlight = Arc)>>>;
@@ -177,15 +330,7 @@ fn is_http(u: &str) -> bool {
impl TextTrainSpec {
pub fn validate(&self) -> Result<()> {
- let kind = self.job.get("kind").and_then(|v| v.as_str()).unwrap_or("");
- if !KINDS.contains(&kind) {
- bail!("job.kind must be one of {KINDS:?}, got {kind:?}");
- }
- let base = self.job.get("base_model").and_then(|v| v.as_str()).unwrap_or("");
- // A Hugging Face repo id (org/name): never a path or a URL on the worker.
- if base.is_empty() || base.len() > 200 || base.starts_with('/') || base.contains("..") || base.contains("://") {
- bail!("job.base_model must be a model repo id like org/name");
- }
+ validate_job(&self.job)?;
if !crate::temporal::is_sha256_ref(&self.dataset_hash) {
bail!("dataset_hash must be sha256:<64 lowercase hex>");
}
@@ -323,11 +468,20 @@ pub async fn fetch_corpus(net: &Arc, buyer: Option, spec:
/// Run one accepted job: fetch the corpus, run `infernet model train --job
/// --json`, relay every event, return the adapter dir and its weights hash.
/// `buyer` is the peer the corpus pieces are fetched from first.
-pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec, net: &Arc, buyer: Option) -> Result<(PathBuf, String)> {
+pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec, net: &Arc, buyer: Option, policy: &TextConfig) -> Result<(PathBuf, String)> {
spec.validate()?;
+ // The opened spec is judged again: at bid time a sealed job showed only
+ // its public fields.
+ let base = spec.job["base_model"].as_str().unwrap_or("");
+ if !policy.admits_model(base) {
+ bail!("base model {base} is not one this node runs ([text] base_models)");
+ }
tokio::fs::create_dir_all(job_dir).await?;
+ policy.check_free(job_dir)?;
let dataset = job_dir.join("dataset.jsonl");
fetch_corpus(net, buyer, spec, &dataset).await.context("fetch dataset")?;
+ // The corpus is on disk now; training writes checkpoints and the adapter.
+ policy.check_free(job_dir)?;
let mut job = spec.job.clone();
job["dataset"] = json!({"url": "dataset.jsonl", "sha256": spec.dataset_hash.trim_start_matches("sha256:")});
job["output"] = json!({"destination_url": "adapter"});
@@ -536,7 +690,7 @@ impl Drop for RemoveDirOnDrop {
}
}
-pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc, in_flight: InFlight) -> mpsc::Sender {
+pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc, in_flight: InFlight, policy: Arc) -> mpsc::Sender {
let (tx, mut rx) = mpsc::channel::(8);
tokio::spawn(async move {
while let Some(job) = rx.recv().await {
@@ -553,7 +707,7 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root
let result: Result = crate::job_control::cancellable(&accept.job_id, async {
let spec: TextTrainSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?;
let buyer = accept.buyer_peer_id.parse::().ok();
- let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer).await?;
+ let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer, &policy).await?;
if spec.adapter_pieces {
// Receipt -> manifest -> pieces, all over libp2p.
// Sealed to the buyer for e2e jobs.
@@ -582,8 +736,10 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root
tx
}
-/// Bid only on valid text-train offers, and only while idle.
-pub fn train_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter {
+/// Bid only on valid text-train offers for a base model this node runs, and
+/// only while idle. A job for another model gets no bid, so its buyer is not
+/// charged and a worker that does run the model can take it.
+pub fn train_offer_filter(busy: Arc, policy: Arc) -> crate::dispatch::OfferFilter {
Arc::new(move |offer| {
!busy.load(Ordering::Relaxed)
&& match crate::job_e2e::bid_spec(offer) {
@@ -592,11 +748,11 @@ pub fn train_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter
Some(crate::job_e2e::BidSpec::Public(v)) => {
let kind = v["job"]["kind"].as_str().unwrap_or("");
let base = v["job"]["base_model"].as_str().unwrap_or("");
- KINDS.contains(&kind) && valid_repo_id(base)
- }
- Some(crate::job_e2e::BidSpec::Full(v)) => {
- serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok())
+ KINDS.contains(&kind) && policy.admits_model(base)
}
+ Some(crate::job_e2e::BidSpec::Full(v)) => serde_json::from_value::(v.clone())
+ .ok()
+ .is_some_and(|s| s.validate().is_ok() && policy.admits_model(s.job["base_model"].as_str().unwrap_or(""))),
None => false,
}
})
@@ -652,10 +808,6 @@ pub struct TextInferSpec {
pub adapter_key: Option,
}
-fn valid_repo_id(base: &str) -> bool {
- !(base.is_empty() || base.len() > 200 || base.starts_with('/') || base.contains("..") || base.contains("://"))
-}
-
impl TextInferSpec {
pub fn validate(&self) -> Result<()> {
if !crate::temporal::is_sha256_ref(&self.adapter_hash) {
@@ -674,6 +826,10 @@ impl TextInferSpec {
bail!("message role must be system, user or assistant");
}
bytes += m["content"].as_str().context("message content must be a string")?.len();
+ // The messages go to the generator as they are: nothing rides along.
+ if m.as_object().is_none_or(|o| o.len() != 2) {
+ bail!("a message carries only role and content");
+ }
}
if bytes > MAX_PROMPT_BYTES {
bail!("messages exceed {MAX_PROMPT_BYTES} bytes");
@@ -746,34 +902,109 @@ async fn fetch_adapter(url: &str, want: &str, dir: &Path) -> Result<()> {
Ok(())
}
+/// The digest an adapter cache entry is filed under: every one of
+/// [`ADAPTER_FILES`], in that order, by name and sha256 (or as absent).
+/// Returns it with the weights' `sha256:` hash. Fails when the directory
+/// holds anything else, or no weights: such a directory is never an entry.
+pub fn adapter_cache_key(dir: &Path) -> Result<(String, String)> {
+ for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
+ let entry = entry?;
+ let name = entry.file_name();
+ if !entry.file_type()?.is_file() || !ADAPTER_FILES.iter().any(|f| name.to_str() == Some(*f)) {
+ bail!("{} holds {:?}, which is not an adapter file", dir.display(), name);
+ }
+ }
+ let mut key = Sha256::new();
+ let mut weights = None;
+ for f in ADAPTER_FILES {
+ let p = dir.join(f);
+ let hash = if p.exists() { crate::temporal::worker::file_sha256(&p)? } else { "-".to_string() };
+ key.update(f.as_bytes());
+ key.update([0u8]);
+ key.update(hash.as_bytes());
+ key.update([b'\n']);
+ if f == "adapter_model.safetensors" {
+ weights = Some(hash);
+ }
+ }
+ let weights = weights.filter(|w| w != "-").context("no adapter_model.safetensors")?;
+ Ok((hex::encode(key.finalize()), weights))
+}
+
+/// The cached adapter for `manifest` under `slot` (`//`), if its one entry still is what it was filed as: the directory's
+/// name is the [`adapter_cache_key`] of its contents, and its weights are
+/// the ones the request names.
+fn cached_adapter(slot: &Path, adapter_hash: &str) -> Option {
+ std::fs::read_dir(slot).ok()?.flatten().map(|e| e.path()).find(|dir| {
+ adapter_cache_key(dir).is_ok_and(|(key, weights)| dir.file_name().and_then(|n| n.to_str()) == Some(key.as_str()) && weights == adapter_hash)
+ })
+}
+
/// Answer one request: adapter from the cache (or fetched), then `infernet model generate`.
/// `buyer` is the peer adapter pieces are fetched from first.
-pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option) -> Result> {
+pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option, policy: &TextConfig) -> Result> {
spec.validate()?;
- // A sealed adapter (DIP-0032) is the buyer's private data: it lives only
- // in this job's directory, removed when the job ends, never in the cache.
+ if !policy.admits_model(&spec.base_model) {
+ bail!("base model {} is not one this node runs ([text] base_models)", spec.base_model);
+ }
+ tokio::fs::create_dir_all(cache_root.join("text")).await?;
+ policy.check_free(&cache_root.join("text"))?;
let input_key = spec.adapter_key.as_deref().map(crate::job_e2e::key_from_hex).transpose()?;
- let (adapter, _private_adapter) = match &input_key {
- Some(_) => {
- let d = crate::runner::job_subdir(&cache_root.join("text").join("infer-adapters"), job_id)?;
- (d.clone(), Some(RemoveDirOnDrop(d)))
+ let (adapter, _private_adapter) = match (&input_key, &spec.adapter_manifest) {
+ // An unsealed manifest names every file of the adapter by content
+ // hash, so the same manifest is the same adapter: the one case that
+ // is cached, in a directory filed under the digest of its files and
+ // checked against it on every hit. (Filing by the weights' hash
+ // alone let one buyer leave an adapter_config.json that the next
+ // buyer of the same weights then ran with.)
+ (None, Some(m)) => {
+ let slot = cache_root.join("text").join("adapter-cache").join(m);
+ let cached = cached_adapter(&slot, &spec.adapter_hash);
+ let dir = match cached {
+ Some(dir) => dir,
+ None => {
+ let _ = tokio::fs::remove_dir_all(&slot).await;
+ tokio::fs::create_dir_all(&slot).await?;
+ let tmp = slot.join(format!("fetch-{}", uuid::Uuid::new_v4()));
+ let filed: Result = async {
+ let got = fetch_adapter_pieces(net, buyer, m, &tmp, None).await.context("fetch adapter pieces")?;
+ if got != spec.adapter_hash {
+ bail!("adapter {got} differs from the offer's {}", spec.adapter_hash);
+ }
+ let dir = slot.join(adapter_cache_key(&tmp)?.0);
+ tokio::fs::rename(&tmp, &dir).await?;
+ Ok(dir)
+ }
+ .await;
+ if filed.is_err() {
+ let _ = tokio::fs::remove_dir_all(&slot).await;
+ }
+ filed?
+ }
+ };
+ (dir, None)
}
- None => (cache_root.join("text").join("adapters").join(spec.adapter_hash.trim_start_matches("sha256:")), None),
- };
- let cached = crate::temporal::worker::file_sha256(&adapter.join("adapter_model.safetensors")).ok();
- if cached.as_deref() != Some(spec.adapter_hash.as_str()) {
- match &spec.adapter_manifest {
- Some(m) => {
- let _ = tokio::fs::remove_dir_all(&adapter).await;
- let got = fetch_adapter_pieces(net, buyer, m, &adapter, input_key.as_ref()).await.context("fetch adapter pieces")?;
- if got != spec.adapter_hash {
- let _ = tokio::fs::remove_dir_all(&adapter).await;
- bail!("adapter {got} differs from the offer's {}", spec.adapter_hash);
+ // A sealed adapter (DIP-0032) is the buyer's private data, and an
+ // HTTP bundle names only its weights' hash, not the files beside
+ // them: both live only in this job's directory, removed when the
+ // job ends, never in the cache.
+ _ => {
+ let d = crate::runner::job_subdir(&cache_root.join("text").join("infer-adapters"), job_id)?;
+ let private = RemoveDirOnDrop(d.clone());
+ match &spec.adapter_manifest {
+ Some(m) => {
+ let _ = tokio::fs::remove_dir_all(&d).await;
+ let got = fetch_adapter_pieces(net, buyer, m, &d, input_key.as_ref()).await.context("fetch adapter pieces")?;
+ if got != spec.adapter_hash {
+ bail!("adapter {got} differs from the offer's {}", spec.adapter_hash);
+ }
}
+ None => fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &d).await.context("fetch adapter")?,
}
- None => fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &adapter).await.context("fetch adapter")?,
+ (d, Some(private))
}
- }
+ };
let dir = crate::runner::job_subdir(&cache_root.join("text").join("infer"), job_id)?;
tokio::fs::create_dir_all(&dir).await?;
// The request holds the prompt: removed on every exit, timeouts included.
@@ -796,7 +1027,7 @@ pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &
Ok(serde_json::to_vec(&ans)?)
}
-pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc) -> mpsc::Sender {
+pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, policy: Arc) -> mpsc::Sender {
let (tx, mut rx) = mpsc::channel::(16);
tokio::spawn(async move {
while let Some(job) = rx.recv().await {
@@ -805,7 +1036,7 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root
let result: Result = crate::job_control::cancellable(&accept.job_id, async {
let spec: TextInferSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?;
let buyer = accept.buyer_peer_id.parse::().ok();
- let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer).await?;
+ let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer, &policy).await?;
if spec.answer_chunk {
// Sealed to the buyer for e2e jobs (one chunk either way).
let answer = crate::job_e2e::seal_result(&accept, &answer)?.unwrap_or(answer);
@@ -834,17 +1065,18 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root
tx
}
-pub fn infer_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter {
+/// Bid only on valid requests for a base model this node runs.
+pub fn infer_offer_filter(busy: Arc, policy: Arc) -> crate::dispatch::OfferFilter {
Arc::new(move |offer| {
!busy.load(Ordering::Relaxed)
&& match crate::job_e2e::bid_spec(offer) {
Some(crate::job_e2e::BidSpec::Public(v)) => {
- valid_repo_id(v["base_model"].as_str().unwrap_or(""))
+ policy.admits_model(v["base_model"].as_str().unwrap_or(""))
&& v["max_new_tokens"].as_u64().is_some_and(|n| (1..=1024).contains(&n))
}
- Some(crate::job_e2e::BidSpec::Full(v)) => {
- serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok())
- }
+ Some(crate::job_e2e::BidSpec::Full(v)) => serde_json::from_value::(v.clone())
+ .ok()
+ .is_some_and(|s| s.validate().is_ok() && policy.admits_model(&s.base_model)),
None => false,
}
})
@@ -926,17 +1158,21 @@ pub async fn serve(net_opts: &NetOptions, infernet: PathBuf, cache_root: PathBuf
let net = spawn_node_with_store(net_opts, store.clone()).await?;
let hardware = json!({"version": env!("CARGO_PKG_VERSION"), "text_runtime": "peft", "ram_mb": ram_mb, "cuda": runtime["cuda"]});
tokio::spawn(crate::capabilities::advertise_loop(net.clone(), tags.clone(), hardware, crate::capabilities::local_bench_score(), crate::capabilities::DEFAULT_ADVERTISE_INTERVAL));
+ let policy = Arc::new(TextConfig::load());
+ // Adapters were once cached by their weights' hash alone; those entries
+ // are never read again (see `run_infer`), so they only take up disk.
+ let _ = tokio::fs::remove_dir_all(cache_root.join("text").join("adapters")).await;
let busy = Arc::new(AtomicBool::new(false));
let slots = Arc::new(Semaphore::new(1));
let in_flight = InFlight::default();
- let tx = spawn_train_runner(net.clone(), store.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots, in_flight.clone());
- crate::dispatch::run_worker_subscriber_filtered(net.clone(), TRAIN_WORKLOAD.into(), tags.clone(), Some(tx), Some(train_offer_filter(busy)));
+ let tx = spawn_train_runner(net.clone(), store.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots, in_flight.clone(), policy.clone());
+ crate::dispatch::run_worker_subscriber_filtered(net.clone(), TRAIN_WORKLOAD.into(), tags.clone(), Some(tx), Some(train_offer_filter(busy, policy.clone())));
if infer {
let ibusy = Arc::new(AtomicBool::new(false));
- let itx = spawn_infer_runner(net.clone(), store.clone(), cache_root, infernet, ibusy.clone());
- crate::dispatch::run_worker_subscriber_filtered(net.clone(), INFER_WORKLOAD.into(), tags.clone(), Some(itx), Some(infer_offer_filter(ibusy)));
+ let itx = spawn_infer_runner(net.clone(), store.clone(), cache_root, infernet, ibusy.clone(), policy.clone());
+ crate::dispatch::run_worker_subscriber_filtered(net.clone(), INFER_WORKLOAD.into(), tags.clone(), Some(itx), Some(infer_offer_filter(ibusy, policy.clone())));
}
- info!(peer = %net.peer_id().to_base58(), ?tags, "text: serving");
+ info!(peer = %net.peer_id().to_base58(), ?tags, base_models = ?policy.base_models, "text: serving");
tokio::time::sleep(Duration::from_millis(300)).await;
Ok(Serving { net, tags, runtime, infer, in_flight })
}
@@ -1880,6 +2116,285 @@ mod tests {
assert!(bad.validate().is_err());
}
+ fn allow(entries: &[&str]) -> Vec {
+ entries.iter().map(|e| e.to_string()).collect()
+ }
+
+ #[test]
+ fn base_model_allowlist_matches_exactly_or_by_org() {
+ let list = allow(&["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/*"]);
+ // Exact.
+ assert!(model_allowed(&list, "HuggingFaceTB/SmolLM2-135M-Instruct"));
+ assert!(!model_allowed(&list, "HuggingFaceTB/SmolLM2-360M-Instruct"));
+ assert!(!model_allowed(&list, "HuggingFaceTB/SmolLM2-135M-Instruct-evil"));
+ // A whole organisation, and only that organisation.
+ assert!(model_allowed(&list, "Qwen/Qwen2.5-0.5B-Instruct"));
+ assert!(model_allowed(&list, "Qwen/anything-else"));
+ assert!(!model_allowed(&list, "QwenX/Qwen2.5-0.5B-Instruct"));
+ assert!(!model_allowed(&list, "Qwe/n"));
+ assert!(!model_allowed(&list, "evil/Qwen"));
+ // Case matters: another spelling is another copy in the cache.
+ assert!(!model_allowed(&list, "huggingfacetb/smollm2-135m-instruct"));
+ assert!(!model_allowed(&list, "qwen/Qwen2.5-0.5B-Instruct"));
+ // Paths, traversal and URLs never match, whatever the list says.
+ let any = allow(&["*", "Qwen/*", "../*", "/*"]);
+ for bad in [
+ "", "gpt2", "/etc/passwd", "../x", "Qwen/../etc", "Qwen/..", "./Qwen/x", "Qwen/x/y", "Qwen//x", "/Qwen/x", "Qwen/x/",
+ "https://evil/model", "Qwen/x y", "Qwen/x\n", "Qwen\\x", ".Qwen/x", "Qwen/.x", "-Qwen/x", "Qwen/-x", "Qwen/x.", "Qwen/a--b", "~/model", "Qwen/*",
+ ] {
+ assert!(!model_allowed(&any, bad), "{bad:?}");
+ assert!(!valid_repo_id(bad), "{bad:?}");
+ }
+ assert!(!valid_repo_id(&format!("org/{}", "a".repeat(97))));
+ // "*" is the operator's way back to any model, still only by repo id.
+ assert!(model_allowed(&any, "someone/some-model_v1.5"));
+ assert!(!model_allowed(&[], "Qwen/Qwen2.5-0.5B-Instruct"), "an empty list runs nothing");
+ // Entries are trimmed; an org entry needs its `/*`.
+ assert!(model_allowed(&allow(&[" Qwen/* "]), "Qwen/x"));
+ assert!(!model_allowed(&allow(&["Qwen"]), "Qwen/x"));
+ }
+
+ #[test]
+ fn the_default_allowlist_is_what_the_clients_name_and_config_replaces_it() {
+ let cfg = TextConfig::default();
+ // `c0mpute train lora --base`'s example and the /train page's job file.
+ assert_eq!(cfg.base_models, vec!["HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"]);
+ assert!(cfg.base_models.iter().all(|m| cfg.admits_model(m) && valid_repo_id(m)));
+ assert!(!cfg.admits_model("meta-llama/Llama-3.1-8B-Instruct"));
+ // A config from before the section existed gets the defaults.
+ let old: crate::config::Config = toml::from_str("roles = []\n").unwrap();
+ assert_eq!(old.text, cfg);
+ let set: crate::config::Config = toml::from_str("[text]\nbase_models = [\"meta-llama/*\"]\nmin_free_gb = 50\n").unwrap();
+ assert!(set.text.admits_model("meta-llama/Llama-3.1-8B-Instruct"));
+ assert!(!set.text.admits_model("Qwen/Qwen2.5-0.5B-Instruct"), "the list replaces the default");
+ assert_eq!(set.text.min_free_gb, 50);
+ }
+
+ fn offer(workload: &str, spec: Value, sealed: bool) -> c0mpute_net::topics::JobOffer {
+ let mut o = json!({
+ "job_id": "j", "workload_type": workload, "buyer_peer_id": "buyer", "spec_hash": "",
+ "required_capabilities": [], "max_price_usd": 1.0, "deadline_unix_ms": 2_000_000, "published_at_ms": 1_000_000,
+ });
+ if sealed {
+ // What a buyer publishes for a sealed job: the public view only.
+ o["e2e"] = json!({"v": 1, "buyer_key": "00".repeat(32), "public_spec": crate::job_e2e::public_view(workload, &spec)});
+ } else {
+ o["spec_inline"] = spec;
+ }
+ serde_json::from_value(o).unwrap()
+ }
+
+ #[test]
+ fn a_job_for_a_model_off_the_list_gets_no_bid() {
+ let policy = Arc::new(TextConfig::default());
+ let idle = || Arc::new(AtomicBool::new(false));
+ let train = train_offer_filter(idle(), policy.clone());
+ let infer = infer_offer_filter(idle(), policy.clone());
+ let with_base = |base: &str| {
+ let mut t = p2p_spec();
+ t.job["base_model"] = json!(base);
+ let i = TextInferSpec { adapter_manifest: Some(c0mpute_proto::Hash::of(b"m").to_hex()), answer_chunk: true, base_model: base.into(), ..infer_spec() };
+ (serde_json::to_value(t).unwrap(), serde_json::to_value(i).unwrap())
+ };
+ for sealed in [false, true] {
+ let (t, i) = with_base("Qwen/Qwen2.5-0.5B-Instruct");
+ assert!(train(&offer(TRAIN_WORKLOAD, t, sealed)), "sealed={sealed}");
+ assert!(infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}");
+ // A valid repo id, not one this operator runs.
+ let (t, i) = with_base("meta-llama/Llama-3.1-8B-Instruct");
+ assert!(!train(&offer(TRAIN_WORKLOAD, t, sealed)), "sealed={sealed}");
+ assert!(!infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}");
+ // Shaped like a repo id, but it could as well be a directory
+ // under the trainer's working directory: the list decides.
+ let (t, i) = with_base("models/local-dir");
+ assert!(!train(&offer(TRAIN_WORKLOAD, t, sealed)) && !infer(&offer(INFER_WORKLOAD, i, sealed)), "sealed={sealed}");
+ }
+ // An operator who lists the org bids on it.
+ let wide = Arc::new(TextConfig { base_models: allow(&["meta-llama/*"]), ..TextConfig::default() });
+ let (t, _) = with_base("meta-llama/Llama-3.1-8B-Instruct");
+ assert!(train_offer_filter(idle(), wide)(&offer(TRAIN_WORKLOAD, t.clone(), true)));
+ // Busy workers still do not bid.
+ assert!(!train_offer_filter(Arc::new(AtomicBool::new(true)), Arc::new(TextConfig { base_models: allow(&["*"]), ..TextConfig::default() }))(&offer(TRAIN_WORKLOAD, t, false)));
+ }
+
+ fn train_request(replicas: u32) -> TrainRequest {
+ TrainRequest {
+ job_path: "job.json".into(),
+ dataset_path: None,
+ output: "adapter".into(),
+ workers: vec![],
+ max_price_usd: 1.0,
+ gpu: false,
+ min_ram_gb: None,
+ legacy_http: false,
+ public_url: None,
+ bind: "127.0.0.1:0".parse().unwrap(),
+ timeout_s: 60,
+ retries: 0,
+ bid_window: Duration::from_secs(1),
+ replicas,
+ replica_tolerance: crate::train_verify::DEFAULT_TOLERANCE,
+ penalty_file: None,
+ }
+ }
+
+ #[test]
+ fn job_validation_accepts_every_shape_the_clients_send() {
+ use crate::train::{DatasetReport, LoraOptions};
+ let report = DatasetReport { system_prompt: Some("You are a pricing coach.".into()), ..Default::default() };
+ // `c0mpute train lora` with its defaults, and with every flag set to
+ // each end of the range the CLI itself accepts.
+ let defaults = LoraOptions { base_model: "Qwen/Qwen2.5-0.5B-Instruct".into(), ..Default::default() };
+ let low = LoraOptions { name: "a".into(), epochs: 1, rank: 1, lr: 1e-9, seq_len: 32, batch_size: 1, max_steps: Some(1), system_prompt: None, ..defaults.clone() };
+ let high = LoraOptions {
+ name: "A-b_c.9".repeat(9),
+ epochs: 20,
+ rank: 128,
+ lr: 1e-2,
+ seq_len: 4096,
+ batch_size: 64,
+ max_steps: Some(1_000_000),
+ system_prompt: Some("s".repeat(8 * 1024)),
+ ..defaults.clone()
+ };
+ for opts in [&defaults, &low, &high] {
+ opts.validate().unwrap();
+ validate_job(&opts.job(&report)).unwrap_or_else(|e| panic!("{opts:?}: {e}"));
+ validate_job(&opts.job(&DatasetReport::default())).unwrap();
+ }
+ // With --replicas the buyer pins a seed in the job (and takes
+ // `replicas` out of it before it is sent).
+ let mut job = defaults.job(&report);
+ job["replicas"] = json!(3);
+ assert_eq!(replica_count(&train_request(0), &mut job).unwrap(), 3);
+ assert_eq!(job["hyperparams"]["seed"], 0);
+ validate_job(&job).unwrap();
+ // The job file on the /train page (apps/web/src/app/train/page.tsx),
+ // which has no dataset, output or name the CLI would add.
+ let page: Value = serde_json::from_str(
+ r#"{ "name": "my-agent", "kind": "lora",
+ "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct",
+ "system_prompt": "You are a concise, practical business strategist.",
+ "hyperparams": { "max_steps": 200, "batch_size": 2, "seq_len": 256 } }"#,
+ )
+ .unwrap();
+ validate_job(&page).unwrap();
+ let mut page_replicas = page.clone();
+ assert_eq!(replica_count(&train_request(3), &mut page_replicas).unwrap(), 3);
+ validate_job(&page_replicas).unwrap();
+ // The smallest job, the timeout the worker reads, and `peft`.
+ validate_job(&json!({"kind": "peft", "base_model": "a/b"})).unwrap();
+ validate_job(&json!({"kind": "lora", "base_model": "a/b", "limits": {"timeout_s": 600}, "system_prompt": null})).unwrap();
+ // And the whole spec the buyer builds around it.
+ TextTrainSpec { job: high.job(&report), ..p2p_spec() }.validate().unwrap();
+ }
+
+ #[test]
+ fn job_validation_refuses_what_the_trainer_was_not_asked_for() {
+ let job = |patch: Value| {
+ let mut j = json!({"kind": "lora", "base_model": "Qwen/Qwen2.5-0.5B-Instruct", "hyperparams": {"lr": 2e-4, "epochs": 1}});
+ for (k, v) in patch.as_object().unwrap() {
+ j[k] = v.clone();
+ }
+ j
+ };
+ let err = |patch: Value| validate_job(&job(patch)).unwrap_err().to_string();
+ // Unknown top-level keys are named, with the keys that are accepted.
+ let e = err(json!({"trust_remote_code": true}));
+ assert!(e.contains("job.trust_remote_code") && e.contains("hyperparams") && e.contains("system_prompt"), "{e}");
+ assert!(err(json!({"replicas": 3})).contains("job.replicas"));
+ // Unknown hyperparameters would become trainer arguments.
+ let e = err(json!({"hyperparams": {"output_dir": "/home/op/.ssh"}}));
+ assert!(e.contains("hyperparams.output_dir") && e.contains("lora_r"), "{e}");
+ assert!(err(json!({"hyperparams": {"trust_remote_code": true}})).contains("not accepted"));
+ // Types and ranges.
+ for bad in [
+ json!({"lr": "2e-4"}), json!({"lr": 0}), json!({"lr": -1e-4}), json!({"lr": 0.5}), json!({"lr": [2e-4]}),
+ json!({"epochs": 0}), json!({"epochs": 21}), json!({"epochs": 1.5}), json!({"epochs": -1}), json!({"epochs": "1"}),
+ json!({"batch_size": 65}), json!({"seq_len": 31}), json!({"seq_len": 1_000_000}), json!({"lora_r": 129}),
+ json!({"max_steps": 0}), json!({"max_steps": 1_000_001}), json!({"seed": -1}), json!({"seed": 4294967296u64}),
+ json!({"lora_r": {"nested": 1}}), json!({"epochs": null}),
+ ] {
+ assert!(err(json!({"hyperparams": bad.clone()})).contains("out of range"), "{bad}");
+ }
+ assert!(err(json!({"hyperparams": [1, 2]})).contains("must be an object"));
+ assert!(err(json!({"hyperparams": "lr=1"})).contains("must be an object"));
+ assert!(err(json!({"name": "a b"})).contains("job.name"));
+ assert!(err(json!({"name": 7})).contains("job.name"));
+ assert!(err(json!({"system_prompt": "s".repeat(8 * 1024 + 1)})).contains("system_prompt"));
+ assert!(err(json!({"system_prompt": ["a"]})).contains("system_prompt"));
+ assert!(err(json!({"limits": {"timeout_s": 60, "max_memory": 1}})).contains("limits"));
+ assert!(err(json!({"limits": {"timeout_s": "60"}})).contains("limits"));
+ assert!(err(json!({"dataset": "/etc/passwd"})).contains("job.dataset"));
+ assert!(validate_job(&json!(["lora"])).is_err());
+ // The spec check runs it: a worker never writes such a job to disk.
+ let mut s = p2p_spec();
+ s.job["hyperparams"] = json!({"output_dir": "/tmp/x"});
+ assert!(s.validate().is_err());
+ // A chat message carries a role and its text, nothing else.
+ let mut i = infer_spec();
+ i.messages = vec![json!({"role": "user", "content": "hi", "tool_calls": [{"name": "x"}]})];
+ assert!(i.validate().unwrap_err().to_string().contains("only role and content"));
+ }
+
+ #[test]
+ fn adapter_cache_entries_are_filed_under_every_file_and_checked_on_a_hit() {
+ let tmp = tempfile::tempdir().unwrap();
+ let write = |dir: &Path, config: &[u8]| {
+ std::fs::create_dir_all(dir).unwrap();
+ std::fs::write(dir.join("adapter_model.safetensors"), b"weights").unwrap();
+ std::fs::write(dir.join("adapter_config.json"), config).unwrap();
+ std::fs::write(dir.join("manifest.json"), br#"{"base_model":"org/m"}"#).unwrap();
+ };
+ let (a, b) = (tmp.path().join("a"), tmp.path().join("b"));
+ write(&a, b"{\"r\": 8}");
+ write(&b, b"{\"r\": 8, \"base_model_name_or_path\": \"/home/op/private\"}");
+ let (key_a, weights_a) = adapter_cache_key(&a).unwrap();
+ let (key_b, weights_b) = adapter_cache_key(&b).unwrap();
+ // Same weights, another config: the old cache key was the same.
+ assert_eq!(weights_a, sha256_hex(b"weights"));
+ assert_eq!(weights_a, weights_b);
+ assert_ne!(key_a, key_b, "the key covers adapter_config.json");
+ assert_eq!(adapter_cache_key(&a).unwrap().0, key_a, "and is stable");
+ // A missing file is not the same as an empty one.
+ let (c, d) = (tmp.path().join("c"), tmp.path().join("d"));
+ write(&c, b"");
+ write(&d, b"");
+ std::fs::remove_file(d.join("adapter_config.json")).unwrap();
+ assert_ne!(adapter_cache_key(&c).unwrap().0, adapter_cache_key(&d).unwrap().0);
+ // Anything that is not an adapter file, or no weights: not an entry.
+ std::fs::write(c.join("extra.py"), b"x").unwrap();
+ assert!(adapter_cache_key(&c).is_err());
+ std::fs::remove_file(d.join("adapter_model.safetensors")).unwrap();
+ assert!(adapter_cache_key(&d).is_err());
+
+ // A slot holds the entry under its key; a hit recomputes it.
+ let slot = tmp.path().join("slot");
+ let entry = slot.join(&key_a);
+ write(&entry, b"{\"r\": 8}");
+ assert_eq!(cached_adapter(&slot, &weights_a), Some(entry.clone()));
+ assert_eq!(cached_adapter(&slot, &sha256_hex(b"other weights")), None, "not the weights the request names");
+ assert_eq!(cached_adapter(&tmp.path().join("no-such-slot"), &weights_a), None);
+ // Changed after it was filed (a config swapped in beside the same
+ // weights): never used again.
+ std::fs::write(entry.join("adapter_config.json"), b"{\"r\": 8, \"base_model_name_or_path\": \"/home/op/private\"}").unwrap();
+ assert_eq!(cached_adapter(&slot, &weights_a), None);
+ write(&entry, b"{\"r\": 8}");
+ std::fs::write(entry.join("tokenizer.json"), b"{}").unwrap();
+ assert_eq!(cached_adapter(&slot, &weights_a), None, "a file dropped beside the adapter");
+ }
+
+ #[test]
+ fn a_job_starts_only_with_the_free_space_floor_left() {
+ let tmp = tempfile::tempdir().unwrap();
+ TextConfig { min_free_gb: 0, ..TextConfig::default() }.check_free(Path::new("relative/unknown")).unwrap();
+ // Where the disk is known, a floor above its size refuses the job.
+ if c0mpute_store::free_space_at(tmp.path()).0 != u64::MAX {
+ let err = TextConfig { min_free_gb: 1 << 30, ..TextConfig::default() }.check_free(tmp.path()).unwrap_err();
+ assert!(err.to_string().starts_with("E_DISK:"), "{err}");
+ }
+ }
+
#[test]
fn conversations_require_the_messages_format_tag() {
let text = b"{\"title\":\"a\",\"text\":\"b\"}\n{\"title\":\"c\",\"text\":\"the word \\\"messages\\\" in text\"}\n";
diff --git a/node/crates/c0mpute-core/tests/apps_e2e.rs b/node/crates/c0mpute-core/tests/apps_e2e.rs
index c21a9806..28fde546 100644
--- a/node/crates/c0mpute-core/tests/apps_e2e.rs
+++ b/node/crates/c0mpute-core/tests/apps_e2e.rs
@@ -171,6 +171,12 @@ async fn apps_place_three_copies_serve_snapshot_and_fail_over() {
hosts,
dir: Some(dir.join("apps")),
timing: fast(),
+ // Docker is a script here: no real network to guard (as root the
+ // guard would add iptables rules to the machine running the test),
+ // and nothing read from this machine's config file.
+ egress_guard: Some(false),
+ require_egress_guard: Some(false),
+ max_disk_mb: Some(c0mpute_core::apps::DEFAULT_MAX_DISK_MB),
..Default::default()
};
let d0 = tmp.path().join("n0");
diff --git a/node/crates/c0mpute-gateway/src/lib.rs b/node/crates/c0mpute-gateway/src/lib.rs
index 5315718b..774a2d3d 100644
--- a/node/crates/c0mpute-gateway/src/lib.rs
+++ b/node/crates/c0mpute-gateway/src/lib.rs
@@ -7,6 +7,7 @@
pub mod auth;
pub mod host_guard;
+pub mod site_host;
pub mod storage_api;
use std::net::SocketAddr;
@@ -59,6 +60,9 @@ pub async fn serve_with_storage(
pub async fn serve_router(app: Router, addr: SocketAddr) -> Result<()> {
info!(%addr, "gateway listening");
let listener = tokio::net::TcpListener::bind(addr).await?;
+ // On a per-site host nothing but that site is reachable, whatever
+ // routes were merged into `app` (a no-op until a suffix is configured).
+ let app = site_host::isolate(app);
// Peer addresses reach handlers as ConnectInfo (access log).
axum::serve(listener, app.into_make_service_with_connect_info::()).await?;
Ok(())
diff --git a/node/crates/c0mpute-gateway/src/site_host.rs b/node/crates/c0mpute-gateway/src/site_host.rs
new file mode 100644
index 00000000..8c598711
--- /dev/null
+++ b/node/crates/c0mpute-gateway/src/site_host.rs
@@ -0,0 +1,331 @@
+//! Per-site hostnames: `.` is one hosted site and nothing
+//! else.
+//!
+//! Every hosted site used to be reachable only at `/s//`,
+//! one web origin shared by all sites and by the gateway's own APIs (storage
+//! writes, git, swarm routes): any site's script could read another site's
+//! localStorage and call those APIs as the same origin. With a suffix
+//! configured (`[gateway] site_host_suffix`), each site gets an origin of its
+//! own, and on that origin the only thing served is the site.
+//!
+//! This module is the part every router on the gateway shares: telling a
+//! per-site host from any other ([`site_of_host`]), and making sure a request
+//! on one can reach nothing but site serving ([`isolate`]). The hosting
+//! service serves the site itself.
+
+use std::sync::RwLock;
+
+use axum::{
+ Router,
+ extract::Request,
+ http::{HeaderMap, Method, StatusCode, Uri, header},
+ middleware::{self, Next},
+ response::{IntoResponse, Response},
+};
+
+/// Marks a request that arrived on a per-site host. [`isolate`] adds it
+/// after rewriting the path to `/s//`; whatever serves sites
+/// reads it to know the visitor's URL has no `/s/