diff --git a/.github/workflows/ci-budget.yml b/.github/workflows/ci-budget.yml
new file mode 100644
index 0000000..d16cf6d
--- /dev/null
+++ b/.github/workflows/ci-budget.yml
@@ -0,0 +1,178 @@
+# Reusable ci-budget workflow — the per-plugin CI budget / e2e record.
+#
+# WHAT IT PRODUCES
+# 1. A job summary on every invocation (ephemeral, free, always readable from
+# the run page).
+# 2. A ci-budget.json artifact (machine-readable, for a future fleet roll-up).
+# 3. A single long-lived issue per repo whose BODY is the record, upserted in
+# place — durable and permalinked with ZERO commits and ZERO PRs.
+#
+# Deliberately NOT a committed docs/CI-BUDGET.md: that would produce a commit
+# per run whose only delta is a timestamp, which is the churn pattern that got
+# the nightly devkit-sync deleted.
+#
+# HOW TO INVOKE
+# * on a schedule (weekly) via the caller template — the cheap default; the
+# record is a few API calls, ~20s, no containers, no runner minutes worth
+# counting;
+# * or as a final job of plugin-e2e, if you want the record refreshed the
+# moment an e2e finishes (pass workflow-name and it will pick the run up).
+name: ci-budget
+
+on:
+ workflow_call:
+ inputs:
+ workflow-name:
+ description: "NAME (not filename) of the workflow to budget."
+ required: false
+ type: string
+ default: "plugin-e2e"
+ window-days:
+ description: "Rolling window for minutes + pass/fail counts."
+ required: false
+ type: number
+ default: 30
+ devkit-submodule-path:
+ description: "Path to the vendored devkit submodule."
+ required: false
+ type: string
+ default: "tests/_testkit"
+ publish-issue:
+ description: "Upsert the durable record issue (else summary + artifact only)."
+ required: false
+ type: boolean
+ default: true
+ fail-on-stale:
+ description: "Fail the job when no e2e run exists in the window."
+ required: false
+ type: boolean
+ default: false
+ secrets:
+ # OPTIONAL, and the workflow is fully useful without it. It unlocks the
+ # authoritative billed-minutes tier from the org billing usage endpoint,
+ # which the repo-scoped GITHUB_TOKEN structurally cannot read. Without it
+ # the record says so in plain words instead of guessing a number.
+ BILLING_READ_TOKEN:
+ description: "Org-scoped PAT (read:org) for /organizations/{org}/settings/billing/usage."
+ required: false
+
+jobs:
+ budget:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ actions: read # runs + jobs timing
+ issues: write # upsert the durable record issue
+ steps:
+ - name: Checkout consumer (no submodules)
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ submodules: false
+
+ - name: Init devkit submodule
+ env:
+ DEVKIT_PATH: ${{ inputs.devkit-submodule-path }}
+ # The devkit is public, so this needs no credentials (DEC-075).
+ run: git submodule update --init "$DEVKIT_PATH"
+
+ - name: Resolve plugin name
+ id: resolve
+ run: |
+ set -euo pipefail
+ y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; }
+ plugin_dir=""
+ [ -f .devkit.yml ] && plugin_dir=$(y plugin_dir .devkit.yml)
+ if [ -z "$plugin_dir" ]; then
+ plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)")
+ fi
+ name=$(y name "$plugin_dir/plugin.yaml")
+ [ -n "$name" ] || name="${GITHUB_REPOSITORY##*/}"
+ echo "plugin_name=$name" >> "$GITHUB_OUTPUT"
+ echo "resolved plugin: $name"
+
+ - name: Detect whether a billing token is available
+ id: creds
+ # The `secrets` context is NOT allowed inside an `if:` expression — using
+ # it there invalidates the entire workflow file and the run dies as a
+ # startup_failure with zero jobs. Probe it in a run step (secrets ARE
+ # available there via env) and branch on the step output. Only a yes/no
+ # is printed; the token itself never is.
+ env:
+ BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }}
+ run: |
+ if [ -n "${BILLING_TOKEN:-}" ]; then
+ echo "have_billing=true" >> "$GITHUB_OUTPUT"
+ echo "org billing token present — billed-minutes tier enabled"
+ else
+ echo "have_billing=false" >> "$GITHUB_OUTPUT"
+ echo "no org billing token — record will report billed minutes as unavailable"
+ fi
+
+ - name: Build the record
+ id: build
+ continue-on-error: true
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }}
+ DEVKIT_PATH: ${{ inputs.devkit-submodule-path }}
+ PLUGIN: ${{ steps.resolve.outputs.plugin_name }}
+ WF: ${{ inputs.workflow-name }}
+ DAYS: ${{ inputs.window-days }}
+ run: |
+ set -uo pipefail
+ python3 "$DEVKIT_PATH/e2e/scripts/ci-budget.py" \
+ --repo "$GITHUB_REPOSITORY" \
+ --plugin "$PLUGIN" \
+ --workflow "$WF" \
+ --days "$DAYS" \
+ --json-out ci-budget.json \
+ --md-out ci-budget.md
+ echo "rc=$?" >> "$GITHUB_OUTPUT"
+
+ - name: Write job summary
+ if: always()
+ run: |
+ if [ -f ci-budget.md ]; then
+ cat ci-budget.md >> "$GITHUB_STEP_SUMMARY"
+ else
+ echo "## CI budget — generation FAILED" >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Upload machine-readable record
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ci-budget-${{ steps.resolve.outputs.plugin_name }}
+ path: ci-budget.json
+ if-no-files-found: warn
+ retention-days: 90
+
+ - name: Upsert durable record issue
+ if: always() && inputs.publish-issue
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPO: ${{ github.repository }}
+ TITLE: "CI budget — ${{ steps.resolve.outputs.plugin_name }}"
+ BODY_FILE: ci-budget.md
+ LABEL: ci-budget
+ MODE: always
+ DEVKIT_PATH: ${{ inputs.devkit-submodule-path }}
+ run: |
+ {
+ echo ""
+ echo "---"
+ echo "Maintained automatically by the devkit \`ci-budget\` workflow —"
+ echo "the body is replaced in place on each run, so there is no commit and"
+ echo "no PR. Last refreshed by [run ${GITHUB_RUN_ID}](https://github.com/${REPO}/actions/runs/${GITHUB_RUN_ID})."
+ } >> ci-budget.md
+ bash "$DEVKIT_PATH/e2e/ci/upsert-record-issue.sh"
+
+ - name: Fail on stale e2e
+ # Off by default. Turn it on once the fleet's cadence is steady and you
+ # want "nobody has run e2e in 30 days" to be loud rather than a footnote.
+ if: always() && inputs.fail-on-stale && steps.build.outputs.rc != '0'
+ run: |
+ echo "::error::ci-budget reported a stale or missing e2e record"
+ exit 1
diff --git a/.github/workflows/upstream-compat.yml b/.github/workflows/upstream-compat.yml
new file mode 100644
index 0000000..457fe25
--- /dev/null
+++ b/.github/workflows/upstream-compat.yml
@@ -0,0 +1,415 @@
+# upstream-compat — continuous backwards-support check against LATEST UPSTREAM A0.
+#
+# THE GAP THIS CLOSES
+# Every consumer's .devkit.yml pins the private fork image, and none of the
+# four target plugins overrides `a0_image`. So the fleet's e2e today proves
+# "works on the fork" and says NOTHING about "still works on upstream". The
+# machinery already exists — plugin-e2e.yml takes an a0_image input and the
+# devkit's own sample-plugin-e2e.yml already boots
+# docker.io/agent0ai/agent-zero:latest. What was missing is the scheduled
+# fan-out and somewhere the result is seen. That is all this file adds.
+#
+# HOW THIS DIFFERS FROM THE NIGHTLY devkit-sync THAT WAS DELETED
+# devkit-sync burned 189 runs to produce 29 merged bumps and then silently
+# no-op'd for three weeks. Three concrete differences:
+#
+# 1. CHANGE-GATED, NOT CALENDAR-GATED. A ~20s probe job resolves the upstream
+# manifest digest and only fans out the expensive legs when the digest has
+# actually moved since the last full run (tracked via actions/cache — no
+# commits, no state file). Upstream :latest does not change nightly, so
+# this is the difference between ~1,700 runner-min/month and roughly
+# (upstream releases × 56 min). Detection latency is unchanged: the probe
+# still runs every night.
+# 2. IT CANNOT SILENTLY NO-OP. A weekly floor (max-skip-days) forces a full
+# run even when the digest is unchanged, so "nothing happened for three
+# weeks" is impossible. The probe also HARD-FAILS if it cannot resolve the
+# digest, rather than skipping quietly, and the report job asserts that the
+# matrix actually produced results — an empty matrix is a failure, not a
+# pass. Every outcome is stamped into the record issue with a timestamp, so
+# staleness is visible on the artifact itself.
+# 3. ZERO PR CHURN. It never opens a pull request and never commits. Its only
+# write is one long-lived issue, opened when a plugin breaks against
+# upstream and closed automatically when the fleet is green again. An open
+# issue means "act"; there is no steady-state noise to learn to ignore.
+name: upstream-compat
+
+on:
+ schedule:
+ # 03:20 UTC — off the top of the hour, where GitHub's scheduler queues are
+ # worst. The probe runs nightly; the expensive legs are digest-gated.
+ - cron: "20 3 * * *"
+ workflow_dispatch:
+ inputs:
+ force:
+ description: "Run the full matrix even if the upstream digest is unchanged"
+ type: boolean
+ default: false
+ plugins:
+ description: "Override the plugin list (space-separated repo names)"
+ type: string
+ default: ""
+
+permissions:
+ contents: read
+ actions: write
+ issues: write
+
+env:
+ UPSTREAM_IMAGE: docker.io/agent0ai/agent-zero:latest
+ # The fleet under continuous backwards-support check. Space-separated repo
+ # names within agent-zero-plugins. Adding a plugin is a one-word edit.
+ PLUGINS: >-
+ agent-zero-plugin-chat-comments
+ agent-zero-plugin-diff-visualizer
+ agent-zero-plugin-mermaid-diagrams
+ agent-zero-plugin-share-chat
+ # Force a full run at least this often even if upstream never moves, so the
+ # workflow can never quietly become a no-op.
+ MAX_SKIP_DAYS: 7
+
+jobs:
+ # ---------------------------------------------------------------- 1. probe
+ probe:
+ name: probe upstream digest
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ digest: ${{ steps.digest.outputs.digest }}
+ changed: ${{ steps.decide.outputs.changed }}
+ reason: ${{ steps.decide.outputs.reason }}
+ matrix: ${{ steps.matrix.outputs.matrix }}
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Resolve upstream manifest digest
+ id: digest
+ # Hard-fails if it cannot resolve. A probe that shrugs and exits 0 is how
+ # devkit-sync managed to no-op for three weeks unnoticed.
+ env:
+ IMAGE: ${{ env.UPSTREAM_IMAGE }}
+ run: bash e2e/ci/upstream-digest.sh "$IMAGE"
+
+ - name: Look up last-tested digest
+ id: cache
+ uses: actions/cache/restore@v4
+ with:
+ # Cache HIT on this exact digest ⇒ we already ran the full matrix
+ # against this image. No commits, no state file, no PR.
+ path: .upstream-compat-state
+ key: upstream-compat-${{ steps.digest.outputs.digest_short }}
+ restore-keys: upstream-compat-
+
+ - name: Decide whether to fan out
+ id: decide
+ env:
+ HIT: ${{ steps.cache.outputs.cache-hit }}
+ FORCE: ${{ inputs.force }}
+ DIGEST: ${{ steps.digest.outputs.digest }}
+ run: |
+ set -euo pipefail
+ last_ts=0
+ [ -f .upstream-compat-state ] && last_ts=$(sed -n 's/^ts=//p' .upstream-compat-state | head -1)
+ : "${last_ts:=0}"
+ now=$(date +%s)
+ age_days=$(( (now - last_ts) / 86400 ))
+
+ if [ "${FORCE}" = "true" ]; then
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ echo "reason=forced via workflow_dispatch" >> "$GITHUB_OUTPUT"
+ elif [ "${HIT}" != "true" ]; then
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ echo "reason=upstream digest moved to ${DIGEST}" >> "$GITHUB_OUTPUT"
+ elif [ "$last_ts" -eq 0 ] || [ "$age_days" -ge "${MAX_SKIP_DAYS}" ]; then
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ echo "reason=weekly floor reached (${age_days}d since last full run) — digest unchanged" >> "$GITHUB_OUTPUT"
+ else
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ echo "reason=upstream digest unchanged (${DIGEST}), last full run ${age_days}d ago — skipping the expensive matrix" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Build plugin matrix
+ id: matrix
+ env:
+ OVERRIDE: ${{ inputs.plugins }}
+ run: |
+ set -euo pipefail
+ list="${OVERRIDE:-$PLUGINS}"
+ json=$(printf '%s\n' $list | python3 -c 'import sys,json; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))')
+ n=$(printf '%s' "$json" | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))')
+ # An empty matrix would make every downstream job "succeed" while
+ # testing nothing. Refuse.
+ [ "$n" -gt 0 ] || { echo "::error::plugin matrix is empty"; exit 1; }
+ echo "matrix=$json" >> "$GITHUB_OUTPUT"
+ echo "matrix ($n): $json"
+
+ - name: Probe summary
+ run: |
+ {
+ echo "## upstream-compat probe"
+ echo ""
+ echo "| | |"
+ echo "|---|---|"
+ echo "| image | \`${UPSTREAM_IMAGE}\` |"
+ echo "| digest | \`${{ steps.digest.outputs.digest }}\` |"
+ echo "| fan out? | **${{ steps.decide.outputs.changed }}** |"
+ echo "| reason | ${{ steps.decide.outputs.reason }} |"
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ # ------------------------------------------------------------ 2. the legs
+ compat:
+ name: ${{ matrix.plugin }}
+ needs: probe
+ if: needs.probe.outputs.changed == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ strategy:
+ fail-fast: false # one broken plugin must not hide the state of the others
+ max-parallel: 2 # nested-A0 legs are disk-hungry; stagger them
+ matrix:
+ plugin: ${{ fromJSON(needs.probe.outputs.matrix) }}
+ permissions:
+ contents: read
+ steps:
+ - name: Checkout the plugin under test
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ # Public repos, so the default token suffices for a cross-repo read.
+ repository: ${{ github.repository_owner }}/${{ matrix.plugin }}
+ submodules: recursive
+ path: plugin
+
+ - name: Free disk space
+ run: |
+ df -h / | tail -1
+ sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup \
+ /usr/local/lib/android /opt/hostedtoolcache/CodeQL \
+ /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" 2>/dev/null || true
+ sudo docker image prune -af 2>/dev/null || true
+ echo "after reclaim:"; df -h / | tail -1
+
+ - name: Resolve plugin
+ id: resolve
+ working-directory: plugin
+ run: |
+ set -euo pipefail
+ y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; }
+ plugin_dir="" display_name=""
+ if [ -f .devkit.yml ]; then
+ plugin_dir=$(y plugin_dir .devkit.yml)
+ display_name=$(y display_name .devkit.yml)
+ fi
+ if [ -z "$plugin_dir" ]; then
+ plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)")
+ fi
+ [ -d "$plugin_dir" ] && [ -f "$plugin_dir/plugin.yaml" ] || {
+ echo "::error::could not resolve plugin_dir in ${{ matrix.plugin }}"; exit 1; }
+ plugin_name=$(y name "$plugin_dir/plugin.yaml")
+ [ -n "$display_name" ] || display_name=$(y title "$plugin_dir/plugin.yaml")
+ [ -n "$display_name" ] || display_name="$plugin_name"
+ pod_env=""
+ if [ -f .devkit.yml ]; then
+ pod_env=$(y e2e_pod_env .devkit.yml)
+ if [ -z "$pod_env" ]; then
+ pod_env=$(awk '
+ /^e2e_pod_env:[[:space:]]*$/ {inblk=1; next}
+ inblk && /^[[:space:]]+[A-Za-z_][A-Za-z0-9_]*:/ {
+ l=$0; sub(/^[[:space:]]+/,"",l)
+ k=l; sub(/:.*/,"",k)
+ v=l; sub(/^[^:]*:[[:space:]]*/,"",v); gsub(/^["'\'']|["'\'']$/,"",v)
+ printf "%s=%s ", k, v; next }
+ inblk && /^[^[:space:]]/ {inblk=0}
+ ' .devkit.yml)
+ fi
+ fi
+ echo "plugin_dir=$plugin_dir" >> "$GITHUB_OUTPUT"
+ echo "plugin_name=$plugin_name" >> "$GITHUB_OUTPUT"
+ echo "display_name=$display_name" >> "$GITHUB_OUTPUT"
+ echo "pod_env=$pod_env" >> "$GITHUB_OUTPUT"
+ echo "Resolved: dir=$plugin_dir name=$plugin_name a0=UPSTREAM"
+
+ - name: Build devkit devcontainer (rootless)
+ run: podman build -t plugin-devkit:ci -f devcontainer/Containerfile devcontainer
+
+ - name: Package + run lifecycle against UPSTREAM A0
+ env:
+ PLUGIN_DIR: ${{ steps.resolve.outputs.plugin_dir }}
+ PLUGIN_NAME: ${{ steps.resolve.outputs.plugin_name }}
+ DISPLAY_NAME: ${{ steps.resolve.outputs.display_name }}
+ A0_POD_ENV: ${{ steps.resolve.outputs.pod_env }}
+ # The whole point: pin the digest the probe resolved, not the moving
+ # tag, so every leg of one night tests the SAME image and a red result
+ # names an exact upstream artifact.
+ A0_IMAGE: docker.io/agent0ai/agent-zero@${{ needs.probe.outputs.digest }}
+ run: |
+ set -euo pipefail
+ mkdir -p dist "$GITHUB_WORKSPACE/artifacts"
+ chmod 777 "$GITHUB_WORKSPACE/artifacts"
+ ( cd "plugin/$PLUGIN_DIR" && zip -qr "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip" . \
+ -x '.git/*' 'tests/*' '.github/*' '.skills/*' 'dist/*' '*/node_modules/*' \
+ '.devkit.yml' '.gitmodules' '.gitignore' )
+ BEHAVIOUR_ARG=""
+ [ -f "plugin/tests/e2e/behaviour.mjs" ] && BEHAVIOUR_ARG="/plugin/tests/e2e/behaviour.mjs"
+ SPECS_JSON="[]"
+ if ls plugin/tests/e2e/specs/*.mjs >/dev/null 2>&1; then
+ SPECS_JSON=$(for f in plugin/tests/e2e/specs/*.mjs; do
+ n=$(basename "$f" .mjs); printf '{"name":"%s","path":"/plugin/%s"}\n' "$n" "${f#plugin/}"; done \
+ | paste -sd, - | sed 's/^/[/; s/$/]/')
+ fi
+ HARNESS=run-lifecycle.sh
+ if ls plugin/tests/e2e/features/*.feature >/dev/null 2>&1; then HARNESS=run-bdd.sh; fi
+ echo "harness: $HARNESS image: $A0_IMAGE"
+ podman run --rm \
+ --device /dev/fuse \
+ --security-opt label=disable \
+ --sysctl net.ipv4.ip_unprivileged_port_start=0 \
+ -v "$GITHUB_WORKSPACE":/workspace:ro \
+ -v "$GITHUB_WORKSPACE/plugin":/plugin:ro \
+ -v "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip":/tmp/plugin.zip:ro \
+ -v "$GITHUB_WORKSPACE/artifacts":/artifacts:rw \
+ -e A0_IMAGE="$A0_IMAGE" \
+ -e PLUGIN_ZIP=/tmp/plugin.zip \
+ -e PLUGIN_DISPLAY_NAME="$DISPLAY_NAME" \
+ -e PLUGIN_NAME="$PLUGIN_NAME" \
+ -e HOOK_DIR=/plugin/tests/e2e/hooks \
+ -e BEHAVIOUR_FILE="$BEHAVIOUR_ARG" \
+ -e BEHAVIOUR_SPECS="$SPECS_JSON" \
+ -e A0_POD_ENV="$A0_POD_ENV" \
+ -e ARTIFACT_DIR=/artifacts \
+ -e WORKSPACE=/plugin \
+ -e DEVKIT_PATH=. \
+ -e BDD_TRACE=retain-on-failure \
+ plugin-devkit:ci \
+ -lc 'bash /workspace/e2e/harness/'"$HARNESS"
+
+ - name: Upload failure evidence
+ # retain-on-failure, not always: a nightly that uploads a full trace set
+ # for four green plugins every week is storage the operator pays for and
+ # nobody opens. A red night gets the full trace.
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: upstream-compat-${{ matrix.plugin }}
+ path: artifacts/
+ if-no-files-found: warn
+ retention-days: 30
+
+ # ---------------------------------------------------------------- 3. report
+ report:
+ name: report
+ needs: [probe, compat]
+ if: always() && needs.probe.result == 'success'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ issues: write
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Assemble the record
+ id: rec
+ env:
+ CHANGED: ${{ needs.probe.outputs.changed }}
+ REASON: ${{ needs.probe.outputs.reason }}
+ DIGEST: ${{ needs.probe.outputs.digest }}
+ MATRIX: ${{ needs.probe.outputs.matrix }}
+ COMPAT: ${{ needs.compat.result }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ set -euo pipefail
+ now=$(date -u +'%Y-%m-%d %H:%M UTC')
+ {
+ echo "# Upstream compatibility — \`${UPSTREAM_IMAGE}\`"
+ echo ""
+ echo "| | |"
+ echo "|---|---|"
+ echo "| last checked | **${now}** |"
+ echo "| upstream digest | \`${DIGEST}\` |"
+ echo "| full matrix ran? | **${CHANGED}** — ${REASON} |"
+ echo "| plugins under check | $(printf '%s' "$MATRIX" | python3 -c 'import sys,json; print(", ".join("`%s`" % p for p in json.load(sys.stdin)))') |"
+ echo "| result | ${COMPAT:-skipped} |"
+ echo "| evidence | [${GITHUB_RUN_ID}](${RUN_URL}) |"
+ } > record.md
+
+ # STATUS drives whether the record issue is opened or closed.
+ # compat skipped (digest unchanged) → ok, nothing to report
+ # compat success → ok
+ # anything else → fail
+ case "${COMPAT}" in
+ ""|skipped|success) status=ok ;;
+ *) status=fail ;;
+ esac
+ if [ "$status" = "fail" ]; then
+ {
+ echo ""
+ echo "## ❌ At least one plugin fails against the latest upstream A0"
+ echo ""
+ echo "Open the [run](${RUN_URL}) and expand the red matrix leg. Per-leg"
+ echo "Playwright traces are attached as \`upstream-compat-\` artifacts."
+ echo ""
+ echo "This is a **backwards-support regression**: the plugin still passes on"
+ echo "the fork image its own \`plugin-e2e\` pins, so nothing in the plugin"
+ echo "repo went red. Upstream moved to \`${DIGEST}\` and something the plugin"
+ echo "depends on moved with it."
+ } >> record.md
+ fi
+ {
+ echo ""
+ echo "---"
+ echo "Maintained by the devkit \`upstream-compat\` workflow. The body is"
+ echo "replaced in place — no commits, no pull requests. This issue opens when a"
+ echo "plugin breaks against upstream and closes automatically when the fleet is"
+ echo "green, so an OPEN issue always means action is needed."
+ } >> record.md
+ echo "status=$status" >> "$GITHUB_OUTPUT"
+ cat record.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Guard against a silent no-op
+ # The devkit-sync failure mode, made impossible: if the probe said "fan
+ # out" then the matrix MUST have produced a result. A green report with
+ # nothing tested is a bug, not a pass.
+ env:
+ CHANGED: ${{ needs.probe.outputs.changed }}
+ COMPAT: ${{ needs.compat.result }}
+ run: |
+ if [ "$CHANGED" = "true" ] && { [ -z "$COMPAT" ] || [ "$COMPAT" = "skipped" ]; }; then
+ echo "::error::probe requested a full run but the compat matrix never executed"
+ exit 1
+ fi
+ echo "no-op guard OK (changed=$CHANGED compat=${COMPAT:-none})"
+
+ - name: Upsert the record issue
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPO: ${{ github.repository }}
+ TITLE: "Upstream compatibility — agent0ai/agent-zero:latest"
+ BODY_FILE: record.md
+ LABEL: upstream-compat
+ MODE: only-on-failure
+ STATUS: ${{ steps.rec.outputs.status }}
+ run: bash e2e/ci/upsert-record-issue.sh
+
+ - name: Record the tested digest
+ # Written only on a GREEN full run, so a red night is retried tomorrow
+ # instead of being cached away as "already checked".
+ if: needs.compat.result == 'success'
+ run: |
+ printf 'digest=%s\nts=%s\n' "${{ needs.probe.outputs.digest }}" "$(date +%s)" \
+ > .upstream-compat-state
+
+ - name: Save digest cache
+ if: needs.compat.result == 'success'
+ uses: actions/cache/save@v4
+ with:
+ path: .upstream-compat-state
+ key: upstream-compat-${{ needs.probe.outputs.digest }}
+
+ - name: Fail the run when a plugin is broken
+ # The nightly's own red X is the primary signal — Actions emails the
+ # workflow owner on a scheduled-run failure, which is the "signal someone
+ # sees" requirement. The issue is the durable second channel.
+ if: steps.rec.outputs.status == 'fail'
+ run: |
+ echo "::error::one or more plugins fail against the latest upstream A0"
+ exit 1
diff --git a/SPEC.md b/SPEC.md
index 8e225d8..25450b3 100644
--- a/SPEC.md
+++ b/SPEC.md
@@ -1255,8 +1255,30 @@ job disappeared when the devkit went public. The real cost is **integrity and bl
unreviewed merge changes CI in 24 repos with no canary and no rollback boundary. Fixed as change
management, not as a breach.
+**DEC-082 — Upstream A0 compatibility is tested, because upstream is the only consumer.**
+The reusable workflow defaults `a0_image` to the private fork
+(`ghcr.io/nuevanext/agent-zero:latest-nonroot`) and **none of the four plugins overrode it**, so every
+PR proved the plugin worked on the fork and nothing anywhere proved it worked on upstream — which is
+the only thing anyone actually runs. That is not a gap in coverage so much as coverage pointed at the
+wrong target.
+Two changes, deliberately separate. Each plugin sets `a0_image` to
+`docker.io/agent0ai/agent-zero:latest` in its own `.devkit.yml`, so the **PR gate** now tests what
+customers run — a break is caught before merge, by the author, with the diff in front of them. And a
+scheduled `upstream-compat` workflow re-runs the same suites when **upstream** moves, which no PR can
+catch because nothing in the repo changed.
+*Why this schedule is not the one that was just deleted (DEC-079).* That one opened a PR every night
+whether or not anything had changed, and failed silently behind an approval gate for three weeks. This
+one is gated on the upstream manifest **digest** actually moving — measured, upstream ships ~1.5×/week
+against a 42-minute fan-out, so digest-gating costs ~295 runner-min/month instead of ~1,275 for the
+same ≤24h detection. It opens no PRs, commits nothing, and reports failure as a single issue that
+auto-closes when green, so an open issue always means act. Three structural guards keep it honest: the
+probe hard-fails rather than shrugging, an empty matrix is an error rather than a vacuous pass, and the
+report job asserts the matrix ran if the probe asked for it.
+The runs pin the resolved **digest**, not `:latest`, so a failure is reproducible against the exact
+image that broke it rather than whatever `:latest` points at by the time someone investigates.
+
> **Numbering note:** DEC-070–072 were originally introduced as code comments only. They are now
-> written up above, so this section documents an unbroken 070–081. The lesson is worth keeping: a
+> written up above, so this section documents an unbroken 070–082. The lesson is worth keeping: a
> decision that exists only as a code comment is invisible to anyone reading the SPEC, and all three
> of these were load-bearing (a gate's timeout budget, the image the whole fleet pulls, and a wait
> that was flaking half the runs).
diff --git a/docs/upstream-compat-and-ci-budget.md b/docs/upstream-compat-and-ci-budget.md
new file mode 100644
index 0000000..15d9701
--- /dev/null
+++ b/docs/upstream-compat-and-ci-budget.md
@@ -0,0 +1,335 @@
+# CI budget + nightly upstream-compat — design
+
+Two capabilities for the devkit, built against
+`agent-zero-plugin-development-testkit` and its four consumers
+(`chat-comments`, `diff-visualizer`, `mermaid-diagrams`, `share-chat`).
+
+Everything below that states an API behaviour was **probed live on 2026-08-05**,
+not assumed. Where an endpoint does not give us what the operator asked for, that
+is said plainly rather than papered over.
+
+---
+
+## Part 0 — what the GitHub API actually gives us
+
+I probed four candidate sources before designing anything. Three of the four are
+either dead or useless here.
+
+| # | Endpoint | Result | Verdict |
+|---|---|---|---|
+| 1 | `GET /orgs/{org}/settings/billing/actions` | **HTTP 410** "This endpoint has been moved." (`gh` also reports it wants `admin:org`) | Dead. |
+| 2 | `GET /orgs/{org}/settings/billing/shared-storage` | **HTTP 410**, same | Dead. |
+| 3 | `GET /repos/{r}/actions/runs/{id}/timing` | Reachable with a plain repo `GITHUB_TOKEN`, but returns **`billable.UBUNTU.total_ms = 0`** on every run | **Useless here** — see below. |
+| 4 | `GET /organizations/{org}/settings/billing/usage?year=&month=` | **Works.** Real per-repo, per-day minutes | The only true billing source. |
+
+### 3 is a trap, and it is the obvious thing to reach for
+
+`/timing` looks exactly like the answer. It is not. Three real
+`chat-comments` runs returned:
+
+```json
+{"billable":{"UBUNTU":{"total_ms":0,"jobs":2,
+ "job_runs":[{"job_id":92153588155,"duration_ms":0},
+ {"job_id":92156121906,"duration_ms":0}]}},
+ "run_duration_ms":810000}
+```
+
+All four consumer repos are **public**, and public-repo Actions minutes are free,
+so GitHub reports **zero billable milliseconds**. A budget built on `/timing`
+would report `0 min` for every plugin forever and look like it worked. The design
+therefore *deliberately does not use it*, and the generated record says so on its
+face so nobody "fixes" it back later.
+
+`run_duration_ms` in the same payload is real, but it is whole-run wall clock
+including queue and idle gaps between jobs — not a cost figure.
+
+### 4 works, with one structural limitation that shapes the whole design
+
+```
+GET /organizations/agent-zero-plugins/settings/billing/usage?year=2026&month=8
+```
+
+returns one row per (day, repo, SKU) — 191 rows for August, covering **27 repos
+including all four consumers**:
+
+```json
+{"date":"2026-08-04T00:00:00Z","product":"actions","sku":"Actions Linux",
+ "quantity":78.0,"unitType":"Minutes","netAmount":0.0,
+ "repositoryName":"agent-zero-plugin-chat-comments"}
+```
+
+So **minutes are real and per-repo**. Two caveats worth stating:
+
+- **The unfiltered call lies by omission.** With no `year`/`month` it returns
+ only **8 aggregate rows** covering 5 repos. With `?year=&month=` it returns
+ **191**. Anyone probing this endpoint casually would conclude our plugins have
+ no CI usage. They do.
+- **`read:org` was sufficient** on the operator PAT — this one does *not* need
+ `admin:org`, unlike the two dead endpoints.
+
+**The limitation that drives the architecture:** a workflow's `GITHUB_TOKEN` is
+repo-scoped. There is no `permissions:` key that grants org billing — the
+capability does not exist in the workflow token model at all. So **tier 3 is
+unreachable from inside a plugin repo's own CI unless an org PAT is supplied as
+a secret.** I did not work around this, because the available workarounds all
+involve inventing a number.
+
+### What we use instead, and what it is honestly worth
+
+`GET /repos/{r}/actions/runs/{id}/jobs` gives per-job `started_at` /
+`completed_at` on any token. Summing `(completed − started)` across jobs is a
+**runner-wall-clock proxy** — always available, no secret. Since GitHub bills
+per job rounded up to the whole minute, it tracks the real figure closely but is
+not identical to it.
+
+**It is not the same measurement as tier 3, and the record never implies it is.**
+Measured live on `chat_comments` over the same 30 days:
+
+| | value | scope |
+|---|---|---|
+| billed (tier 3) | **454.0 min** | whole repo, *every* workflow, rounded up per job |
+| observed (tier 4) | **199.8 min** | `plugin-e2e` only |
+
+The 2.3× gap is not error — it is `devkit-sync` + `unit` + Dependabot + rounding.
+An early draft of the record showed these as adjacent bare numbers, which reads
+as "one of these is wrong". The shipped version labels each with its scope and
+carries a one-line note explaining the difference. **Use billed for the invoice,
+observed for what the e2e suite costs.**
+
+One more honesty note: `netAmount` was **not** always `$0.00` despite the repos
+being public (`chat_comments` 30-day net was **$1.60**, `share_chat` **$0.16**).
+The naive "public means free" assumption does not hold in this org's billing
+data. The record reports whatever the API returns and does not assert the repos
+are free.
+
+---
+
+## Part A — CI budget / e2e record per plugin
+
+### Where the record lives, and why
+
+The operator asked for a record "on every one of the plugin records", showing CI
+minutes, when e2e last ran, and its result. Options considered:
+
+| Option | Durable | Zero churn | Verdict |
+|---|---|---|---|
+| Committed `docs/CI-BUDGET.md` | yes | **no** — a commit (or PR) per run whose only delta is a timestamp | **rejected** |
+| Job summary only | **no** — dies with the run | yes | insufficient alone |
+| README badge | yes | no (same commit churn), and cannot show minutes | insufficient |
+| shields.io endpoint badge | yes | yes | needs external JSON hosting we don't have |
+| **Long-lived issue, body upserted in place** | **yes** | **yes** | **chosen** |
+
+**Chosen: all three tiers of output, with the issue as the durable one.**
+
+1. **Job summary** on every invocation — free, immediate, on the run page.
+2. **`ci-budget.json` artifact** — machine-readable, so a future fleet roll-up
+ has something to consume without re-querying.
+3. **One long-lived issue per repo**, titled `CI budget — `, label
+ `ci-budget`, whose **body is replaced in place**.
+
+Why the issue wins:
+
+- **Zero commits, zero PRs, zero branches.** This is the hard constraint after
+ the devkit-sync deletion. A committed markdown file fails it outright.
+- **Editing an issue body sends no notification**, so it can refresh as often as
+ we like without training anyone to filter it out.
+- **Durable and permalinked**, unlike a job summary that vanishes with the run's
+ retention.
+- It is already where maintainers look, and it is pinnable per repo.
+
+### Staleness is a failure, not a footnote
+
+`ci-budget.py` **exits 2** when there is no completed e2e run in the window, and
+the record renders `⚠️ never` rather than a blank. This is the same defect class
+as devkit-sync's silent three-week no-op: a record that quietly says nothing is
+worse than no record. `fail-on-stale` (default `false`) promotes it to a red
+job once the fleet's cadence is steady.
+
+### Cadence
+
+Weekly (`cron: 40 6 * * 1`), not per-PR. It is a rolling 30-day view — running
+it per-PR would spend runner time to change a timestamp. Also
+`workflow_dispatch`-able for an on-demand read. The job itself is a handful of
+API calls, ~20s, no containers.
+
+### Files
+
+- `scripts/ci-budget.py` → `e2e/scripts/ci-budget.py`
+- `scripts/upsert-record-issue.sh` → `e2e/ci/upsert-record-issue.sh`
+- `workflows/ci-budget.yml` → `.github/workflows/ci-budget.yml` (reusable)
+- `templates/ci-budget.caller.yml` → `templates/`, distributed by
+ `make link-workflows`
+
+`BILLING_READ_TOKEN` is an **optional** secret. Absent, the record is fully
+useful and states in words that billed minutes are unavailable and why. Present,
+tier 3 lights up. Both paths are live-verified below.
+
+---
+
+## Part B — nightly upstream-compat
+
+### The gap it closes (verified, not assumed)
+
+All four consumers' `.devkit.yml` files were read directly. **None sets
+`a0_image`**, so all four inherit the reusable workflow's default —
+`ghcr.io/nuevanext/agent-zero:latest-nonroot`, the private fork. The fleet's e2e
+today proves *"works on the fork"* and says **nothing** about *"still works on
+upstream"*.
+
+The machinery genuinely does mostly exist: `plugin-e2e.yml` accepts `a0_image`,
+and `sample-plugin-e2e.yml` already boots `docker.io/agent0ai/agent-zero:latest`.
+What is missing is the scheduled fan-out and a place the result is seen. That is
+all this adds.
+
+### How it differs from the deleted devkit-sync nightly
+
+devkit-sync burned 189 runs for 29 merged bumps, then silently no-op'd for three
+weeks. Three concrete structural differences:
+
+**1. Change-gated, not calendar-gated.**
+A ~20s `probe` job resolves the upstream **manifest digest** anonymously from
+Docker Hub and only fans out when the digest actually moved since the last green
+full run. State lives in `actions/cache` keyed by digest — no commits, no state
+file, no PR. Live-verified:
+
+```
+$ bash scripts/upstream-digest.sh docker.io/agent0ai/agent-zero:latest
+sha256:8c5eff81a46fc956bc247e01a345beadc09acbeecf31873324b843dacf937c4c
+```
+
+The probe still runs **every night**, so detection latency is unchanged (≤24h).
+Only the expensive part is gated.
+
+**2. It cannot silently no-op.** Four independent guards:
+- the probe **hard-fails** if it cannot resolve a digest (a shrug-and-exit-0
+ probe is exactly how devkit-sync died unnoticed);
+- an **empty matrix is a hard error**, not a vacuous pass;
+- a **`MAX_SKIP_DAYS: 7` floor** forces a full run even if upstream never moves;
+- the `report` job asserts that *if the probe requested a fan-out, the matrix
+ actually produced a result* — otherwise it fails.
+- Every outcome stamps a `last checked` timestamp into the record, so staleness
+ is visible on the artifact itself.
+
+**3. Zero PR churn.** It never opens a PR and never commits. Its only write is
+**one** long-lived issue in `MODE=only-on-failure`: opened when a plugin breaks
+against upstream, **closed automatically** when the fleet is green again. An open
+issue always means "act"; there is no steady-state noise to learn to ignore.
+
+### Signal someone sees
+
+Primary: the **scheduled workflow's own red X**, which GitHub emails to the
+workflow owner on a scheduled-run failure. Secondary and durable: the
+auto-opening record issue. Tertiary: per-leg Playwright traces uploaded
+`if: failure()` only — a green nightly uploading four full trace sets every week
+is storage the operator pays for and nobody opens.
+
+### Cost/benefit — the honest numbers
+
+Real per-plugin e2e durations, measured from the most recent green run of each:
+
+| plugin | e2e job |
+|---|---|
+| chat-comments | 12m 39s |
+| diff-visualizer | 13m 02s |
+| mermaid-diagrams | 11m 00s |
+| share-chat | 5m 49s |
+| **one full fan-out** | **≈ 42 min** |
+
+Upstream release cadence, from Docker Hub tag timestamps (`v2.8` 08-01, `v2.7`
+07-27, `v2.6` 07-23, `v2.5` 07-17, `v2.4` 07-10, `v2.3` 07-09, `v2.2` 07-02,
+`v2.1` 06-26, `v2.0` 06-24) — **≈ 9 releases in 6 weeks ≈ 1.5/week ≈ 6.5/month.**
+
+| strategy | runner-min / month |
+|---|---|
+| naive calendar nightly (30 × 42) | **≈ 1,275** |
+| digest-gated (≈7 full runs + 30 probes) | **≈ 295 + ~10** |
+
+**≈ 76% cheaper for identical detection latency.** Because upstream ships more
+often than weekly, the `MAX_SKIP_DAYS` floor will rarely fire — it is insurance
+against upstream going quiet, not a routine cost.
+
+**The honest caveat:** at ~1.5 releases/week the digest changes often enough that
+this is *not* nearly-free — it is roughly 295 min/month, ~23% of a naive nightly
+but still a real recurring spend. Whether that is worth it depends on one
+question the operator should answer, not me: **do these plugins need to work on
+upstream A0 at all, or only on the fork they are deployed against?** If only the
+fork, this workflow is ~295 min/month of information nobody acts on and should
+not be adopted. If upstream compatibility is a shipped promise, this is the
+cheapest way to hold it, and the current state — where *nothing* tests upstream —
+means the promise is untested today.
+
+A cheaper variant if the answer is "nice to have": change `MAX_SKIP_DAYS` to 14
+and add a `plugins` subset so only the two most upstream-coupled plugins run per
+digest. I did not default to that because a partial check that silently omits
+plugins is the ambiguity this whole design is trying to remove.
+
+### `secrets` in `if:` — avoided by construction
+
+Per the explicit warning: the `secrets` context is **not** permitted in an `if:`
+expression; using it invalidates the entire workflow file and the run dies as
+`startup_failure` with zero jobs. `ci-budget.yml` therefore probes
+`BILLING_READ_TOKEN` in a **`run:` step** (where secrets *are* available via
+`env:`) and branches on `steps.creds.outputs.have_billing`. This mirrors the
+existing pattern already proven in `plugin-e2e.yml`'s "Detect whether App
+credentials are available" step. `upstream-compat.yml` uses no secrets at all —
+the upstream image is public and the digest probe is anonymous.
+
+---
+
+## Validation performed
+
+```
+YAML (python3 -c "import yaml; yaml.safe_load(open(...))")
+ OK workflows/ci-budget.yml
+ OK workflows/upstream-compat.yml
+ OK templates/ci-budget.caller.yml
+
+bash -n
+ OK scripts/upsert-record-issue.sh
+ OK scripts/upstream-digest.sh
+
+shellcheck CLEAN (both scripts, zero findings)
+python3 -m py_compile OK scripts/ci-budget.py
+```
+
+**Live end-to-end runs against the real API** (not mocks):
+
+- `upstream-digest.sh` → resolved the real Docker Hub digest anonymously.
+- `ci-budget.py` **without** a billing token (simulating `GITHUB_TOKEN`) →
+ rendered a complete record, billed minutes marked *not available* with the
+ reason. rc=0.
+- `ci-budget.py` **with** the org token → tier 3 lit up: `454.0 min`,
+ `net $1.60`, `20.768 GB-hours`. rc=0.
+- `ci-budget.py` against `share-chat` → `105.0 min` / `54.3 min` observed. rc=0.
+- **Stale path**: asked for a workflow that has never run → rendered
+ `⚠️ never — no completed upstream-compat run in the last 30 days` and
+ **exited 2**, as designed.
+
+### Not validated
+
+The two workflows have **not been executed on GitHub Actions** — that needs a
+commit to a repo, which this task explicitly forbids. YAML validity, script
+correctness and every API call they make are verified; job wiring
+(`needs`/`outputs`/matrix `fromJSON`) is reviewed but not run. The nested-A0 leg
+in `upstream-compat.yml` is adapted from the proven `plugin-e2e.yml` body with
+one real change — the plugin is checked out into `plugin/` and mounted at
+`/plugin`, because this workflow runs in the *devkit's* workspace rather than the
+consumer's. That path change is the highest-risk untested detail and should be
+the first thing a trial `workflow_dispatch` exercises.
+
+---
+
+## Install map
+
+| built here | goes to |
+|---|---|
+| `scripts/ci-budget.py` | devkit `e2e/scripts/ci-budget.py` |
+| `scripts/upsert-record-issue.sh` | devkit `e2e/ci/upsert-record-issue.sh` |
+| `scripts/upstream-digest.sh` | devkit `e2e/ci/upstream-digest.sh` |
+| `workflows/ci-budget.yml` | devkit `.github/workflows/ci-budget.yml` |
+| `workflows/upstream-compat.yml` | devkit `.github/workflows/upstream-compat.yml` |
+| `templates/ci-budget.caller.yml` | devkit `templates/` + one line in `link-workflows` |
+
+`upstream-compat.yml` lives **only** in the devkit and needs no consumer change —
+it checks the plugin repos out itself. Only `ci-budget` needs a caller
+distributed to the four consumers.
diff --git a/e2e/ci/ci-budget.py b/e2e/ci/ci-budget.py
new file mode 100755
index 0000000..ce4534c
--- /dev/null
+++ b/e2e/ci/ci-budget.py
@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""
+Build the per-plugin CI budget / e2e record.
+
+WHY THIS IS A SCRIPT AND NOT INLINE YAML
+ Same reason as e2e/ci/prune-artifacts.sh: it is called from the reusable
+ ci-budget.yml AND from the nightly upstream-compat.yml, and it is the kind
+ of logic (pagination, unit conversion, graceful degradation) that rots
+ instantly when copy-pasted into two workflows.
+
+WHAT THE GITHUB API ACTUALLY GIVES US (probed 2026-08-05, not assumed)
+
+ 1. repos/{r}/actions/runs/{id}/timing → billable.UBUNTU.total_ms
+ Reachable with the workflow's own GITHUB_TOKEN (actions:read).
+ USELESS HERE: it returns 0 for every run on these repos, because they are
+ PUBLIC and public-repo Actions minutes are free, so GitHub reports zero
+ *billable* time. Verified on three real chat-comments runs:
+ {"billable":{"UBUNTU":{"total_ms":0,...}},"run_duration_ms":810000}
+ We therefore do NOT use it as the minute source. run_duration_ms is real
+ but is whole-run wall clock including queue/idle between jobs.
+
+ 2. GET /orgs/{org}/settings/billing/actions → HTTP 410 "moved"
+ GET /orgs/{org}/settings/billing/shared-storage → HTTP 410 "moved"
+ Both are gone. gh additionally reports they'd want admin:org. Dead ends.
+
+ 3. GET /organizations/{org}/settings/billing/usage?year=&month=
+ THE one that works. Needs an ORG-scoped token (read:org was enough for
+ the operator PAT; the endpoint is on the "enhanced billing platform").
+ Returns one row per (day, repo, SKU):
+ {"date":"2026-08-04...","product":"actions","sku":"Actions Linux",
+ "quantity":78.0,"unitType":"Minutes","netAmount":0.0,
+ "repositoryName":"agent-zero-plugin-chat-comments"}
+ So MINUTES ARE REAL AND PER-REPO. netAmount is 0.0 because public repos
+ are free — the *cost* is genuinely zero, the *minutes* are not.
+ LIMITATION: a workflow's GITHUB_TOKEN is repo-scoped and has no billing
+ permission at all (there is no `permissions:` key that grants it), so this
+ tier only lights up when an org PAT is provided as a secret. Without it we
+ say so in the record rather than inventing a number.
+
+ 4. repos/{r}/actions/runs/{id}/jobs → per-job started_at/completed_at.
+ Always reachable, no secret. Summing (completed-started) over jobs is our
+ ALWAYS-AVAILABLE minute proxy. It is runner wall clock, which is what
+ GitHub rounds up per job to bill, so it tracks tier-3 closely — it is a
+ proxy, and the record labels it as one.
+
+Output (stdout is the markdown record; --json-out writes the machine copy):
+ minutes_billed from (3) when available, else null + a stated reason
+ minutes_observed from (4), always
+ last e2e run: when, conclusion, duration, commit, link
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from datetime import datetime, timedelta, timezone
+
+API = "https://api.github.com"
+
+
+def _get(url: str, token: str) -> object:
+ req = urllib.request.Request(
+ url,
+ headers={
+ "Authorization": f"Bearer {token}",
+ "Accept": "application/vnd.github+json",
+ "X-GitHub-Api-Version": "2022-11-28",
+ "User-Agent": "a0-devkit-ci-budget",
+ },
+ )
+ with urllib.request.urlopen(req, timeout=60) as r:
+ return json.load(r)
+
+
+def _iso(s: str | None) -> datetime | None:
+ if not s:
+ return None
+ return datetime.fromisoformat(s.replace("Z", "+00:00"))
+
+
+def _fmt_dur(seconds: float) -> str:
+ m, s = divmod(int(seconds), 60)
+ return f"{m}m {s:02d}s"
+
+
+def _age(dt: datetime | None, now: datetime) -> str:
+ if dt is None:
+ return "never"
+ d = now - dt
+ if d < timedelta(hours=1):
+ return f"{int(d.total_seconds() // 60)}m ago"
+ if d < timedelta(days=1):
+ return f"{int(d.total_seconds() // 3600)}h ago"
+ return f"{d.days}d ago"
+
+
+# ---------------------------------------------------------------- runs / jobs
+
+def collect_runs(repo: str, token: str, workflow: str, days: int) -> list[dict]:
+ """Runs of `workflow` in the last `days`. Repo-scoped: always available."""
+ cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+ out: list[dict] = []
+ page = 1
+ while page <= 10: # hard cap: 1000 runs is far more than any window needs
+ url = (
+ f"{API}/repos/{repo}/actions/runs"
+ f"?per_page=100&page={page}&exclude_pull_requests=false"
+ )
+ try:
+ data = _get(url, token)
+ except urllib.error.HTTPError as e: # pragma: no cover - network
+ print(f"::warning::runs query failed: {e}", file=sys.stderr)
+ break
+ runs = data.get("workflow_runs", []) # type: ignore[union-attr]
+ if not runs:
+ break
+ stop = False
+ for r in runs:
+ created = _iso(r.get("created_at"))
+ if created and created < cutoff:
+ stop = True
+ continue
+ if workflow and r.get("name") != workflow:
+ continue
+ out.append(r)
+ if stop:
+ break
+ page += 1
+ return out
+
+
+def job_seconds(repo: str, run_id: int, token: str) -> float:
+ """Summed per-job runner wall clock. Our always-available minute proxy."""
+ try:
+ data = _get(f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", token)
+ except urllib.error.HTTPError: # pragma: no cover - network
+ return 0.0
+ total = 0.0
+ for j in data.get("jobs", []): # type: ignore[union-attr]
+ a, b = _iso(j.get("started_at")), _iso(j.get("completed_at"))
+ if a and b and b > a:
+ total += (b - a).total_seconds()
+ return total
+
+
+# ------------------------------------------------------------------- billing
+
+def billed_minutes(org: str, repo_name: str, token: str | None, days: int) -> dict:
+ """
+ Tier-3 authoritative minutes from the enhanced-billing usage endpoint.
+
+ Returns {"available": False, "reason": ...} when we have no org token, which
+ is the DEFAULT for a plugin repo running on GITHUB_TOKEN. We never guess.
+ """
+ if not token:
+ return {
+ "available": False,
+ "reason": (
+ "no org-scoped billing token supplied "
+ "(GITHUB_TOKEN is repo-scoped and cannot read org billing)"
+ ),
+ }
+ now = datetime.now(timezone.utc)
+ cutoff = now - timedelta(days=days)
+ months = {(now.year, now.month), (cutoff.year, cutoff.month)}
+ minutes = 0.0
+ net = 0.0
+ gross = 0.0
+ gb_hours = 0.0
+ seen = False
+ for year, month in sorted(months):
+ url = (
+ f"{API}/organizations/{org}/settings/billing/usage"
+ f"?year={year}&month={month}"
+ )
+ try:
+ data = _get(url, token)
+ except urllib.error.HTTPError as e:
+ return {
+ "available": False,
+ "reason": f"billing usage endpoint returned HTTP {e.code}",
+ }
+ for it in data.get("usageItems", []): # type: ignore[union-attr]
+ if it.get("repositoryName") != repo_name:
+ continue
+ d = _iso(it.get("date"))
+ if d and d < cutoff:
+ continue
+ seen = True
+ if it.get("unitType") == "Minutes":
+ minutes += float(it.get("quantity") or 0)
+ net += float(it.get("netAmount") or 0)
+ gross += float(it.get("grossAmount") or 0)
+ elif it.get("unitType") == "GigabyteHours":
+ gb_hours += float(it.get("quantity") or 0)
+ return {
+ "available": True,
+ "minutes": round(minutes, 1),
+ "storage_gb_hours": round(gb_hours, 3),
+ "net_usd": round(net, 4),
+ "gross_usd": round(gross, 4),
+ "note": (
+ "netAmount is $0.00 because this repo is PUBLIC and public-repo "
+ "Actions minutes are free. The minutes are real; the bill is not."
+ )
+ if seen and net == 0
+ else "",
+ }
+
+
+# -------------------------------------------------------------------- record
+
+def build_record(args: argparse.Namespace) -> dict:
+ now = datetime.now(timezone.utc)
+ org = args.repo.split("/")[0]
+ repo_name = args.repo.split("/")[1]
+
+ runs = collect_runs(args.repo, args.token, args.workflow, args.days)
+ finished = [r for r in runs if r.get("status") == "completed"]
+
+ observed = 0.0
+ # Cost control: per-run /jobs is one API call each. Cap how many we walk;
+ # beyond the cap we report what we measured and say the window was capped.
+ walked = finished[: args.max_run_lookups]
+ for r in walked:
+ observed += job_seconds(args.repo, r["id"], args.token)
+
+ last = finished[0] if finished else None
+ last_seconds = job_seconds(args.repo, last["id"], args.token) if last else 0.0
+
+ passes = sum(1 for r in finished if r.get("conclusion") == "success")
+ fails = sum(1 for r in finished if r.get("conclusion") == "failure")
+
+ billing = billed_minutes(org, repo_name, args.billing_token, args.days)
+
+ return {
+ "schema": "a0-devkit/ci-budget@1",
+ "generated_at": now.isoformat(),
+ "repo": args.repo,
+ "plugin": args.plugin,
+ "workflow": args.workflow,
+ "window_days": args.days,
+ "runs_total": len(finished),
+ "runs_measured": len(walked),
+ "runs_capped": len(finished) > len(walked),
+ "runs_success": passes,
+ "runs_failure": fails,
+ "minutes_observed": round(observed / 60, 1),
+ "minutes_billed": billing,
+ "last_run": None
+ if not last
+ else {
+ "id": last["id"],
+ "conclusion": last.get("conclusion"),
+ "event": last.get("event"),
+ "created_at": last.get("created_at"),
+ "age": _age(_iso(last.get("created_at")), now),
+ "duration": _fmt_dur(last_seconds),
+ "duration_seconds": round(last_seconds),
+ "sha": (last.get("head_sha") or "")[:8],
+ "url": last.get("html_url"),
+ },
+ }
+
+
+def render(rec: dict) -> str:
+ ico = {"success": "✅", "failure": "❌", "cancelled": "⚪", "timed_out": "⏱️"}
+ last = rec["last_run"]
+ b = rec["minutes_billed"]
+
+ if b["available"]:
+ billed = f"**{b['minutes']} min** (net ${b['net_usd']:.2f})"
+ billed_note = b.get("note") or ""
+ else:
+ billed = "_not available_"
+ billed_note = b["reason"]
+
+ # These two numbers are NOT the same measurement and must never be shown as
+ # if one validated the other. Measured live on chat_comments: 454 billed min
+ # vs 199.8 observed min over the same 30 days. The gap is not error — billing
+ # is REPO-WIDE (devkit-sync, unit, Dependabot, every workflow) and rounds each
+ # job up to a whole minute, while `observed` is filtered to one workflow.
+ scope_note = (
+ f"Scopes differ by design: **billed** is repo-wide across *all* workflows "
+ f"and rounds each job up to a whole minute; **observed** is `{rec['workflow']}` "
+ f"only. Use billed for the invoice, observed for what the e2e suite costs."
+ )
+
+ lines = [
+ f"## CI budget — `{rec['plugin']}`",
+ "",
+ f"Window: last **{rec['window_days']} days** · workflow `{rec['workflow']}` "
+ f"· generated {rec['generated_at'][:16].replace('T', ' ')} UTC",
+ "",
+ "| | |",
+ "|---|---|",
+ ]
+ if last:
+ lines += [
+ f"| **e2e last ran** | [{last['created_at'][:16].replace('T', ' ')} UTC]({last['url']}) "
+ f"({last['age']}) |",
+ f"| **result** | {ico.get(last['conclusion'], '❔')} `{last['conclusion']}` "
+ f"on `{last['sha']}` (via `{last['event']}`) |",
+ f"| **that run took** | {last['duration']} |",
+ ]
+ else:
+ lines += [
+ f"| **e2e last ran** | ⚠️ **never** — no completed `{rec['workflow']}` run "
+ f"in the last {rec['window_days']} days |",
+ "| **result** | — |",
+ "| **that run took** | — |",
+ ]
+ lines += [
+ f"| **runs in window** | {rec['runs_total']} "
+ f"({rec['runs_success']} ✅ / {rec['runs_failure']} ❌) |",
+ f"| **minutes billed** _(whole repo, all workflows)_ | {billed} |",
+ f"| **minutes observed** _(`{rec['workflow']}` only, runner wall clock)_ | "
+ f"{rec['minutes_observed']} min over {rec['runs_measured']} run(s)"
+ + (" _(window capped)_" if rec["runs_capped"] else "")
+ + " |",
+ ]
+ if b["available"] and b.get("storage_gb_hours"):
+ lines.append(
+ f"| **artifact storage** _(whole repo)_ | {b['storage_gb_hours']} GB-hours |"
+ )
+ lines += ["", f"> {scope_note}"]
+ if billed_note:
+ lines += ["", f"> {billed_note}"]
+ lines += [
+ "",
+ "`minutes (observed)` is summed per-job runner wall clock from the "
+ "Actions API — a proxy, always available. `minutes (billed)` is the "
+ "authoritative org billing figure and only appears when an org-scoped "
+ "token is configured. The per-run `/timing` endpoint is deliberately "
+ "unused: it reports 0 billable ms on public repos.",
+ ]
+ return "\n".join(lines)
+
+
+def main() -> int:
+ p = argparse.ArgumentParser(description="Generate the plugin CI budget record.")
+ p.add_argument("--repo", required=True, help="owner/name")
+ p.add_argument("--plugin", required=True)
+ p.add_argument("--workflow", default="plugin-e2e", help="workflow NAME to budget")
+ p.add_argument("--days", type=int, default=30)
+ p.add_argument("--max-run-lookups", type=int, default=40)
+ p.add_argument("--json-out", default="")
+ p.add_argument("--md-out", default="")
+ p.add_argument("--token", default=os.environ.get("GITHUB_TOKEN", ""))
+ p.add_argument(
+ "--billing-token",
+ default=os.environ.get("BILLING_TOKEN", "") or None,
+ help="org-scoped PAT for /organizations/{org}/settings/billing/usage",
+ )
+ args = p.parse_args()
+ if not args.token:
+ print("::error::no GITHUB_TOKEN", file=sys.stderr)
+ return 1
+
+ rec = build_record(args)
+ md = render(rec)
+
+ if args.json_out:
+ with open(args.json_out, "w", encoding="utf-8") as fh:
+ json.dump(rec, fh, indent=2)
+ fh.write("\n")
+ if args.md_out:
+ with open(args.md_out, "w", encoding="utf-8") as fh:
+ fh.write(md + "\n")
+ print(md)
+
+ # Staleness is a FAILURE MODE, not a footnote. If the operator asked for
+ # "when last they ran", a record that quietly says "never" is the devkit-sync
+ # bug again. Exit non-zero so a human sees it.
+ if args.days and rec["last_run"] is None:
+ print(
+ f"::error::no completed {args.workflow} run in {args.days} days "
+ f"for {args.repo}",
+ file=sys.stderr,
+ )
+ return 2
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/e2e/ci/upsert-record-issue.sh b/e2e/ci/upsert-record-issue.sh
new file mode 100755
index 0000000..99a9ee0
--- /dev/null
+++ b/e2e/ci/upsert-record-issue.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Upsert a single long-lived GitHub issue whose BODY is the record.
+#
+# WHY AN ISSUE AND NOT A COMMITTED FILE
+# A committed docs/CI-BUDGET.md would be rewritten on every run, so every e2e
+# run would produce a commit (or a PR) whose only content is a changed
+# timestamp. That is precisely the churn that got the nightly devkit-sync
+# deleted — 189 runs, 29 useful bumps. An issue body is:
+# * durable and permalinked (unlike a job summary, which dies with the run),
+# * editable in place with zero commits, zero PRs, zero branch noise,
+# * already where maintainers look, and pinnable per repo,
+# * silent on edit (editing a body sends no notification), so it can update
+# as often as we like without training anyone to ignore it.
+#
+# ANTI-STALENESS
+# MODE=only-on-failure closes the issue when everything is green and (re)opens
+# it when something is red — so an OPEN issue always means "act on this", and
+# its absence is not ambiguous. The nightly's own last-run timestamp is written
+# into the body either way, so "it silently stopped running" is visible rather
+# than invisible.
+#
+# Env:
+# GH_TOKEN token with issues:write on $REPO
+# REPO owner/name
+# TITLE exact issue title used as the upsert key
+# BODY_FILE file whose contents become the issue body
+# LABEL label to apply/create (default: ci-budget)
+# MODE always | only-on-failure (default: always)
+# STATUS ok | fail — consulted only when MODE=only-on-failure
+#
+# Never fails the caller: record-keeping must not turn a green run red.
+set -euo pipefail
+
+: "${REPO:?set REPO}"
+: "${TITLE:?set TITLE}"
+: "${BODY_FILE:?set BODY_FILE}"
+LABEL="${LABEL:-ci-budget}"
+MODE="${MODE:-always}"
+STATUS="${STATUS:-ok}"
+
+[ -f "$BODY_FILE" ] || { echo "::warning::$BODY_FILE missing — nothing to upsert"; exit 0; }
+
+# Ensure the label exists (idempotent; ignore "already exists").
+gh label create "$LABEL" --repo "$REPO" \
+ --color 0e8a16 --description "Automated devkit record — do not close by hand" \
+ >/dev/null 2>&1 || true
+
+# Find the existing record issue by exact title, open OR closed, so we reuse the
+# same one forever instead of accumulating a new issue per run.
+num=$(gh issue list --repo "$REPO" --label "$LABEL" --state all --limit 50 \
+ --json number,title \
+ --jq "[.[] | select(.title == \"$TITLE\")] | first | .number" 2>/dev/null || echo "")
+[ "$num" = "null" ] && num=""
+
+if [ -z "$num" ]; then
+ if [ "$MODE" = "only-on-failure" ] && [ "$STATUS" = "ok" ]; then
+ echo "all green and no existing record issue — nothing to open"
+ exit 0
+ fi
+ num=$(gh issue create --repo "$REPO" --title "$TITLE" \
+ --body-file "$BODY_FILE" --label "$LABEL" \
+ --json number --jq .number 2>/dev/null \
+ || gh issue create --repo "$REPO" --title "$TITLE" \
+ --body-file "$BODY_FILE" --label "$LABEL" | grep -oE '[0-9]+$')
+ echo "created record issue #$num"
+else
+ gh issue edit "$num" --repo "$REPO" --body-file "$BODY_FILE" >/dev/null
+ echo "updated record issue #$num (body replaced, no comment, no notification)"
+fi
+
+if [ "$MODE" = "only-on-failure" ]; then
+ if [ "$STATUS" = "ok" ]; then
+ gh issue close "$num" --repo "$REPO" \
+ --comment "All plugins green against upstream — closing. This issue reopens automatically on the next red night." \
+ >/dev/null 2>&1 || true
+ echo "green → record issue #$num closed"
+ else
+ gh issue reopen "$num" --repo "$REPO" >/dev/null 2>&1 || true
+ echo "red → record issue #$num open"
+ fi
+fi
+
+echo "record: https://github.com/$REPO/issues/$num"
diff --git a/e2e/ci/upstream-digest.sh b/e2e/ci/upstream-digest.sh
new file mode 100755
index 0000000..7c5431e
--- /dev/null
+++ b/e2e/ci/upstream-digest.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Resolve the current manifest digest of an image on Docker Hub, anonymously.
+#
+# WHY THIS EXISTS — it is the whole cost argument for the nightly.
+# A calendar-driven nightly re-runs 4 × ~14min e2e every night whether or not
+# upstream changed. docker.io/agent0ai/agent-zero:latest does NOT change
+# nightly. Gating the expensive fan-out on "did the digest actually move"
+# turns ~1,700 runner-min/month into roughly the number of upstream releases
+# × 56 min, plus a ~20s probe every night. Same detection latency, ~10% of the
+# spend, and — unlike a calendar trigger — every expensive run corresponds to a
+# real upstream change, so a red result is never ambiguous.
+#
+# Uses the anonymous pull token, so no Docker credentials are needed and this
+# works on a fork PR. Prints the digest on stdout; writes GH outputs if
+# GITHUB_OUTPUT is set.
+#
+# Env/args:
+# IMAGE default docker.io/agent0ai/agent-zero:latest
+set -euo pipefail
+
+IMAGE="${1:-${IMAGE:-docker.io/agent0ai/agent-zero:latest}}"
+
+ref="${IMAGE#docker.io/}"
+repo="${ref%:*}"
+tag="${ref##*:}"
+[ "$tag" = "$repo" ] && tag=latest
+case "$repo" in */*) ;; *) repo="library/$repo" ;; esac
+
+token=$(curl -fsS "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \
+ | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
+
+digest=$(curl -fsSI -H "Authorization: Bearer ${token}" \
+ -H 'Accept: application/vnd.oci.image.index.v1+json' \
+ -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
+ -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+ "https://registry-1.docker.io/v2/${repo}/manifests/${tag}" \
+ | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:"{print $2}')
+
+if [ -z "${digest:-}" ]; then
+ echo "::error::could not resolve digest for ${IMAGE}" >&2
+ exit 1
+fi
+
+echo "$digest"
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ {
+ echo "digest=${digest}"
+ echo "digest_short=${digest#sha256:}"
+ echo "image=${IMAGE}"
+ } >> "$GITHUB_OUTPUT"
+fi
diff --git a/templates/ci-budget.caller.yml b/templates/ci-budget.caller.yml
new file mode 100644
index 0000000..937b10d
--- /dev/null
+++ b/templates/ci-budget.caller.yml
@@ -0,0 +1,37 @@
+# ci-budget — caller template.
+#
+# Copied into a consumer plugin repo at .github/workflows/ci-budget.yml by
+# `make link-workflows`. Refreshes the plugin's CI budget / e2e record:
+# a job summary, a ci-budget.json artifact, and one long-lived record issue
+# updated in place (no commits, no PRs).
+#
+# Weekly, not per-PR: the record is a rolling 30-day view, so refreshing it on
+# every PR would spend runner time to change a timestamp. It is also
+# workflow_dispatch-able for an on-demand read.
+name: ci-budget
+
+on:
+ schedule:
+ # Mondays 06:40 UTC. Off the hour (GitHub's :00 queues are the worst) and
+ # early enough that the record is fresh when the week starts.
+ - cron: "40 6 * * 1"
+ workflow_dispatch:
+ inputs:
+ window_days:
+ description: "Rolling window in days"
+ type: string
+ default: "30"
+
+permissions:
+ contents: read
+ actions: read
+ issues: write
+
+jobs:
+ budget:
+ uses: agent-zero-plugins/agent-zero-plugin-development-testkit/.github/workflows/ci-budget.yml@main
+ # Forwards BILLING_READ_TOKEN if the org has set it. Absent → the record
+ # honestly reports billed minutes as unavailable rather than guessing.
+ secrets: inherit
+ with:
+ window-days: ${{ fromJSON(inputs.window_days || '30') }}