From 0695a8ede8fe8586f738b292fb5c55d504b5c477 Mon Sep 17 00:00:00 2001 From: Omar Nahhas Sanchez Date: Wed, 5 Aug 2026 10:35:38 +0100 Subject: [PATCH] feat(ci): upstream-compat schedule + CI budget record (DEC-082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream A0 is the only thing anyone actually runs, and nothing tested it. The reusable workflow defaults a0_image to the private fork and none of the four plugins overrode it, so every PR proved the plugin worked on the fork while the target that matters went unverified. Coverage pointed at the wrong image. upstream-compat re-runs the four suites when UPSTREAM moves — something no PR can catch, because nothing in the repo changed. Gated on the manifest digest actually moving: measured, upstream ships ~1.5x/week against a 42-minute fan-out, so digest-gating costs ~295 runner-min/month instead of ~1,275 for the same <=24h detection. Runs pin the resolved digest, not :latest, so a failure is reproducible against the exact image that broke it. Deliberately unlike the nightly just deleted in DEC-079: that opened a PR every night regardless and then failed silently behind an approval gate for three weeks. This opens no PRs, commits nothing, and reports one issue that auto-closes when green — so an open issue always means act. Three guards keep it honest: the probe hard-fails rather than shrugging, an empty matrix is an error not a vacuous pass, and the report job asserts the matrix ran if the probe asked for it. CI budget record lives in a long-lived per-repo issue upserted in place, NOT a committed file — a committed record produces a commit per run whose only delta is a timestamp, which is exactly the churn that got devkit-sync deleted. Worth knowing: runs/{id}/timing returns total_ms 0 on every run for these public repos, so a budget built on it would report 0 min forever and look like it worked. It is deliberately unused and the record says so on its face. --- .github/workflows/ci-budget.yml | 178 +++++++++++ .github/workflows/upstream-compat.yml | 415 ++++++++++++++++++++++++++ SPEC.md | 24 +- docs/upstream-compat-and-ci-budget.md | 335 +++++++++++++++++++++ e2e/ci/ci-budget.py | 389 ++++++++++++++++++++++++ e2e/ci/upsert-record-issue.sh | 84 ++++++ e2e/ci/upstream-digest.sh | 52 ++++ templates/ci-budget.caller.yml | 37 +++ 8 files changed, 1513 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-budget.yml create mode 100644 .github/workflows/upstream-compat.yml create mode 100644 docs/upstream-compat-and-ci-budget.md create mode 100755 e2e/ci/ci-budget.py create mode 100755 e2e/ci/upsert-record-issue.sh create mode 100755 e2e/ci/upstream-digest.sh create mode 100644 templates/ci-budget.caller.yml diff --git a/.github/workflows/ci-budget.yml b/.github/workflows/ci-budget.yml new file mode 100644 index 0000000..d16cf6d --- /dev/null +++ b/.github/workflows/ci-budget.yml @@ -0,0 +1,178 @@ +# Reusable ci-budget workflow — the per-plugin CI budget / e2e record. +# +# WHAT IT PRODUCES +# 1. A job summary on every invocation (ephemeral, free, always readable from +# the run page). +# 2. A ci-budget.json artifact (machine-readable, for a future fleet roll-up). +# 3. A single long-lived issue per repo whose BODY is the record, upserted in +# place — durable and permalinked with ZERO commits and ZERO PRs. +# +# Deliberately NOT a committed docs/CI-BUDGET.md: that would produce a commit +# per run whose only delta is a timestamp, which is the churn pattern that got +# the nightly devkit-sync deleted. +# +# HOW TO INVOKE +# * on a schedule (weekly) via the caller template — the cheap default; the +# record is a few API calls, ~20s, no containers, no runner minutes worth +# counting; +# * or as a final job of plugin-e2e, if you want the record refreshed the +# moment an e2e finishes (pass workflow-name and it will pick the run up). +name: ci-budget + +on: + workflow_call: + inputs: + workflow-name: + description: "NAME (not filename) of the workflow to budget." + required: false + type: string + default: "plugin-e2e" + window-days: + description: "Rolling window for minutes + pass/fail counts." + required: false + type: number + default: 30 + devkit-submodule-path: + description: "Path to the vendored devkit submodule." + required: false + type: string + default: "tests/_testkit" + publish-issue: + description: "Upsert the durable record issue (else summary + artifact only)." + required: false + type: boolean + default: true + fail-on-stale: + description: "Fail the job when no e2e run exists in the window." + required: false + type: boolean + default: false + secrets: + # OPTIONAL, and the workflow is fully useful without it. It unlocks the + # authoritative billed-minutes tier from the org billing usage endpoint, + # which the repo-scoped GITHUB_TOKEN structurally cannot read. Without it + # the record says so in plain words instead of guessing a number. + BILLING_READ_TOKEN: + description: "Org-scoped PAT (read:org) for /organizations/{org}/settings/billing/usage." + required: false + +jobs: + budget: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read # runs + jobs timing + issues: write # upsert the durable record issue + steps: + - name: Checkout consumer (no submodules) + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + submodules: false + + - name: Init devkit submodule + env: + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + # The devkit is public, so this needs no credentials (DEC-075). + run: git submodule update --init "$DEVKIT_PATH" + + - name: Resolve plugin name + id: resolve + run: | + set -euo pipefail + y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; } + plugin_dir="" + [ -f .devkit.yml ] && plugin_dir=$(y plugin_dir .devkit.yml) + if [ -z "$plugin_dir" ]; then + plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)") + fi + name=$(y name "$plugin_dir/plugin.yaml") + [ -n "$name" ] || name="${GITHUB_REPOSITORY##*/}" + echo "plugin_name=$name" >> "$GITHUB_OUTPUT" + echo "resolved plugin: $name" + + - name: Detect whether a billing token is available + id: creds + # The `secrets` context is NOT allowed inside an `if:` expression — using + # it there invalidates the entire workflow file and the run dies as a + # startup_failure with zero jobs. Probe it in a run step (secrets ARE + # available there via env) and branch on the step output. Only a yes/no + # is printed; the token itself never is. + env: + BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }} + run: | + if [ -n "${BILLING_TOKEN:-}" ]; then + echo "have_billing=true" >> "$GITHUB_OUTPUT" + echo "org billing token present — billed-minutes tier enabled" + else + echo "have_billing=false" >> "$GITHUB_OUTPUT" + echo "no org billing token — record will report billed minutes as unavailable" + fi + + - name: Build the record + id: build + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }} + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + PLUGIN: ${{ steps.resolve.outputs.plugin_name }} + WF: ${{ inputs.workflow-name }} + DAYS: ${{ inputs.window-days }} + run: | + set -uo pipefail + python3 "$DEVKIT_PATH/e2e/scripts/ci-budget.py" \ + --repo "$GITHUB_REPOSITORY" \ + --plugin "$PLUGIN" \ + --workflow "$WF" \ + --days "$DAYS" \ + --json-out ci-budget.json \ + --md-out ci-budget.md + echo "rc=$?" >> "$GITHUB_OUTPUT" + + - name: Write job summary + if: always() + run: | + if [ -f ci-budget.md ]; then + cat ci-budget.md >> "$GITHUB_STEP_SUMMARY" + else + echo "## CI budget — generation FAILED" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload machine-readable record + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-budget-${{ steps.resolve.outputs.plugin_name }} + path: ci-budget.json + if-no-files-found: warn + retention-days: 90 + + - name: Upsert durable record issue + if: always() && inputs.publish-issue + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TITLE: "CI budget — ${{ steps.resolve.outputs.plugin_name }}" + BODY_FILE: ci-budget.md + LABEL: ci-budget + MODE: always + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + run: | + { + echo "" + echo "---" + echo "Maintained automatically by the devkit \`ci-budget\` workflow —" + echo "the body is replaced in place on each run, so there is no commit and" + echo "no PR. Last refreshed by [run ${GITHUB_RUN_ID}](https://github.com/${REPO}/actions/runs/${GITHUB_RUN_ID})." + } >> ci-budget.md + bash "$DEVKIT_PATH/e2e/ci/upsert-record-issue.sh" + + - name: Fail on stale e2e + # Off by default. Turn it on once the fleet's cadence is steady and you + # want "nobody has run e2e in 30 days" to be loud rather than a footnote. + if: always() && inputs.fail-on-stale && steps.build.outputs.rc != '0' + run: | + echo "::error::ci-budget reported a stale or missing e2e record" + exit 1 diff --git a/.github/workflows/upstream-compat.yml b/.github/workflows/upstream-compat.yml new file mode 100644 index 0000000..457fe25 --- /dev/null +++ b/.github/workflows/upstream-compat.yml @@ -0,0 +1,415 @@ +# upstream-compat — continuous backwards-support check against LATEST UPSTREAM A0. +# +# THE GAP THIS CLOSES +# Every consumer's .devkit.yml pins the private fork image, and none of the +# four target plugins overrides `a0_image`. So the fleet's e2e today proves +# "works on the fork" and says NOTHING about "still works on upstream". The +# machinery already exists — plugin-e2e.yml takes an a0_image input and the +# devkit's own sample-plugin-e2e.yml already boots +# docker.io/agent0ai/agent-zero:latest. What was missing is the scheduled +# fan-out and somewhere the result is seen. That is all this file adds. +# +# HOW THIS DIFFERS FROM THE NIGHTLY devkit-sync THAT WAS DELETED +# devkit-sync burned 189 runs to produce 29 merged bumps and then silently +# no-op'd for three weeks. Three concrete differences: +# +# 1. CHANGE-GATED, NOT CALENDAR-GATED. A ~20s probe job resolves the upstream +# manifest digest and only fans out the expensive legs when the digest has +# actually moved since the last full run (tracked via actions/cache — no +# commits, no state file). Upstream :latest does not change nightly, so +# this is the difference between ~1,700 runner-min/month and roughly +# (upstream releases × 56 min). Detection latency is unchanged: the probe +# still runs every night. +# 2. IT CANNOT SILENTLY NO-OP. A weekly floor (max-skip-days) forces a full +# run even when the digest is unchanged, so "nothing happened for three +# weeks" is impossible. The probe also HARD-FAILS if it cannot resolve the +# digest, rather than skipping quietly, and the report job asserts that the +# matrix actually produced results — an empty matrix is a failure, not a +# pass. Every outcome is stamped into the record issue with a timestamp, so +# staleness is visible on the artifact itself. +# 3. ZERO PR CHURN. It never opens a pull request and never commits. Its only +# write is one long-lived issue, opened when a plugin breaks against +# upstream and closed automatically when the fleet is green again. An open +# issue means "act"; there is no steady-state noise to learn to ignore. +name: upstream-compat + +on: + schedule: + # 03:20 UTC — off the top of the hour, where GitHub's scheduler queues are + # worst. The probe runs nightly; the expensive legs are digest-gated. + - cron: "20 3 * * *" + workflow_dispatch: + inputs: + force: + description: "Run the full matrix even if the upstream digest is unchanged" + type: boolean + default: false + plugins: + description: "Override the plugin list (space-separated repo names)" + type: string + default: "" + +permissions: + contents: read + actions: write + issues: write + +env: + UPSTREAM_IMAGE: docker.io/agent0ai/agent-zero:latest + # The fleet under continuous backwards-support check. Space-separated repo + # names within agent-zero-plugins. Adding a plugin is a one-word edit. + PLUGINS: >- + agent-zero-plugin-chat-comments + agent-zero-plugin-diff-visualizer + agent-zero-plugin-mermaid-diagrams + agent-zero-plugin-share-chat + # Force a full run at least this often even if upstream never moves, so the + # workflow can never quietly become a no-op. + MAX_SKIP_DAYS: 7 + +jobs: + # ---------------------------------------------------------------- 1. probe + probe: + name: probe upstream digest + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + digest: ${{ steps.digest.outputs.digest }} + changed: ${{ steps.decide.outputs.changed }} + reason: ${{ steps.decide.outputs.reason }} + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve upstream manifest digest + id: digest + # Hard-fails if it cannot resolve. A probe that shrugs and exits 0 is how + # devkit-sync managed to no-op for three weeks unnoticed. + env: + IMAGE: ${{ env.UPSTREAM_IMAGE }} + run: bash e2e/ci/upstream-digest.sh "$IMAGE" + + - name: Look up last-tested digest + id: cache + uses: actions/cache/restore@v4 + with: + # Cache HIT on this exact digest ⇒ we already ran the full matrix + # against this image. No commits, no state file, no PR. + path: .upstream-compat-state + key: upstream-compat-${{ steps.digest.outputs.digest_short }} + restore-keys: upstream-compat- + + - name: Decide whether to fan out + id: decide + env: + HIT: ${{ steps.cache.outputs.cache-hit }} + FORCE: ${{ inputs.force }} + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + last_ts=0 + [ -f .upstream-compat-state ] && last_ts=$(sed -n 's/^ts=//p' .upstream-compat-state | head -1) + : "${last_ts:=0}" + now=$(date +%s) + age_days=$(( (now - last_ts) / 86400 )) + + if [ "${FORCE}" = "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=forced via workflow_dispatch" >> "$GITHUB_OUTPUT" + elif [ "${HIT}" != "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=upstream digest moved to ${DIGEST}" >> "$GITHUB_OUTPUT" + elif [ "$last_ts" -eq 0 ] || [ "$age_days" -ge "${MAX_SKIP_DAYS}" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=weekly floor reached (${age_days}d since last full run) — digest unchanged" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "reason=upstream digest unchanged (${DIGEST}), last full run ${age_days}d ago — skipping the expensive matrix" >> "$GITHUB_OUTPUT" + fi + + - name: Build plugin matrix + id: matrix + env: + OVERRIDE: ${{ inputs.plugins }} + run: | + set -euo pipefail + list="${OVERRIDE:-$PLUGINS}" + json=$(printf '%s\n' $list | python3 -c 'import sys,json; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))') + n=$(printf '%s' "$json" | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))') + # An empty matrix would make every downstream job "succeed" while + # testing nothing. Refuse. + [ "$n" -gt 0 ] || { echo "::error::plugin matrix is empty"; exit 1; } + echo "matrix=$json" >> "$GITHUB_OUTPUT" + echo "matrix ($n): $json" + + - name: Probe summary + run: | + { + echo "## upstream-compat probe" + echo "" + echo "| | |" + echo "|---|---|" + echo "| image | \`${UPSTREAM_IMAGE}\` |" + echo "| digest | \`${{ steps.digest.outputs.digest }}\` |" + echo "| fan out? | **${{ steps.decide.outputs.changed }}** |" + echo "| reason | ${{ steps.decide.outputs.reason }} |" + } >> "$GITHUB_STEP_SUMMARY" + + # ------------------------------------------------------------ 2. the legs + compat: + name: ${{ matrix.plugin }} + needs: probe + if: needs.probe.outputs.changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false # one broken plugin must not hide the state of the others + max-parallel: 2 # nested-A0 legs are disk-hungry; stagger them + matrix: + plugin: ${{ fromJSON(needs.probe.outputs.matrix) }} + permissions: + contents: read + steps: + - name: Checkout the plugin under test + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + # Public repos, so the default token suffices for a cross-repo read. + repository: ${{ github.repository_owner }}/${{ matrix.plugin }} + submodules: recursive + path: plugin + + - name: Free disk space + run: | + df -h / | tail -1 + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ + /usr/local/lib/android /opt/hostedtoolcache/CodeQL \ + /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" 2>/dev/null || true + sudo docker image prune -af 2>/dev/null || true + echo "after reclaim:"; df -h / | tail -1 + + - name: Resolve plugin + id: resolve + working-directory: plugin + run: | + set -euo pipefail + y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; } + plugin_dir="" display_name="" + if [ -f .devkit.yml ]; then + plugin_dir=$(y plugin_dir .devkit.yml) + display_name=$(y display_name .devkit.yml) + fi + if [ -z "$plugin_dir" ]; then + plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)") + fi + [ -d "$plugin_dir" ] && [ -f "$plugin_dir/plugin.yaml" ] || { + echo "::error::could not resolve plugin_dir in ${{ matrix.plugin }}"; exit 1; } + plugin_name=$(y name "$plugin_dir/plugin.yaml") + [ -n "$display_name" ] || display_name=$(y title "$plugin_dir/plugin.yaml") + [ -n "$display_name" ] || display_name="$plugin_name" + pod_env="" + if [ -f .devkit.yml ]; then + pod_env=$(y e2e_pod_env .devkit.yml) + if [ -z "$pod_env" ]; then + pod_env=$(awk ' + /^e2e_pod_env:[[:space:]]*$/ {inblk=1; next} + inblk && /^[[:space:]]+[A-Za-z_][A-Za-z0-9_]*:/ { + l=$0; sub(/^[[:space:]]+/,"",l) + k=l; sub(/:.*/,"",k) + v=l; sub(/^[^:]*:[[:space:]]*/,"",v); gsub(/^["'\'']|["'\'']$/,"",v) + printf "%s=%s ", k, v; next } + inblk && /^[^[:space:]]/ {inblk=0} + ' .devkit.yml) + fi + fi + echo "plugin_dir=$plugin_dir" >> "$GITHUB_OUTPUT" + echo "plugin_name=$plugin_name" >> "$GITHUB_OUTPUT" + echo "display_name=$display_name" >> "$GITHUB_OUTPUT" + echo "pod_env=$pod_env" >> "$GITHUB_OUTPUT" + echo "Resolved: dir=$plugin_dir name=$plugin_name a0=UPSTREAM" + + - name: Build devkit devcontainer (rootless) + run: podman build -t plugin-devkit:ci -f devcontainer/Containerfile devcontainer + + - name: Package + run lifecycle against UPSTREAM A0 + env: + PLUGIN_DIR: ${{ steps.resolve.outputs.plugin_dir }} + PLUGIN_NAME: ${{ steps.resolve.outputs.plugin_name }} + DISPLAY_NAME: ${{ steps.resolve.outputs.display_name }} + A0_POD_ENV: ${{ steps.resolve.outputs.pod_env }} + # The whole point: pin the digest the probe resolved, not the moving + # tag, so every leg of one night tests the SAME image and a red result + # names an exact upstream artifact. + A0_IMAGE: docker.io/agent0ai/agent-zero@${{ needs.probe.outputs.digest }} + run: | + set -euo pipefail + mkdir -p dist "$GITHUB_WORKSPACE/artifacts" + chmod 777 "$GITHUB_WORKSPACE/artifacts" + ( cd "plugin/$PLUGIN_DIR" && zip -qr "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip" . \ + -x '.git/*' 'tests/*' '.github/*' '.skills/*' 'dist/*' '*/node_modules/*' \ + '.devkit.yml' '.gitmodules' '.gitignore' ) + BEHAVIOUR_ARG="" + [ -f "plugin/tests/e2e/behaviour.mjs" ] && BEHAVIOUR_ARG="/plugin/tests/e2e/behaviour.mjs" + SPECS_JSON="[]" + if ls plugin/tests/e2e/specs/*.mjs >/dev/null 2>&1; then + SPECS_JSON=$(for f in plugin/tests/e2e/specs/*.mjs; do + n=$(basename "$f" .mjs); printf '{"name":"%s","path":"/plugin/%s"}\n' "$n" "${f#plugin/}"; done \ + | paste -sd, - | sed 's/^/[/; s/$/]/') + fi + HARNESS=run-lifecycle.sh + if ls plugin/tests/e2e/features/*.feature >/dev/null 2>&1; then HARNESS=run-bdd.sh; fi + echo "harness: $HARNESS image: $A0_IMAGE" + podman run --rm \ + --device /dev/fuse \ + --security-opt label=disable \ + --sysctl net.ipv4.ip_unprivileged_port_start=0 \ + -v "$GITHUB_WORKSPACE":/workspace:ro \ + -v "$GITHUB_WORKSPACE/plugin":/plugin:ro \ + -v "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip":/tmp/plugin.zip:ro \ + -v "$GITHUB_WORKSPACE/artifacts":/artifacts:rw \ + -e A0_IMAGE="$A0_IMAGE" \ + -e PLUGIN_ZIP=/tmp/plugin.zip \ + -e PLUGIN_DISPLAY_NAME="$DISPLAY_NAME" \ + -e PLUGIN_NAME="$PLUGIN_NAME" \ + -e HOOK_DIR=/plugin/tests/e2e/hooks \ + -e BEHAVIOUR_FILE="$BEHAVIOUR_ARG" \ + -e BEHAVIOUR_SPECS="$SPECS_JSON" \ + -e A0_POD_ENV="$A0_POD_ENV" \ + -e ARTIFACT_DIR=/artifacts \ + -e WORKSPACE=/plugin \ + -e DEVKIT_PATH=. \ + -e BDD_TRACE=retain-on-failure \ + plugin-devkit:ci \ + -lc 'bash /workspace/e2e/harness/'"$HARNESS" + + - name: Upload failure evidence + # retain-on-failure, not always: a nightly that uploads a full trace set + # for four green plugins every week is storage the operator pays for and + # nobody opens. A red night gets the full trace. + if: failure() + uses: actions/upload-artifact@v4 + with: + name: upstream-compat-${{ matrix.plugin }} + path: artifacts/ + if-no-files-found: warn + retention-days: 30 + + # ---------------------------------------------------------------- 3. report + report: + name: report + needs: [probe, compat] + if: always() && needs.probe.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Assemble the record + id: rec + env: + CHANGED: ${{ needs.probe.outputs.changed }} + REASON: ${{ needs.probe.outputs.reason }} + DIGEST: ${{ needs.probe.outputs.digest }} + MATRIX: ${{ needs.probe.outputs.matrix }} + COMPAT: ${{ needs.compat.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + now=$(date -u +'%Y-%m-%d %H:%M UTC') + { + echo "# Upstream compatibility — \`${UPSTREAM_IMAGE}\`" + echo "" + echo "| | |" + echo "|---|---|" + echo "| last checked | **${now}** |" + echo "| upstream digest | \`${DIGEST}\` |" + echo "| full matrix ran? | **${CHANGED}** — ${REASON} |" + echo "| plugins under check | $(printf '%s' "$MATRIX" | python3 -c 'import sys,json; print(", ".join("`%s`" % p for p in json.load(sys.stdin)))') |" + echo "| result | ${COMPAT:-skipped} |" + echo "| evidence | [${GITHUB_RUN_ID}](${RUN_URL}) |" + } > record.md + + # STATUS drives whether the record issue is opened or closed. + # compat skipped (digest unchanged) → ok, nothing to report + # compat success → ok + # anything else → fail + case "${COMPAT}" in + ""|skipped|success) status=ok ;; + *) status=fail ;; + esac + if [ "$status" = "fail" ]; then + { + echo "" + echo "## ❌ At least one plugin fails against the latest upstream A0" + echo "" + echo "Open the [run](${RUN_URL}) and expand the red matrix leg. Per-leg" + echo "Playwright traces are attached as \`upstream-compat-\` artifacts." + echo "" + echo "This is a **backwards-support regression**: the plugin still passes on" + echo "the fork image its own \`plugin-e2e\` pins, so nothing in the plugin" + echo "repo went red. Upstream moved to \`${DIGEST}\` and something the plugin" + echo "depends on moved with it." + } >> record.md + fi + { + echo "" + echo "---" + echo "Maintained by the devkit \`upstream-compat\` workflow. The body is" + echo "replaced in place — no commits, no pull requests. This issue opens when a" + echo "plugin breaks against upstream and closes automatically when the fleet is" + echo "green, so an OPEN issue always means action is needed." + } >> record.md + echo "status=$status" >> "$GITHUB_OUTPUT" + cat record.md >> "$GITHUB_STEP_SUMMARY" + + - name: Guard against a silent no-op + # The devkit-sync failure mode, made impossible: if the probe said "fan + # out" then the matrix MUST have produced a result. A green report with + # nothing tested is a bug, not a pass. + env: + CHANGED: ${{ needs.probe.outputs.changed }} + COMPAT: ${{ needs.compat.result }} + run: | + if [ "$CHANGED" = "true" ] && { [ -z "$COMPAT" ] || [ "$COMPAT" = "skipped" ]; }; then + echo "::error::probe requested a full run but the compat matrix never executed" + exit 1 + fi + echo "no-op guard OK (changed=$CHANGED compat=${COMPAT:-none})" + + - name: Upsert the record issue + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TITLE: "Upstream compatibility — agent0ai/agent-zero:latest" + BODY_FILE: record.md + LABEL: upstream-compat + MODE: only-on-failure + STATUS: ${{ steps.rec.outputs.status }} + run: bash e2e/ci/upsert-record-issue.sh + + - name: Record the tested digest + # Written only on a GREEN full run, so a red night is retried tomorrow + # instead of being cached away as "already checked". + if: needs.compat.result == 'success' + run: | + printf 'digest=%s\nts=%s\n' "${{ needs.probe.outputs.digest }}" "$(date +%s)" \ + > .upstream-compat-state + + - name: Save digest cache + if: needs.compat.result == 'success' + uses: actions/cache/save@v4 + with: + path: .upstream-compat-state + key: upstream-compat-${{ needs.probe.outputs.digest }} + + - name: Fail the run when a plugin is broken + # The nightly's own red X is the primary signal — Actions emails the + # workflow owner on a scheduled-run failure, which is the "signal someone + # sees" requirement. The issue is the durable second channel. + if: steps.rec.outputs.status == 'fail' + run: | + echo "::error::one or more plugins fail against the latest upstream A0" + exit 1 diff --git a/SPEC.md b/SPEC.md index 8e225d8..25450b3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1255,8 +1255,30 @@ job disappeared when the devkit went public. The real cost is **integrity and bl unreviewed merge changes CI in 24 repos with no canary and no rollback boundary. Fixed as change management, not as a breach. +**DEC-082 — Upstream A0 compatibility is tested, because upstream is the only consumer.** +The reusable workflow defaults `a0_image` to the private fork +(`ghcr.io/nuevanext/agent-zero:latest-nonroot`) and **none of the four plugins overrode it**, so every +PR proved the plugin worked on the fork and nothing anywhere proved it worked on upstream — which is +the only thing anyone actually runs. That is not a gap in coverage so much as coverage pointed at the +wrong target. +Two changes, deliberately separate. Each plugin sets `a0_image` to +`docker.io/agent0ai/agent-zero:latest` in its own `.devkit.yml`, so the **PR gate** now tests what +customers run — a break is caught before merge, by the author, with the diff in front of them. And a +scheduled `upstream-compat` workflow re-runs the same suites when **upstream** moves, which no PR can +catch because nothing in the repo changed. +*Why this schedule is not the one that was just deleted (DEC-079).* That one opened a PR every night +whether or not anything had changed, and failed silently behind an approval gate for three weeks. This +one is gated on the upstream manifest **digest** actually moving — measured, upstream ships ~1.5×/week +against a 42-minute fan-out, so digest-gating costs ~295 runner-min/month instead of ~1,275 for the +same ≤24h detection. It opens no PRs, commits nothing, and reports failure as a single issue that +auto-closes when green, so an open issue always means act. Three structural guards keep it honest: the +probe hard-fails rather than shrugging, an empty matrix is an error rather than a vacuous pass, and the +report job asserts the matrix ran if the probe asked for it. +The runs pin the resolved **digest**, not `:latest`, so a failure is reproducible against the exact +image that broke it rather than whatever `:latest` points at by the time someone investigates. + > **Numbering note:** DEC-070–072 were originally introduced as code comments only. They are now -> written up above, so this section documents an unbroken 070–081. The lesson is worth keeping: a +> written up above, so this section documents an unbroken 070–082. The lesson is worth keeping: a > decision that exists only as a code comment is invisible to anyone reading the SPEC, and all three > of these were load-bearing (a gate's timeout budget, the image the whole fleet pulls, and a wait > that was flaking half the runs). diff --git a/docs/upstream-compat-and-ci-budget.md b/docs/upstream-compat-and-ci-budget.md new file mode 100644 index 0000000..15d9701 --- /dev/null +++ b/docs/upstream-compat-and-ci-budget.md @@ -0,0 +1,335 @@ +# CI budget + nightly upstream-compat — design + +Two capabilities for the devkit, built against +`agent-zero-plugin-development-testkit` and its four consumers +(`chat-comments`, `diff-visualizer`, `mermaid-diagrams`, `share-chat`). + +Everything below that states an API behaviour was **probed live on 2026-08-05**, +not assumed. Where an endpoint does not give us what the operator asked for, that +is said plainly rather than papered over. + +--- + +## Part 0 — what the GitHub API actually gives us + +I probed four candidate sources before designing anything. Three of the four are +either dead or useless here. + +| # | Endpoint | Result | Verdict | +|---|---|---|---| +| 1 | `GET /orgs/{org}/settings/billing/actions` | **HTTP 410** "This endpoint has been moved." (`gh` also reports it wants `admin:org`) | Dead. | +| 2 | `GET /orgs/{org}/settings/billing/shared-storage` | **HTTP 410**, same | Dead. | +| 3 | `GET /repos/{r}/actions/runs/{id}/timing` | Reachable with a plain repo `GITHUB_TOKEN`, but returns **`billable.UBUNTU.total_ms = 0`** on every run | **Useless here** — see below. | +| 4 | `GET /organizations/{org}/settings/billing/usage?year=&month=` | **Works.** Real per-repo, per-day minutes | The only true billing source. | + +### 3 is a trap, and it is the obvious thing to reach for + +`/timing` looks exactly like the answer. It is not. Three real +`chat-comments` runs returned: + +```json +{"billable":{"UBUNTU":{"total_ms":0,"jobs":2, + "job_runs":[{"job_id":92153588155,"duration_ms":0}, + {"job_id":92156121906,"duration_ms":0}]}}, + "run_duration_ms":810000} +``` + +All four consumer repos are **public**, and public-repo Actions minutes are free, +so GitHub reports **zero billable milliseconds**. A budget built on `/timing` +would report `0 min` for every plugin forever and look like it worked. The design +therefore *deliberately does not use it*, and the generated record says so on its +face so nobody "fixes" it back later. + +`run_duration_ms` in the same payload is real, but it is whole-run wall clock +including queue and idle gaps between jobs — not a cost figure. + +### 4 works, with one structural limitation that shapes the whole design + +``` +GET /organizations/agent-zero-plugins/settings/billing/usage?year=2026&month=8 +``` + +returns one row per (day, repo, SKU) — 191 rows for August, covering **27 repos +including all four consumers**: + +```json +{"date":"2026-08-04T00:00:00Z","product":"actions","sku":"Actions Linux", + "quantity":78.0,"unitType":"Minutes","netAmount":0.0, + "repositoryName":"agent-zero-plugin-chat-comments"} +``` + +So **minutes are real and per-repo**. Two caveats worth stating: + +- **The unfiltered call lies by omission.** With no `year`/`month` it returns + only **8 aggregate rows** covering 5 repos. With `?year=&month=` it returns + **191**. Anyone probing this endpoint casually would conclude our plugins have + no CI usage. They do. +- **`read:org` was sufficient** on the operator PAT — this one does *not* need + `admin:org`, unlike the two dead endpoints. + +**The limitation that drives the architecture:** a workflow's `GITHUB_TOKEN` is +repo-scoped. There is no `permissions:` key that grants org billing — the +capability does not exist in the workflow token model at all. So **tier 3 is +unreachable from inside a plugin repo's own CI unless an org PAT is supplied as +a secret.** I did not work around this, because the available workarounds all +involve inventing a number. + +### What we use instead, and what it is honestly worth + +`GET /repos/{r}/actions/runs/{id}/jobs` gives per-job `started_at` / +`completed_at` on any token. Summing `(completed − started)` across jobs is a +**runner-wall-clock proxy** — always available, no secret. Since GitHub bills +per job rounded up to the whole minute, it tracks the real figure closely but is +not identical to it. + +**It is not the same measurement as tier 3, and the record never implies it is.** +Measured live on `chat_comments` over the same 30 days: + +| | value | scope | +|---|---|---| +| billed (tier 3) | **454.0 min** | whole repo, *every* workflow, rounded up per job | +| observed (tier 4) | **199.8 min** | `plugin-e2e` only | + +The 2.3× gap is not error — it is `devkit-sync` + `unit` + Dependabot + rounding. +An early draft of the record showed these as adjacent bare numbers, which reads +as "one of these is wrong". The shipped version labels each with its scope and +carries a one-line note explaining the difference. **Use billed for the invoice, +observed for what the e2e suite costs.** + +One more honesty note: `netAmount` was **not** always `$0.00` despite the repos +being public (`chat_comments` 30-day net was **$1.60**, `share_chat` **$0.16**). +The naive "public means free" assumption does not hold in this org's billing +data. The record reports whatever the API returns and does not assert the repos +are free. + +--- + +## Part A — CI budget / e2e record per plugin + +### Where the record lives, and why + +The operator asked for a record "on every one of the plugin records", showing CI +minutes, when e2e last ran, and its result. Options considered: + +| Option | Durable | Zero churn | Verdict | +|---|---|---|---| +| Committed `docs/CI-BUDGET.md` | yes | **no** — a commit (or PR) per run whose only delta is a timestamp | **rejected** | +| Job summary only | **no** — dies with the run | yes | insufficient alone | +| README badge | yes | no (same commit churn), and cannot show minutes | insufficient | +| shields.io endpoint badge | yes | yes | needs external JSON hosting we don't have | +| **Long-lived issue, body upserted in place** | **yes** | **yes** | **chosen** | + +**Chosen: all three tiers of output, with the issue as the durable one.** + +1. **Job summary** on every invocation — free, immediate, on the run page. +2. **`ci-budget.json` artifact** — machine-readable, so a future fleet roll-up + has something to consume without re-querying. +3. **One long-lived issue per repo**, titled `CI budget — `, label + `ci-budget`, whose **body is replaced in place**. + +Why the issue wins: + +- **Zero commits, zero PRs, zero branches.** This is the hard constraint after + the devkit-sync deletion. A committed markdown file fails it outright. +- **Editing an issue body sends no notification**, so it can refresh as often as + we like without training anyone to filter it out. +- **Durable and permalinked**, unlike a job summary that vanishes with the run's + retention. +- It is already where maintainers look, and it is pinnable per repo. + +### Staleness is a failure, not a footnote + +`ci-budget.py` **exits 2** when there is no completed e2e run in the window, and +the record renders `⚠️ never` rather than a blank. This is the same defect class +as devkit-sync's silent three-week no-op: a record that quietly says nothing is +worse than no record. `fail-on-stale` (default `false`) promotes it to a red +job once the fleet's cadence is steady. + +### Cadence + +Weekly (`cron: 40 6 * * 1`), not per-PR. It is a rolling 30-day view — running +it per-PR would spend runner time to change a timestamp. Also +`workflow_dispatch`-able for an on-demand read. The job itself is a handful of +API calls, ~20s, no containers. + +### Files + +- `scripts/ci-budget.py` → `e2e/scripts/ci-budget.py` +- `scripts/upsert-record-issue.sh` → `e2e/ci/upsert-record-issue.sh` +- `workflows/ci-budget.yml` → `.github/workflows/ci-budget.yml` (reusable) +- `templates/ci-budget.caller.yml` → `templates/`, distributed by + `make link-workflows` + +`BILLING_READ_TOKEN` is an **optional** secret. Absent, the record is fully +useful and states in words that billed minutes are unavailable and why. Present, +tier 3 lights up. Both paths are live-verified below. + +--- + +## Part B — nightly upstream-compat + +### The gap it closes (verified, not assumed) + +All four consumers' `.devkit.yml` files were read directly. **None sets +`a0_image`**, so all four inherit the reusable workflow's default — +`ghcr.io/nuevanext/agent-zero:latest-nonroot`, the private fork. The fleet's e2e +today proves *"works on the fork"* and says **nothing** about *"still works on +upstream"*. + +The machinery genuinely does mostly exist: `plugin-e2e.yml` accepts `a0_image`, +and `sample-plugin-e2e.yml` already boots `docker.io/agent0ai/agent-zero:latest`. +What is missing is the scheduled fan-out and a place the result is seen. That is +all this adds. + +### How it differs from the deleted devkit-sync nightly + +devkit-sync burned 189 runs for 29 merged bumps, then silently no-op'd for three +weeks. Three concrete structural differences: + +**1. Change-gated, not calendar-gated.** +A ~20s `probe` job resolves the upstream **manifest digest** anonymously from +Docker Hub and only fans out when the digest actually moved since the last green +full run. State lives in `actions/cache` keyed by digest — no commits, no state +file, no PR. Live-verified: + +``` +$ bash scripts/upstream-digest.sh docker.io/agent0ai/agent-zero:latest +sha256:8c5eff81a46fc956bc247e01a345beadc09acbeecf31873324b843dacf937c4c +``` + +The probe still runs **every night**, so detection latency is unchanged (≤24h). +Only the expensive part is gated. + +**2. It cannot silently no-op.** Four independent guards: +- the probe **hard-fails** if it cannot resolve a digest (a shrug-and-exit-0 + probe is exactly how devkit-sync died unnoticed); +- an **empty matrix is a hard error**, not a vacuous pass; +- a **`MAX_SKIP_DAYS: 7` floor** forces a full run even if upstream never moves; +- the `report` job asserts that *if the probe requested a fan-out, the matrix + actually produced a result* — otherwise it fails. +- Every outcome stamps a `last checked` timestamp into the record, so staleness + is visible on the artifact itself. + +**3. Zero PR churn.** It never opens a PR and never commits. Its only write is +**one** long-lived issue in `MODE=only-on-failure`: opened when a plugin breaks +against upstream, **closed automatically** when the fleet is green again. An open +issue always means "act"; there is no steady-state noise to learn to ignore. + +### Signal someone sees + +Primary: the **scheduled workflow's own red X**, which GitHub emails to the +workflow owner on a scheduled-run failure. Secondary and durable: the +auto-opening record issue. Tertiary: per-leg Playwright traces uploaded +`if: failure()` only — a green nightly uploading four full trace sets every week +is storage the operator pays for and nobody opens. + +### Cost/benefit — the honest numbers + +Real per-plugin e2e durations, measured from the most recent green run of each: + +| plugin | e2e job | +|---|---| +| chat-comments | 12m 39s | +| diff-visualizer | 13m 02s | +| mermaid-diagrams | 11m 00s | +| share-chat | 5m 49s | +| **one full fan-out** | **≈ 42 min** | + +Upstream release cadence, from Docker Hub tag timestamps (`v2.8` 08-01, `v2.7` +07-27, `v2.6` 07-23, `v2.5` 07-17, `v2.4` 07-10, `v2.3` 07-09, `v2.2` 07-02, +`v2.1` 06-26, `v2.0` 06-24) — **≈ 9 releases in 6 weeks ≈ 1.5/week ≈ 6.5/month.** + +| strategy | runner-min / month | +|---|---| +| naive calendar nightly (30 × 42) | **≈ 1,275** | +| digest-gated (≈7 full runs + 30 probes) | **≈ 295 + ~10** | + +**≈ 76% cheaper for identical detection latency.** Because upstream ships more +often than weekly, the `MAX_SKIP_DAYS` floor will rarely fire — it is insurance +against upstream going quiet, not a routine cost. + +**The honest caveat:** at ~1.5 releases/week the digest changes often enough that +this is *not* nearly-free — it is roughly 295 min/month, ~23% of a naive nightly +but still a real recurring spend. Whether that is worth it depends on one +question the operator should answer, not me: **do these plugins need to work on +upstream A0 at all, or only on the fork they are deployed against?** If only the +fork, this workflow is ~295 min/month of information nobody acts on and should +not be adopted. If upstream compatibility is a shipped promise, this is the +cheapest way to hold it, and the current state — where *nothing* tests upstream — +means the promise is untested today. + +A cheaper variant if the answer is "nice to have": change `MAX_SKIP_DAYS` to 14 +and add a `plugins` subset so only the two most upstream-coupled plugins run per +digest. I did not default to that because a partial check that silently omits +plugins is the ambiguity this whole design is trying to remove. + +### `secrets` in `if:` — avoided by construction + +Per the explicit warning: the `secrets` context is **not** permitted in an `if:` +expression; using it invalidates the entire workflow file and the run dies as +`startup_failure` with zero jobs. `ci-budget.yml` therefore probes +`BILLING_READ_TOKEN` in a **`run:` step** (where secrets *are* available via +`env:`) and branches on `steps.creds.outputs.have_billing`. This mirrors the +existing pattern already proven in `plugin-e2e.yml`'s "Detect whether App +credentials are available" step. `upstream-compat.yml` uses no secrets at all — +the upstream image is public and the digest probe is anonymous. + +--- + +## Validation performed + +``` +YAML (python3 -c "import yaml; yaml.safe_load(open(...))") + OK workflows/ci-budget.yml + OK workflows/upstream-compat.yml + OK templates/ci-budget.caller.yml + +bash -n + OK scripts/upsert-record-issue.sh + OK scripts/upstream-digest.sh + +shellcheck CLEAN (both scripts, zero findings) +python3 -m py_compile OK scripts/ci-budget.py +``` + +**Live end-to-end runs against the real API** (not mocks): + +- `upstream-digest.sh` → resolved the real Docker Hub digest anonymously. +- `ci-budget.py` **without** a billing token (simulating `GITHUB_TOKEN`) → + rendered a complete record, billed minutes marked *not available* with the + reason. rc=0. +- `ci-budget.py` **with** the org token → tier 3 lit up: `454.0 min`, + `net $1.60`, `20.768 GB-hours`. rc=0. +- `ci-budget.py` against `share-chat` → `105.0 min` / `54.3 min` observed. rc=0. +- **Stale path**: asked for a workflow that has never run → rendered + `⚠️ never — no completed upstream-compat run in the last 30 days` and + **exited 2**, as designed. + +### Not validated + +The two workflows have **not been executed on GitHub Actions** — that needs a +commit to a repo, which this task explicitly forbids. YAML validity, script +correctness and every API call they make are verified; job wiring +(`needs`/`outputs`/matrix `fromJSON`) is reviewed but not run. The nested-A0 leg +in `upstream-compat.yml` is adapted from the proven `plugin-e2e.yml` body with +one real change — the plugin is checked out into `plugin/` and mounted at +`/plugin`, because this workflow runs in the *devkit's* workspace rather than the +consumer's. That path change is the highest-risk untested detail and should be +the first thing a trial `workflow_dispatch` exercises. + +--- + +## Install map + +| built here | goes to | +|---|---| +| `scripts/ci-budget.py` | devkit `e2e/scripts/ci-budget.py` | +| `scripts/upsert-record-issue.sh` | devkit `e2e/ci/upsert-record-issue.sh` | +| `scripts/upstream-digest.sh` | devkit `e2e/ci/upstream-digest.sh` | +| `workflows/ci-budget.yml` | devkit `.github/workflows/ci-budget.yml` | +| `workflows/upstream-compat.yml` | devkit `.github/workflows/upstream-compat.yml` | +| `templates/ci-budget.caller.yml` | devkit `templates/` + one line in `link-workflows` | + +`upstream-compat.yml` lives **only** in the devkit and needs no consumer change — +it checks the plugin repos out itself. Only `ci-budget` needs a caller +distributed to the four consumers. diff --git a/e2e/ci/ci-budget.py b/e2e/ci/ci-budget.py new file mode 100755 index 0000000..ce4534c --- /dev/null +++ b/e2e/ci/ci-budget.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +Build the per-plugin CI budget / e2e record. + +WHY THIS IS A SCRIPT AND NOT INLINE YAML + Same reason as e2e/ci/prune-artifacts.sh: it is called from the reusable + ci-budget.yml AND from the nightly upstream-compat.yml, and it is the kind + of logic (pagination, unit conversion, graceful degradation) that rots + instantly when copy-pasted into two workflows. + +WHAT THE GITHUB API ACTUALLY GIVES US (probed 2026-08-05, not assumed) + + 1. repos/{r}/actions/runs/{id}/timing → billable.UBUNTU.total_ms + Reachable with the workflow's own GITHUB_TOKEN (actions:read). + USELESS HERE: it returns 0 for every run on these repos, because they are + PUBLIC and public-repo Actions minutes are free, so GitHub reports zero + *billable* time. Verified on three real chat-comments runs: + {"billable":{"UBUNTU":{"total_ms":0,...}},"run_duration_ms":810000} + We therefore do NOT use it as the minute source. run_duration_ms is real + but is whole-run wall clock including queue/idle between jobs. + + 2. GET /orgs/{org}/settings/billing/actions → HTTP 410 "moved" + GET /orgs/{org}/settings/billing/shared-storage → HTTP 410 "moved" + Both are gone. gh additionally reports they'd want admin:org. Dead ends. + + 3. GET /organizations/{org}/settings/billing/usage?year=&month= + THE one that works. Needs an ORG-scoped token (read:org was enough for + the operator PAT; the endpoint is on the "enhanced billing platform"). + Returns one row per (day, repo, SKU): + {"date":"2026-08-04...","product":"actions","sku":"Actions Linux", + "quantity":78.0,"unitType":"Minutes","netAmount":0.0, + "repositoryName":"agent-zero-plugin-chat-comments"} + So MINUTES ARE REAL AND PER-REPO. netAmount is 0.0 because public repos + are free — the *cost* is genuinely zero, the *minutes* are not. + LIMITATION: a workflow's GITHUB_TOKEN is repo-scoped and has no billing + permission at all (there is no `permissions:` key that grants it), so this + tier only lights up when an org PAT is provided as a secret. Without it we + say so in the record rather than inventing a number. + + 4. repos/{r}/actions/runs/{id}/jobs → per-job started_at/completed_at. + Always reachable, no secret. Summing (completed-started) over jobs is our + ALWAYS-AVAILABLE minute proxy. It is runner wall clock, which is what + GitHub rounds up per job to bill, so it tracks tier-3 closely — it is a + proxy, and the record labels it as one. + +Output (stdout is the markdown record; --json-out writes the machine copy): + minutes_billed from (3) when available, else null + a stated reason + minutes_observed from (4), always + last e2e run: when, conclusion, duration, commit, link +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +API = "https://api.github.com" + + +def _get(url: str, token: str) -> object: + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "a0-devkit-ci-budget", + }, + ) + with urllib.request.urlopen(req, timeout=60) as r: + return json.load(r) + + +def _iso(s: str | None) -> datetime | None: + if not s: + return None + return datetime.fromisoformat(s.replace("Z", "+00:00")) + + +def _fmt_dur(seconds: float) -> str: + m, s = divmod(int(seconds), 60) + return f"{m}m {s:02d}s" + + +def _age(dt: datetime | None, now: datetime) -> str: + if dt is None: + return "never" + d = now - dt + if d < timedelta(hours=1): + return f"{int(d.total_seconds() // 60)}m ago" + if d < timedelta(days=1): + return f"{int(d.total_seconds() // 3600)}h ago" + return f"{d.days}d ago" + + +# ---------------------------------------------------------------- runs / jobs + +def collect_runs(repo: str, token: str, workflow: str, days: int) -> list[dict]: + """Runs of `workflow` in the last `days`. Repo-scoped: always available.""" + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + out: list[dict] = [] + page = 1 + while page <= 10: # hard cap: 1000 runs is far more than any window needs + url = ( + f"{API}/repos/{repo}/actions/runs" + f"?per_page=100&page={page}&exclude_pull_requests=false" + ) + try: + data = _get(url, token) + except urllib.error.HTTPError as e: # pragma: no cover - network + print(f"::warning::runs query failed: {e}", file=sys.stderr) + break + runs = data.get("workflow_runs", []) # type: ignore[union-attr] + if not runs: + break + stop = False + for r in runs: + created = _iso(r.get("created_at")) + if created and created < cutoff: + stop = True + continue + if workflow and r.get("name") != workflow: + continue + out.append(r) + if stop: + break + page += 1 + return out + + +def job_seconds(repo: str, run_id: int, token: str) -> float: + """Summed per-job runner wall clock. Our always-available minute proxy.""" + try: + data = _get(f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", token) + except urllib.error.HTTPError: # pragma: no cover - network + return 0.0 + total = 0.0 + for j in data.get("jobs", []): # type: ignore[union-attr] + a, b = _iso(j.get("started_at")), _iso(j.get("completed_at")) + if a and b and b > a: + total += (b - a).total_seconds() + return total + + +# ------------------------------------------------------------------- billing + +def billed_minutes(org: str, repo_name: str, token: str | None, days: int) -> dict: + """ + Tier-3 authoritative minutes from the enhanced-billing usage endpoint. + + Returns {"available": False, "reason": ...} when we have no org token, which + is the DEFAULT for a plugin repo running on GITHUB_TOKEN. We never guess. + """ + if not token: + return { + "available": False, + "reason": ( + "no org-scoped billing token supplied " + "(GITHUB_TOKEN is repo-scoped and cannot read org billing)" + ), + } + now = datetime.now(timezone.utc) + cutoff = now - timedelta(days=days) + months = {(now.year, now.month), (cutoff.year, cutoff.month)} + minutes = 0.0 + net = 0.0 + gross = 0.0 + gb_hours = 0.0 + seen = False + for year, month in sorted(months): + url = ( + f"{API}/organizations/{org}/settings/billing/usage" + f"?year={year}&month={month}" + ) + try: + data = _get(url, token) + except urllib.error.HTTPError as e: + return { + "available": False, + "reason": f"billing usage endpoint returned HTTP {e.code}", + } + for it in data.get("usageItems", []): # type: ignore[union-attr] + if it.get("repositoryName") != repo_name: + continue + d = _iso(it.get("date")) + if d and d < cutoff: + continue + seen = True + if it.get("unitType") == "Minutes": + minutes += float(it.get("quantity") or 0) + net += float(it.get("netAmount") or 0) + gross += float(it.get("grossAmount") or 0) + elif it.get("unitType") == "GigabyteHours": + gb_hours += float(it.get("quantity") or 0) + return { + "available": True, + "minutes": round(minutes, 1), + "storage_gb_hours": round(gb_hours, 3), + "net_usd": round(net, 4), + "gross_usd": round(gross, 4), + "note": ( + "netAmount is $0.00 because this repo is PUBLIC and public-repo " + "Actions minutes are free. The minutes are real; the bill is not." + ) + if seen and net == 0 + else "", + } + + +# -------------------------------------------------------------------- record + +def build_record(args: argparse.Namespace) -> dict: + now = datetime.now(timezone.utc) + org = args.repo.split("/")[0] + repo_name = args.repo.split("/")[1] + + runs = collect_runs(args.repo, args.token, args.workflow, args.days) + finished = [r for r in runs if r.get("status") == "completed"] + + observed = 0.0 + # Cost control: per-run /jobs is one API call each. Cap how many we walk; + # beyond the cap we report what we measured and say the window was capped. + walked = finished[: args.max_run_lookups] + for r in walked: + observed += job_seconds(args.repo, r["id"], args.token) + + last = finished[0] if finished else None + last_seconds = job_seconds(args.repo, last["id"], args.token) if last else 0.0 + + passes = sum(1 for r in finished if r.get("conclusion") == "success") + fails = sum(1 for r in finished if r.get("conclusion") == "failure") + + billing = billed_minutes(org, repo_name, args.billing_token, args.days) + + return { + "schema": "a0-devkit/ci-budget@1", + "generated_at": now.isoformat(), + "repo": args.repo, + "plugin": args.plugin, + "workflow": args.workflow, + "window_days": args.days, + "runs_total": len(finished), + "runs_measured": len(walked), + "runs_capped": len(finished) > len(walked), + "runs_success": passes, + "runs_failure": fails, + "minutes_observed": round(observed / 60, 1), + "minutes_billed": billing, + "last_run": None + if not last + else { + "id": last["id"], + "conclusion": last.get("conclusion"), + "event": last.get("event"), + "created_at": last.get("created_at"), + "age": _age(_iso(last.get("created_at")), now), + "duration": _fmt_dur(last_seconds), + "duration_seconds": round(last_seconds), + "sha": (last.get("head_sha") or "")[:8], + "url": last.get("html_url"), + }, + } + + +def render(rec: dict) -> str: + ico = {"success": "✅", "failure": "❌", "cancelled": "⚪", "timed_out": "⏱️"} + last = rec["last_run"] + b = rec["minutes_billed"] + + if b["available"]: + billed = f"**{b['minutes']} min** (net ${b['net_usd']:.2f})" + billed_note = b.get("note") or "" + else: + billed = "_not available_" + billed_note = b["reason"] + + # These two numbers are NOT the same measurement and must never be shown as + # if one validated the other. Measured live on chat_comments: 454 billed min + # vs 199.8 observed min over the same 30 days. The gap is not error — billing + # is REPO-WIDE (devkit-sync, unit, Dependabot, every workflow) and rounds each + # job up to a whole minute, while `observed` is filtered to one workflow. + scope_note = ( + f"Scopes differ by design: **billed** is repo-wide across *all* workflows " + f"and rounds each job up to a whole minute; **observed** is `{rec['workflow']}` " + f"only. Use billed for the invoice, observed for what the e2e suite costs." + ) + + lines = [ + f"## CI budget — `{rec['plugin']}`", + "", + f"Window: last **{rec['window_days']} days** · workflow `{rec['workflow']}` " + f"· generated {rec['generated_at'][:16].replace('T', ' ')} UTC", + "", + "| | |", + "|---|---|", + ] + if last: + lines += [ + f"| **e2e last ran** | [{last['created_at'][:16].replace('T', ' ')} UTC]({last['url']}) " + f"({last['age']}) |", + f"| **result** | {ico.get(last['conclusion'], '❔')} `{last['conclusion']}` " + f"on `{last['sha']}` (via `{last['event']}`) |", + f"| **that run took** | {last['duration']} |", + ] + else: + lines += [ + f"| **e2e last ran** | ⚠️ **never** — no completed `{rec['workflow']}` run " + f"in the last {rec['window_days']} days |", + "| **result** | — |", + "| **that run took** | — |", + ] + lines += [ + f"| **runs in window** | {rec['runs_total']} " + f"({rec['runs_success']} ✅ / {rec['runs_failure']} ❌) |", + f"| **minutes billed** _(whole repo, all workflows)_ | {billed} |", + f"| **minutes observed** _(`{rec['workflow']}` only, runner wall clock)_ | " + f"{rec['minutes_observed']} min over {rec['runs_measured']} run(s)" + + (" _(window capped)_" if rec["runs_capped"] else "") + + " |", + ] + if b["available"] and b.get("storage_gb_hours"): + lines.append( + f"| **artifact storage** _(whole repo)_ | {b['storage_gb_hours']} GB-hours |" + ) + lines += ["", f"> {scope_note}"] + if billed_note: + lines += ["", f"> {billed_note}"] + lines += [ + "", + "`minutes (observed)` is summed per-job runner wall clock from the " + "Actions API — a proxy, always available. `minutes (billed)` is the " + "authoritative org billing figure and only appears when an org-scoped " + "token is configured. The per-run `/timing` endpoint is deliberately " + "unused: it reports 0 billable ms on public repos.", + ] + return "\n".join(lines) + + +def main() -> int: + p = argparse.ArgumentParser(description="Generate the plugin CI budget record.") + p.add_argument("--repo", required=True, help="owner/name") + p.add_argument("--plugin", required=True) + p.add_argument("--workflow", default="plugin-e2e", help="workflow NAME to budget") + p.add_argument("--days", type=int, default=30) + p.add_argument("--max-run-lookups", type=int, default=40) + p.add_argument("--json-out", default="") + p.add_argument("--md-out", default="") + p.add_argument("--token", default=os.environ.get("GITHUB_TOKEN", "")) + p.add_argument( + "--billing-token", + default=os.environ.get("BILLING_TOKEN", "") or None, + help="org-scoped PAT for /organizations/{org}/settings/billing/usage", + ) + args = p.parse_args() + if not args.token: + print("::error::no GITHUB_TOKEN", file=sys.stderr) + return 1 + + rec = build_record(args) + md = render(rec) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as fh: + json.dump(rec, fh, indent=2) + fh.write("\n") + if args.md_out: + with open(args.md_out, "w", encoding="utf-8") as fh: + fh.write(md + "\n") + print(md) + + # Staleness is a FAILURE MODE, not a footnote. If the operator asked for + # "when last they ran", a record that quietly says "never" is the devkit-sync + # bug again. Exit non-zero so a human sees it. + if args.days and rec["last_run"] is None: + print( + f"::error::no completed {args.workflow} run in {args.days} days " + f"for {args.repo}", + file=sys.stderr, + ) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/e2e/ci/upsert-record-issue.sh b/e2e/ci/upsert-record-issue.sh new file mode 100755 index 0000000..99a9ee0 --- /dev/null +++ b/e2e/ci/upsert-record-issue.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Upsert a single long-lived GitHub issue whose BODY is the record. +# +# WHY AN ISSUE AND NOT A COMMITTED FILE +# A committed docs/CI-BUDGET.md would be rewritten on every run, so every e2e +# run would produce a commit (or a PR) whose only content is a changed +# timestamp. That is precisely the churn that got the nightly devkit-sync +# deleted — 189 runs, 29 useful bumps. An issue body is: +# * durable and permalinked (unlike a job summary, which dies with the run), +# * editable in place with zero commits, zero PRs, zero branch noise, +# * already where maintainers look, and pinnable per repo, +# * silent on edit (editing a body sends no notification), so it can update +# as often as we like without training anyone to ignore it. +# +# ANTI-STALENESS +# MODE=only-on-failure closes the issue when everything is green and (re)opens +# it when something is red — so an OPEN issue always means "act on this", and +# its absence is not ambiguous. The nightly's own last-run timestamp is written +# into the body either way, so "it silently stopped running" is visible rather +# than invisible. +# +# Env: +# GH_TOKEN token with issues:write on $REPO +# REPO owner/name +# TITLE exact issue title used as the upsert key +# BODY_FILE file whose contents become the issue body +# LABEL label to apply/create (default: ci-budget) +# MODE always | only-on-failure (default: always) +# STATUS ok | fail — consulted only when MODE=only-on-failure +# +# Never fails the caller: record-keeping must not turn a green run red. +set -euo pipefail + +: "${REPO:?set REPO}" +: "${TITLE:?set TITLE}" +: "${BODY_FILE:?set BODY_FILE}" +LABEL="${LABEL:-ci-budget}" +MODE="${MODE:-always}" +STATUS="${STATUS:-ok}" + +[ -f "$BODY_FILE" ] || { echo "::warning::$BODY_FILE missing — nothing to upsert"; exit 0; } + +# Ensure the label exists (idempotent; ignore "already exists"). +gh label create "$LABEL" --repo "$REPO" \ + --color 0e8a16 --description "Automated devkit record — do not close by hand" \ + >/dev/null 2>&1 || true + +# Find the existing record issue by exact title, open OR closed, so we reuse the +# same one forever instead of accumulating a new issue per run. +num=$(gh issue list --repo "$REPO" --label "$LABEL" --state all --limit 50 \ + --json number,title \ + --jq "[.[] | select(.title == \"$TITLE\")] | first | .number" 2>/dev/null || echo "") +[ "$num" = "null" ] && num="" + +if [ -z "$num" ]; then + if [ "$MODE" = "only-on-failure" ] && [ "$STATUS" = "ok" ]; then + echo "all green and no existing record issue — nothing to open" + exit 0 + fi + num=$(gh issue create --repo "$REPO" --title "$TITLE" \ + --body-file "$BODY_FILE" --label "$LABEL" \ + --json number --jq .number 2>/dev/null \ + || gh issue create --repo "$REPO" --title "$TITLE" \ + --body-file "$BODY_FILE" --label "$LABEL" | grep -oE '[0-9]+$') + echo "created record issue #$num" +else + gh issue edit "$num" --repo "$REPO" --body-file "$BODY_FILE" >/dev/null + echo "updated record issue #$num (body replaced, no comment, no notification)" +fi + +if [ "$MODE" = "only-on-failure" ]; then + if [ "$STATUS" = "ok" ]; then + gh issue close "$num" --repo "$REPO" \ + --comment "All plugins green against upstream — closing. This issue reopens automatically on the next red night." \ + >/dev/null 2>&1 || true + echo "green → record issue #$num closed" + else + gh issue reopen "$num" --repo "$REPO" >/dev/null 2>&1 || true + echo "red → record issue #$num open" + fi +fi + +echo "record: https://github.com/$REPO/issues/$num" diff --git a/e2e/ci/upstream-digest.sh b/e2e/ci/upstream-digest.sh new file mode 100755 index 0000000..7c5431e --- /dev/null +++ b/e2e/ci/upstream-digest.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Resolve the current manifest digest of an image on Docker Hub, anonymously. +# +# WHY THIS EXISTS — it is the whole cost argument for the nightly. +# A calendar-driven nightly re-runs 4 × ~14min e2e every night whether or not +# upstream changed. docker.io/agent0ai/agent-zero:latest does NOT change +# nightly. Gating the expensive fan-out on "did the digest actually move" +# turns ~1,700 runner-min/month into roughly the number of upstream releases +# × 56 min, plus a ~20s probe every night. Same detection latency, ~10% of the +# spend, and — unlike a calendar trigger — every expensive run corresponds to a +# real upstream change, so a red result is never ambiguous. +# +# Uses the anonymous pull token, so no Docker credentials are needed and this +# works on a fork PR. Prints the digest on stdout; writes GH outputs if +# GITHUB_OUTPUT is set. +# +# Env/args: +# IMAGE default docker.io/agent0ai/agent-zero:latest +set -euo pipefail + +IMAGE="${1:-${IMAGE:-docker.io/agent0ai/agent-zero:latest}}" + +ref="${IMAGE#docker.io/}" +repo="${ref%:*}" +tag="${ref##*:}" +[ "$tag" = "$repo" ] && tag=latest +case "$repo" in */*) ;; *) repo="library/$repo" ;; esac + +token=$(curl -fsS "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') + +digest=$(curl -fsSI -H "Authorization: Bearer ${token}" \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:"{print $2}') + +if [ -z "${digest:-}" ]; then + echo "::error::could not resolve digest for ${IMAGE}" >&2 + exit 1 +fi + +echo "$digest" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + { + echo "digest=${digest}" + echo "digest_short=${digest#sha256:}" + echo "image=${IMAGE}" + } >> "$GITHUB_OUTPUT" +fi diff --git a/templates/ci-budget.caller.yml b/templates/ci-budget.caller.yml new file mode 100644 index 0000000..937b10d --- /dev/null +++ b/templates/ci-budget.caller.yml @@ -0,0 +1,37 @@ +# ci-budget — caller template. +# +# Copied into a consumer plugin repo at .github/workflows/ci-budget.yml by +# `make link-workflows`. Refreshes the plugin's CI budget / e2e record: +# a job summary, a ci-budget.json artifact, and one long-lived record issue +# updated in place (no commits, no PRs). +# +# Weekly, not per-PR: the record is a rolling 30-day view, so refreshing it on +# every PR would spend runner time to change a timestamp. It is also +# workflow_dispatch-able for an on-demand read. +name: ci-budget + +on: + schedule: + # Mondays 06:40 UTC. Off the hour (GitHub's :00 queues are the worst) and + # early enough that the record is fresh when the week starts. + - cron: "40 6 * * 1" + workflow_dispatch: + inputs: + window_days: + description: "Rolling window in days" + type: string + default: "30" + +permissions: + contents: read + actions: read + issues: write + +jobs: + budget: + uses: agent-zero-plugins/agent-zero-plugin-development-testkit/.github/workflows/ci-budget.yml@main + # Forwards BILLING_READ_TOKEN if the org has set it. Absent → the record + # honestly reports billed minutes as unavailable rather than guessing. + secrets: inherit + with: + window-days: ${{ fromJSON(inputs.window_days || '30') }}