diff --git a/.github/workflows/ci-budget.yml b/.github/workflows/ci-budget.yml new file mode 100644 index 0000000..d16cf6d --- /dev/null +++ b/.github/workflows/ci-budget.yml @@ -0,0 +1,178 @@ +# Reusable ci-budget workflow — the per-plugin CI budget / e2e record. +# +# WHAT IT PRODUCES +# 1. A job summary on every invocation (ephemeral, free, always readable from +# the run page). +# 2. A ci-budget.json artifact (machine-readable, for a future fleet roll-up). +# 3. A single long-lived issue per repo whose BODY is the record, upserted in +# place — durable and permalinked with ZERO commits and ZERO PRs. +# +# Deliberately NOT a committed docs/CI-BUDGET.md: that would produce a commit +# per run whose only delta is a timestamp, which is the churn pattern that got +# the nightly devkit-sync deleted. +# +# HOW TO INVOKE +# * on a schedule (weekly) via the caller template — the cheap default; the +# record is a few API calls, ~20s, no containers, no runner minutes worth +# counting; +# * or as a final job of plugin-e2e, if you want the record refreshed the +# moment an e2e finishes (pass workflow-name and it will pick the run up). +name: ci-budget + +on: + workflow_call: + inputs: + workflow-name: + description: "NAME (not filename) of the workflow to budget." + required: false + type: string + default: "plugin-e2e" + window-days: + description: "Rolling window for minutes + pass/fail counts." + required: false + type: number + default: 30 + devkit-submodule-path: + description: "Path to the vendored devkit submodule." + required: false + type: string + default: "tests/_testkit" + publish-issue: + description: "Upsert the durable record issue (else summary + artifact only)." + required: false + type: boolean + default: true + fail-on-stale: + description: "Fail the job when no e2e run exists in the window." + required: false + type: boolean + default: false + secrets: + # OPTIONAL, and the workflow is fully useful without it. It unlocks the + # authoritative billed-minutes tier from the org billing usage endpoint, + # which the repo-scoped GITHUB_TOKEN structurally cannot read. Without it + # the record says so in plain words instead of guessing a number. + BILLING_READ_TOKEN: + description: "Org-scoped PAT (read:org) for /organizations/{org}/settings/billing/usage." + required: false + +jobs: + budget: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read # runs + jobs timing + issues: write # upsert the durable record issue + steps: + - name: Checkout consumer (no submodules) + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + submodules: false + + - name: Init devkit submodule + env: + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + # The devkit is public, so this needs no credentials (DEC-075). + run: git submodule update --init "$DEVKIT_PATH" + + - name: Resolve plugin name + id: resolve + run: | + set -euo pipefail + y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; } + plugin_dir="" + [ -f .devkit.yml ] && plugin_dir=$(y plugin_dir .devkit.yml) + if [ -z "$plugin_dir" ]; then + plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)") + fi + name=$(y name "$plugin_dir/plugin.yaml") + [ -n "$name" ] || name="${GITHUB_REPOSITORY##*/}" + echo "plugin_name=$name" >> "$GITHUB_OUTPUT" + echo "resolved plugin: $name" + + - name: Detect whether a billing token is available + id: creds + # The `secrets` context is NOT allowed inside an `if:` expression — using + # it there invalidates the entire workflow file and the run dies as a + # startup_failure with zero jobs. Probe it in a run step (secrets ARE + # available there via env) and branch on the step output. Only a yes/no + # is printed; the token itself never is. + env: + BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }} + run: | + if [ -n "${BILLING_TOKEN:-}" ]; then + echo "have_billing=true" >> "$GITHUB_OUTPUT" + echo "org billing token present — billed-minutes tier enabled" + else + echo "have_billing=false" >> "$GITHUB_OUTPUT" + echo "no org billing token — record will report billed minutes as unavailable" + fi + + - name: Build the record + id: build + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + BILLING_TOKEN: ${{ secrets.BILLING_READ_TOKEN }} + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + PLUGIN: ${{ steps.resolve.outputs.plugin_name }} + WF: ${{ inputs.workflow-name }} + DAYS: ${{ inputs.window-days }} + run: | + set -uo pipefail + python3 "$DEVKIT_PATH/e2e/scripts/ci-budget.py" \ + --repo "$GITHUB_REPOSITORY" \ + --plugin "$PLUGIN" \ + --workflow "$WF" \ + --days "$DAYS" \ + --json-out ci-budget.json \ + --md-out ci-budget.md + echo "rc=$?" >> "$GITHUB_OUTPUT" + + - name: Write job summary + if: always() + run: | + if [ -f ci-budget.md ]; then + cat ci-budget.md >> "$GITHUB_STEP_SUMMARY" + else + echo "## CI budget — generation FAILED" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload machine-readable record + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-budget-${{ steps.resolve.outputs.plugin_name }} + path: ci-budget.json + if-no-files-found: warn + retention-days: 90 + + - name: Upsert durable record issue + if: always() && inputs.publish-issue + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TITLE: "CI budget — ${{ steps.resolve.outputs.plugin_name }}" + BODY_FILE: ci-budget.md + LABEL: ci-budget + MODE: always + DEVKIT_PATH: ${{ inputs.devkit-submodule-path }} + run: | + { + echo "" + echo "---" + echo "Maintained automatically by the devkit \`ci-budget\` workflow —" + echo "the body is replaced in place on each run, so there is no commit and" + echo "no PR. Last refreshed by [run ${GITHUB_RUN_ID}](https://github.com/${REPO}/actions/runs/${GITHUB_RUN_ID})." + } >> ci-budget.md + bash "$DEVKIT_PATH/e2e/ci/upsert-record-issue.sh" + + - name: Fail on stale e2e + # Off by default. Turn it on once the fleet's cadence is steady and you + # want "nobody has run e2e in 30 days" to be loud rather than a footnote. + if: always() && inputs.fail-on-stale && steps.build.outputs.rc != '0' + run: | + echo "::error::ci-budget reported a stale or missing e2e record" + exit 1 diff --git a/.github/workflows/upstream-compat.yml b/.github/workflows/upstream-compat.yml new file mode 100644 index 0000000..457fe25 --- /dev/null +++ b/.github/workflows/upstream-compat.yml @@ -0,0 +1,415 @@ +# upstream-compat — continuous backwards-support check against LATEST UPSTREAM A0. +# +# THE GAP THIS CLOSES +# Every consumer's .devkit.yml pins the private fork image, and none of the +# four target plugins overrides `a0_image`. So the fleet's e2e today proves +# "works on the fork" and says NOTHING about "still works on upstream". The +# machinery already exists — plugin-e2e.yml takes an a0_image input and the +# devkit's own sample-plugin-e2e.yml already boots +# docker.io/agent0ai/agent-zero:latest. What was missing is the scheduled +# fan-out and somewhere the result is seen. That is all this file adds. +# +# HOW THIS DIFFERS FROM THE NIGHTLY devkit-sync THAT WAS DELETED +# devkit-sync burned 189 runs to produce 29 merged bumps and then silently +# no-op'd for three weeks. Three concrete differences: +# +# 1. CHANGE-GATED, NOT CALENDAR-GATED. A ~20s probe job resolves the upstream +# manifest digest and only fans out the expensive legs when the digest has +# actually moved since the last full run (tracked via actions/cache — no +# commits, no state file). Upstream :latest does not change nightly, so +# this is the difference between ~1,700 runner-min/month and roughly +# (upstream releases × 56 min). Detection latency is unchanged: the probe +# still runs every night. +# 2. IT CANNOT SILENTLY NO-OP. A weekly floor (max-skip-days) forces a full +# run even when the digest is unchanged, so "nothing happened for three +# weeks" is impossible. The probe also HARD-FAILS if it cannot resolve the +# digest, rather than skipping quietly, and the report job asserts that the +# matrix actually produced results — an empty matrix is a failure, not a +# pass. Every outcome is stamped into the record issue with a timestamp, so +# staleness is visible on the artifact itself. +# 3. ZERO PR CHURN. It never opens a pull request and never commits. Its only +# write is one long-lived issue, opened when a plugin breaks against +# upstream and closed automatically when the fleet is green again. An open +# issue means "act"; there is no steady-state noise to learn to ignore. +name: upstream-compat + +on: + schedule: + # 03:20 UTC — off the top of the hour, where GitHub's scheduler queues are + # worst. The probe runs nightly; the expensive legs are digest-gated. + - cron: "20 3 * * *" + workflow_dispatch: + inputs: + force: + description: "Run the full matrix even if the upstream digest is unchanged" + type: boolean + default: false + plugins: + description: "Override the plugin list (space-separated repo names)" + type: string + default: "" + +permissions: + contents: read + actions: write + issues: write + +env: + UPSTREAM_IMAGE: docker.io/agent0ai/agent-zero:latest + # The fleet under continuous backwards-support check. Space-separated repo + # names within agent-zero-plugins. Adding a plugin is a one-word edit. + PLUGINS: >- + agent-zero-plugin-chat-comments + agent-zero-plugin-diff-visualizer + agent-zero-plugin-mermaid-diagrams + agent-zero-plugin-share-chat + # Force a full run at least this often even if upstream never moves, so the + # workflow can never quietly become a no-op. + MAX_SKIP_DAYS: 7 + +jobs: + # ---------------------------------------------------------------- 1. probe + probe: + name: probe upstream digest + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + digest: ${{ steps.digest.outputs.digest }} + changed: ${{ steps.decide.outputs.changed }} + reason: ${{ steps.decide.outputs.reason }} + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve upstream manifest digest + id: digest + # Hard-fails if it cannot resolve. A probe that shrugs and exits 0 is how + # devkit-sync managed to no-op for three weeks unnoticed. + env: + IMAGE: ${{ env.UPSTREAM_IMAGE }} + run: bash e2e/ci/upstream-digest.sh "$IMAGE" + + - name: Look up last-tested digest + id: cache + uses: actions/cache/restore@v4 + with: + # Cache HIT on this exact digest ⇒ we already ran the full matrix + # against this image. No commits, no state file, no PR. + path: .upstream-compat-state + key: upstream-compat-${{ steps.digest.outputs.digest_short }} + restore-keys: upstream-compat- + + - name: Decide whether to fan out + id: decide + env: + HIT: ${{ steps.cache.outputs.cache-hit }} + FORCE: ${{ inputs.force }} + DIGEST: ${{ steps.digest.outputs.digest }} + run: | + set -euo pipefail + last_ts=0 + [ -f .upstream-compat-state ] && last_ts=$(sed -n 's/^ts=//p' .upstream-compat-state | head -1) + : "${last_ts:=0}" + now=$(date +%s) + age_days=$(( (now - last_ts) / 86400 )) + + if [ "${FORCE}" = "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=forced via workflow_dispatch" >> "$GITHUB_OUTPUT" + elif [ "${HIT}" != "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=upstream digest moved to ${DIGEST}" >> "$GITHUB_OUTPUT" + elif [ "$last_ts" -eq 0 ] || [ "$age_days" -ge "${MAX_SKIP_DAYS}" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=weekly floor reached (${age_days}d since last full run) — digest unchanged" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "reason=upstream digest unchanged (${DIGEST}), last full run ${age_days}d ago — skipping the expensive matrix" >> "$GITHUB_OUTPUT" + fi + + - name: Build plugin matrix + id: matrix + env: + OVERRIDE: ${{ inputs.plugins }} + run: | + set -euo pipefail + list="${OVERRIDE:-$PLUGINS}" + json=$(printf '%s\n' $list | python3 -c 'import sys,json; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))') + n=$(printf '%s' "$json" | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))') + # An empty matrix would make every downstream job "succeed" while + # testing nothing. Refuse. + [ "$n" -gt 0 ] || { echo "::error::plugin matrix is empty"; exit 1; } + echo "matrix=$json" >> "$GITHUB_OUTPUT" + echo "matrix ($n): $json" + + - name: Probe summary + run: | + { + echo "## upstream-compat probe" + echo "" + echo "| | |" + echo "|---|---|" + echo "| image | \`${UPSTREAM_IMAGE}\` |" + echo "| digest | \`${{ steps.digest.outputs.digest }}\` |" + echo "| fan out? | **${{ steps.decide.outputs.changed }}** |" + echo "| reason | ${{ steps.decide.outputs.reason }} |" + } >> "$GITHUB_STEP_SUMMARY" + + # ------------------------------------------------------------ 2. the legs + compat: + name: ${{ matrix.plugin }} + needs: probe + if: needs.probe.outputs.changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false # one broken plugin must not hide the state of the others + max-parallel: 2 # nested-A0 legs are disk-hungry; stagger them + matrix: + plugin: ${{ fromJSON(needs.probe.outputs.matrix) }} + permissions: + contents: read + steps: + - name: Checkout the plugin under test + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + # Public repos, so the default token suffices for a cross-repo read. + repository: ${{ github.repository_owner }}/${{ matrix.plugin }} + submodules: recursive + path: plugin + + - name: Free disk space + run: | + df -h / | tail -1 + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup \ + /usr/local/lib/android /opt/hostedtoolcache/CodeQL \ + /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" 2>/dev/null || true + sudo docker image prune -af 2>/dev/null || true + echo "after reclaim:"; df -h / | tail -1 + + - name: Resolve plugin + id: resolve + working-directory: plugin + run: | + set -euo pipefail + y(){ sed -nE "s/^$1:[[:space:]]*\"?([^\"]+)\"?.*/\1/p" "$2" 2>/dev/null | head -1; } + plugin_dir="" display_name="" + if [ -f .devkit.yml ]; then + plugin_dir=$(y plugin_dir .devkit.yml) + display_name=$(y display_name .devkit.yml) + fi + if [ -z "$plugin_dir" ]; then + plugin_dir=$(dirname "$(find usr/plugins -maxdepth 2 -name plugin.yaml 2>/dev/null | head -1)") + fi + [ -d "$plugin_dir" ] && [ -f "$plugin_dir/plugin.yaml" ] || { + echo "::error::could not resolve plugin_dir in ${{ matrix.plugin }}"; exit 1; } + plugin_name=$(y name "$plugin_dir/plugin.yaml") + [ -n "$display_name" ] || display_name=$(y title "$plugin_dir/plugin.yaml") + [ -n "$display_name" ] || display_name="$plugin_name" + pod_env="" + if [ -f .devkit.yml ]; then + pod_env=$(y e2e_pod_env .devkit.yml) + if [ -z "$pod_env" ]; then + pod_env=$(awk ' + /^e2e_pod_env:[[:space:]]*$/ {inblk=1; next} + inblk && /^[[:space:]]+[A-Za-z_][A-Za-z0-9_]*:/ { + l=$0; sub(/^[[:space:]]+/,"",l) + k=l; sub(/:.*/,"",k) + v=l; sub(/^[^:]*:[[:space:]]*/,"",v); gsub(/^["'\'']|["'\'']$/,"",v) + printf "%s=%s ", k, v; next } + inblk && /^[^[:space:]]/ {inblk=0} + ' .devkit.yml) + fi + fi + echo "plugin_dir=$plugin_dir" >> "$GITHUB_OUTPUT" + echo "plugin_name=$plugin_name" >> "$GITHUB_OUTPUT" + echo "display_name=$display_name" >> "$GITHUB_OUTPUT" + echo "pod_env=$pod_env" >> "$GITHUB_OUTPUT" + echo "Resolved: dir=$plugin_dir name=$plugin_name a0=UPSTREAM" + + - name: Build devkit devcontainer (rootless) + run: podman build -t plugin-devkit:ci -f devcontainer/Containerfile devcontainer + + - name: Package + run lifecycle against UPSTREAM A0 + env: + PLUGIN_DIR: ${{ steps.resolve.outputs.plugin_dir }} + PLUGIN_NAME: ${{ steps.resolve.outputs.plugin_name }} + DISPLAY_NAME: ${{ steps.resolve.outputs.display_name }} + A0_POD_ENV: ${{ steps.resolve.outputs.pod_env }} + # The whole point: pin the digest the probe resolved, not the moving + # tag, so every leg of one night tests the SAME image and a red result + # names an exact upstream artifact. + A0_IMAGE: docker.io/agent0ai/agent-zero@${{ needs.probe.outputs.digest }} + run: | + set -euo pipefail + mkdir -p dist "$GITHUB_WORKSPACE/artifacts" + chmod 777 "$GITHUB_WORKSPACE/artifacts" + ( cd "plugin/$PLUGIN_DIR" && zip -qr "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip" . \ + -x '.git/*' 'tests/*' '.github/*' '.skills/*' 'dist/*' '*/node_modules/*' \ + '.devkit.yml' '.gitmodules' '.gitignore' ) + BEHAVIOUR_ARG="" + [ -f "plugin/tests/e2e/behaviour.mjs" ] && BEHAVIOUR_ARG="/plugin/tests/e2e/behaviour.mjs" + SPECS_JSON="[]" + if ls plugin/tests/e2e/specs/*.mjs >/dev/null 2>&1; then + SPECS_JSON=$(for f in plugin/tests/e2e/specs/*.mjs; do + n=$(basename "$f" .mjs); printf '{"name":"%s","path":"/plugin/%s"}\n' "$n" "${f#plugin/}"; done \ + | paste -sd, - | sed 's/^/[/; s/$/]/') + fi + HARNESS=run-lifecycle.sh + if ls plugin/tests/e2e/features/*.feature >/dev/null 2>&1; then HARNESS=run-bdd.sh; fi + echo "harness: $HARNESS image: $A0_IMAGE" + podman run --rm \ + --device /dev/fuse \ + --security-opt label=disable \ + --sysctl net.ipv4.ip_unprivileged_port_start=0 \ + -v "$GITHUB_WORKSPACE":/workspace:ro \ + -v "$GITHUB_WORKSPACE/plugin":/plugin:ro \ + -v "$GITHUB_WORKSPACE/dist/$PLUGIN_NAME.zip":/tmp/plugin.zip:ro \ + -v "$GITHUB_WORKSPACE/artifacts":/artifacts:rw \ + -e A0_IMAGE="$A0_IMAGE" \ + -e PLUGIN_ZIP=/tmp/plugin.zip \ + -e PLUGIN_DISPLAY_NAME="$DISPLAY_NAME" \ + -e PLUGIN_NAME="$PLUGIN_NAME" \ + -e HOOK_DIR=/plugin/tests/e2e/hooks \ + -e BEHAVIOUR_FILE="$BEHAVIOUR_ARG" \ + -e BEHAVIOUR_SPECS="$SPECS_JSON" \ + -e A0_POD_ENV="$A0_POD_ENV" \ + -e ARTIFACT_DIR=/artifacts \ + -e WORKSPACE=/plugin \ + -e DEVKIT_PATH=. \ + -e BDD_TRACE=retain-on-failure \ + plugin-devkit:ci \ + -lc 'bash /workspace/e2e/harness/'"$HARNESS" + + - name: Upload failure evidence + # retain-on-failure, not always: a nightly that uploads a full trace set + # for four green plugins every week is storage the operator pays for and + # nobody opens. A red night gets the full trace. + if: failure() + uses: actions/upload-artifact@v4 + with: + name: upstream-compat-${{ matrix.plugin }} + path: artifacts/ + if-no-files-found: warn + retention-days: 30 + + # ---------------------------------------------------------------- 3. report + report: + name: report + needs: [probe, compat] + if: always() && needs.probe.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Assemble the record + id: rec + env: + CHANGED: ${{ needs.probe.outputs.changed }} + REASON: ${{ needs.probe.outputs.reason }} + DIGEST: ${{ needs.probe.outputs.digest }} + MATRIX: ${{ needs.probe.outputs.matrix }} + COMPAT: ${{ needs.compat.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + now=$(date -u +'%Y-%m-%d %H:%M UTC') + { + echo "# Upstream compatibility — \`${UPSTREAM_IMAGE}\`" + echo "" + echo "| | |" + echo "|---|---|" + echo "| last checked | **${now}** |" + echo "| upstream digest | \`${DIGEST}\` |" + echo "| full matrix ran? | **${CHANGED}** — ${REASON} |" + echo "| plugins under check | $(printf '%s' "$MATRIX" | python3 -c 'import sys,json; print(", ".join("`%s`" % p for p in json.load(sys.stdin)))') |" + echo "| result | ${COMPAT:-skipped} |" + echo "| evidence | [${GITHUB_RUN_ID}](${RUN_URL}) |" + } > record.md + + # STATUS drives whether the record issue is opened or closed. + # compat skipped (digest unchanged) → ok, nothing to report + # compat success → ok + # anything else → fail + case "${COMPAT}" in + ""|skipped|success) status=ok ;; + *) status=fail ;; + esac + if [ "$status" = "fail" ]; then + { + echo "" + echo "## ❌ At least one plugin fails against the latest upstream A0" + echo "" + echo "Open the [run](${RUN_URL}) and expand the red matrix leg. Per-leg" + echo "Playwright traces are attached as \`upstream-compat-\` artifacts." + echo "" + echo "This is a **backwards-support regression**: the plugin still passes on" + echo "the fork image its own \`plugin-e2e\` pins, so nothing in the plugin" + echo "repo went red. Upstream moved to \`${DIGEST}\` and something the plugin" + echo "depends on moved with it." + } >> record.md + fi + { + echo "" + echo "---" + echo "Maintained by the devkit \`upstream-compat\` workflow. The body is" + echo "replaced in place — no commits, no pull requests. This issue opens when a" + echo "plugin breaks against upstream and closes automatically when the fleet is" + echo "green, so an OPEN issue always means action is needed." + } >> record.md + echo "status=$status" >> "$GITHUB_OUTPUT" + cat record.md >> "$GITHUB_STEP_SUMMARY" + + - name: Guard against a silent no-op + # The devkit-sync failure mode, made impossible: if the probe said "fan + # out" then the matrix MUST have produced a result. A green report with + # nothing tested is a bug, not a pass. + env: + CHANGED: ${{ needs.probe.outputs.changed }} + COMPAT: ${{ needs.compat.result }} + run: | + if [ "$CHANGED" = "true" ] && { [ -z "$COMPAT" ] || [ "$COMPAT" = "skipped" ]; }; then + echo "::error::probe requested a full run but the compat matrix never executed" + exit 1 + fi + echo "no-op guard OK (changed=$CHANGED compat=${COMPAT:-none})" + + - name: Upsert the record issue + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TITLE: "Upstream compatibility — agent0ai/agent-zero:latest" + BODY_FILE: record.md + LABEL: upstream-compat + MODE: only-on-failure + STATUS: ${{ steps.rec.outputs.status }} + run: bash e2e/ci/upsert-record-issue.sh + + - name: Record the tested digest + # Written only on a GREEN full run, so a red night is retried tomorrow + # instead of being cached away as "already checked". + if: needs.compat.result == 'success' + run: | + printf 'digest=%s\nts=%s\n' "${{ needs.probe.outputs.digest }}" "$(date +%s)" \ + > .upstream-compat-state + + - name: Save digest cache + if: needs.compat.result == 'success' + uses: actions/cache/save@v4 + with: + path: .upstream-compat-state + key: upstream-compat-${{ needs.probe.outputs.digest }} + + - name: Fail the run when a plugin is broken + # The nightly's own red X is the primary signal — Actions emails the + # workflow owner on a scheduled-run failure, which is the "signal someone + # sees" requirement. The issue is the durable second channel. + if: steps.rec.outputs.status == 'fail' + run: | + echo "::error::one or more plugins fail against the latest upstream A0" + exit 1 diff --git a/SPEC.md b/SPEC.md index 8e225d8..25450b3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1255,8 +1255,30 @@ job disappeared when the devkit went public. The real cost is **integrity and bl unreviewed merge changes CI in 24 repos with no canary and no rollback boundary. Fixed as change management, not as a breach. +**DEC-082 — Upstream A0 compatibility is tested, because upstream is the only consumer.** +The reusable workflow defaults `a0_image` to the private fork +(`ghcr.io/nuevanext/agent-zero:latest-nonroot`) and **none of the four plugins overrode it**, so every +PR proved the plugin worked on the fork and nothing anywhere proved it worked on upstream — which is +the only thing anyone actually runs. That is not a gap in coverage so much as coverage pointed at the +wrong target. +Two changes, deliberately separate. Each plugin sets `a0_image` to +`docker.io/agent0ai/agent-zero:latest` in its own `.devkit.yml`, so the **PR gate** now tests what +customers run — a break is caught before merge, by the author, with the diff in front of them. And a +scheduled `upstream-compat` workflow re-runs the same suites when **upstream** moves, which no PR can +catch because nothing in the repo changed. +*Why this schedule is not the one that was just deleted (DEC-079).* That one opened a PR every night +whether or not anything had changed, and failed silently behind an approval gate for three weeks. This +one is gated on the upstream manifest **digest** actually moving — measured, upstream ships ~1.5×/week +against a 42-minute fan-out, so digest-gating costs ~295 runner-min/month instead of ~1,275 for the +same ≤24h detection. It opens no PRs, commits nothing, and reports failure as a single issue that +auto-closes when green, so an open issue always means act. Three structural guards keep it honest: the +probe hard-fails rather than shrugging, an empty matrix is an error rather than a vacuous pass, and the +report job asserts the matrix ran if the probe asked for it. +The runs pin the resolved **digest**, not `:latest`, so a failure is reproducible against the exact +image that broke it rather than whatever `:latest` points at by the time someone investigates. + > **Numbering note:** DEC-070–072 were originally introduced as code comments only. They are now -> written up above, so this section documents an unbroken 070–081. The lesson is worth keeping: a +> written up above, so this section documents an unbroken 070–082. The lesson is worth keeping: a > decision that exists only as a code comment is invisible to anyone reading the SPEC, and all three > of these were load-bearing (a gate's timeout budget, the image the whole fleet pulls, and a wait > that was flaking half the runs). diff --git a/docs/upstream-compat-and-ci-budget.md b/docs/upstream-compat-and-ci-budget.md new file mode 100644 index 0000000..15d9701 --- /dev/null +++ b/docs/upstream-compat-and-ci-budget.md @@ -0,0 +1,335 @@ +# CI budget + nightly upstream-compat — design + +Two capabilities for the devkit, built against +`agent-zero-plugin-development-testkit` and its four consumers +(`chat-comments`, `diff-visualizer`, `mermaid-diagrams`, `share-chat`). + +Everything below that states an API behaviour was **probed live on 2026-08-05**, +not assumed. Where an endpoint does not give us what the operator asked for, that +is said plainly rather than papered over. + +--- + +## Part 0 — what the GitHub API actually gives us + +I probed four candidate sources before designing anything. Three of the four are +either dead or useless here. + +| # | Endpoint | Result | Verdict | +|---|---|---|---| +| 1 | `GET /orgs/{org}/settings/billing/actions` | **HTTP 410** "This endpoint has been moved." (`gh` also reports it wants `admin:org`) | Dead. | +| 2 | `GET /orgs/{org}/settings/billing/shared-storage` | **HTTP 410**, same | Dead. | +| 3 | `GET /repos/{r}/actions/runs/{id}/timing` | Reachable with a plain repo `GITHUB_TOKEN`, but returns **`billable.UBUNTU.total_ms = 0`** on every run | **Useless here** — see below. | +| 4 | `GET /organizations/{org}/settings/billing/usage?year=&month=` | **Works.** Real per-repo, per-day minutes | The only true billing source. | + +### 3 is a trap, and it is the obvious thing to reach for + +`/timing` looks exactly like the answer. It is not. Three real +`chat-comments` runs returned: + +```json +{"billable":{"UBUNTU":{"total_ms":0,"jobs":2, + "job_runs":[{"job_id":92153588155,"duration_ms":0}, + {"job_id":92156121906,"duration_ms":0}]}}, + "run_duration_ms":810000} +``` + +All four consumer repos are **public**, and public-repo Actions minutes are free, +so GitHub reports **zero billable milliseconds**. A budget built on `/timing` +would report `0 min` for every plugin forever and look like it worked. The design +therefore *deliberately does not use it*, and the generated record says so on its +face so nobody "fixes" it back later. + +`run_duration_ms` in the same payload is real, but it is whole-run wall clock +including queue and idle gaps between jobs — not a cost figure. + +### 4 works, with one structural limitation that shapes the whole design + +``` +GET /organizations/agent-zero-plugins/settings/billing/usage?year=2026&month=8 +``` + +returns one row per (day, repo, SKU) — 191 rows for August, covering **27 repos +including all four consumers**: + +```json +{"date":"2026-08-04T00:00:00Z","product":"actions","sku":"Actions Linux", + "quantity":78.0,"unitType":"Minutes","netAmount":0.0, + "repositoryName":"agent-zero-plugin-chat-comments"} +``` + +So **minutes are real and per-repo**. Two caveats worth stating: + +- **The unfiltered call lies by omission.** With no `year`/`month` it returns + only **8 aggregate rows** covering 5 repos. With `?year=&month=` it returns + **191**. Anyone probing this endpoint casually would conclude our plugins have + no CI usage. They do. +- **`read:org` was sufficient** on the operator PAT — this one does *not* need + `admin:org`, unlike the two dead endpoints. + +**The limitation that drives the architecture:** a workflow's `GITHUB_TOKEN` is +repo-scoped. There is no `permissions:` key that grants org billing — the +capability does not exist in the workflow token model at all. So **tier 3 is +unreachable from inside a plugin repo's own CI unless an org PAT is supplied as +a secret.** I did not work around this, because the available workarounds all +involve inventing a number. + +### What we use instead, and what it is honestly worth + +`GET /repos/{r}/actions/runs/{id}/jobs` gives per-job `started_at` / +`completed_at` on any token. Summing `(completed − started)` across jobs is a +**runner-wall-clock proxy** — always available, no secret. Since GitHub bills +per job rounded up to the whole minute, it tracks the real figure closely but is +not identical to it. + +**It is not the same measurement as tier 3, and the record never implies it is.** +Measured live on `chat_comments` over the same 30 days: + +| | value | scope | +|---|---|---| +| billed (tier 3) | **454.0 min** | whole repo, *every* workflow, rounded up per job | +| observed (tier 4) | **199.8 min** | `plugin-e2e` only | + +The 2.3× gap is not error — it is `devkit-sync` + `unit` + Dependabot + rounding. +An early draft of the record showed these as adjacent bare numbers, which reads +as "one of these is wrong". The shipped version labels each with its scope and +carries a one-line note explaining the difference. **Use billed for the invoice, +observed for what the e2e suite costs.** + +One more honesty note: `netAmount` was **not** always `$0.00` despite the repos +being public (`chat_comments` 30-day net was **$1.60**, `share_chat` **$0.16**). +The naive "public means free" assumption does not hold in this org's billing +data. The record reports whatever the API returns and does not assert the repos +are free. + +--- + +## Part A — CI budget / e2e record per plugin + +### Where the record lives, and why + +The operator asked for a record "on every one of the plugin records", showing CI +minutes, when e2e last ran, and its result. Options considered: + +| Option | Durable | Zero churn | Verdict | +|---|---|---|---| +| Committed `docs/CI-BUDGET.md` | yes | **no** — a commit (or PR) per run whose only delta is a timestamp | **rejected** | +| Job summary only | **no** — dies with the run | yes | insufficient alone | +| README badge | yes | no (same commit churn), and cannot show minutes | insufficient | +| shields.io endpoint badge | yes | yes | needs external JSON hosting we don't have | +| **Long-lived issue, body upserted in place** | **yes** | **yes** | **chosen** | + +**Chosen: all three tiers of output, with the issue as the durable one.** + +1. **Job summary** on every invocation — free, immediate, on the run page. +2. **`ci-budget.json` artifact** — machine-readable, so a future fleet roll-up + has something to consume without re-querying. +3. **One long-lived issue per repo**, titled `CI budget — `, label + `ci-budget`, whose **body is replaced in place**. + +Why the issue wins: + +- **Zero commits, zero PRs, zero branches.** This is the hard constraint after + the devkit-sync deletion. A committed markdown file fails it outright. +- **Editing an issue body sends no notification**, so it can refresh as often as + we like without training anyone to filter it out. +- **Durable and permalinked**, unlike a job summary that vanishes with the run's + retention. +- It is already where maintainers look, and it is pinnable per repo. + +### Staleness is a failure, not a footnote + +`ci-budget.py` **exits 2** when there is no completed e2e run in the window, and +the record renders `⚠️ never` rather than a blank. This is the same defect class +as devkit-sync's silent three-week no-op: a record that quietly says nothing is +worse than no record. `fail-on-stale` (default `false`) promotes it to a red +job once the fleet's cadence is steady. + +### Cadence + +Weekly (`cron: 40 6 * * 1`), not per-PR. It is a rolling 30-day view — running +it per-PR would spend runner time to change a timestamp. Also +`workflow_dispatch`-able for an on-demand read. The job itself is a handful of +API calls, ~20s, no containers. + +### Files + +- `scripts/ci-budget.py` → `e2e/scripts/ci-budget.py` +- `scripts/upsert-record-issue.sh` → `e2e/ci/upsert-record-issue.sh` +- `workflows/ci-budget.yml` → `.github/workflows/ci-budget.yml` (reusable) +- `templates/ci-budget.caller.yml` → `templates/`, distributed by + `make link-workflows` + +`BILLING_READ_TOKEN` is an **optional** secret. Absent, the record is fully +useful and states in words that billed minutes are unavailable and why. Present, +tier 3 lights up. Both paths are live-verified below. + +--- + +## Part B — nightly upstream-compat + +### The gap it closes (verified, not assumed) + +All four consumers' `.devkit.yml` files were read directly. **None sets +`a0_image`**, so all four inherit the reusable workflow's default — +`ghcr.io/nuevanext/agent-zero:latest-nonroot`, the private fork. The fleet's e2e +today proves *"works on the fork"* and says **nothing** about *"still works on +upstream"*. + +The machinery genuinely does mostly exist: `plugin-e2e.yml` accepts `a0_image`, +and `sample-plugin-e2e.yml` already boots `docker.io/agent0ai/agent-zero:latest`. +What is missing is the scheduled fan-out and a place the result is seen. That is +all this adds. + +### How it differs from the deleted devkit-sync nightly + +devkit-sync burned 189 runs for 29 merged bumps, then silently no-op'd for three +weeks. Three concrete structural differences: + +**1. Change-gated, not calendar-gated.** +A ~20s `probe` job resolves the upstream **manifest digest** anonymously from +Docker Hub and only fans out when the digest actually moved since the last green +full run. State lives in `actions/cache` keyed by digest — no commits, no state +file, no PR. Live-verified: + +``` +$ bash scripts/upstream-digest.sh docker.io/agent0ai/agent-zero:latest +sha256:8c5eff81a46fc956bc247e01a345beadc09acbeecf31873324b843dacf937c4c +``` + +The probe still runs **every night**, so detection latency is unchanged (≤24h). +Only the expensive part is gated. + +**2. It cannot silently no-op.** Four independent guards: +- the probe **hard-fails** if it cannot resolve a digest (a shrug-and-exit-0 + probe is exactly how devkit-sync died unnoticed); +- an **empty matrix is a hard error**, not a vacuous pass; +- a **`MAX_SKIP_DAYS: 7` floor** forces a full run even if upstream never moves; +- the `report` job asserts that *if the probe requested a fan-out, the matrix + actually produced a result* — otherwise it fails. +- Every outcome stamps a `last checked` timestamp into the record, so staleness + is visible on the artifact itself. + +**3. Zero PR churn.** It never opens a PR and never commits. Its only write is +**one** long-lived issue in `MODE=only-on-failure`: opened when a plugin breaks +against upstream, **closed automatically** when the fleet is green again. An open +issue always means "act"; there is no steady-state noise to learn to ignore. + +### Signal someone sees + +Primary: the **scheduled workflow's own red X**, which GitHub emails to the +workflow owner on a scheduled-run failure. Secondary and durable: the +auto-opening record issue. Tertiary: per-leg Playwright traces uploaded +`if: failure()` only — a green nightly uploading four full trace sets every week +is storage the operator pays for and nobody opens. + +### Cost/benefit — the honest numbers + +Real per-plugin e2e durations, measured from the most recent green run of each: + +| plugin | e2e job | +|---|---| +| chat-comments | 12m 39s | +| diff-visualizer | 13m 02s | +| mermaid-diagrams | 11m 00s | +| share-chat | 5m 49s | +| **one full fan-out** | **≈ 42 min** | + +Upstream release cadence, from Docker Hub tag timestamps (`v2.8` 08-01, `v2.7` +07-27, `v2.6` 07-23, `v2.5` 07-17, `v2.4` 07-10, `v2.3` 07-09, `v2.2` 07-02, +`v2.1` 06-26, `v2.0` 06-24) — **≈ 9 releases in 6 weeks ≈ 1.5/week ≈ 6.5/month.** + +| strategy | runner-min / month | +|---|---| +| naive calendar nightly (30 × 42) | **≈ 1,275** | +| digest-gated (≈7 full runs + 30 probes) | **≈ 295 + ~10** | + +**≈ 76% cheaper for identical detection latency.** Because upstream ships more +often than weekly, the `MAX_SKIP_DAYS` floor will rarely fire — it is insurance +against upstream going quiet, not a routine cost. + +**The honest caveat:** at ~1.5 releases/week the digest changes often enough that +this is *not* nearly-free — it is roughly 295 min/month, ~23% of a naive nightly +but still a real recurring spend. Whether that is worth it depends on one +question the operator should answer, not me: **do these plugins need to work on +upstream A0 at all, or only on the fork they are deployed against?** If only the +fork, this workflow is ~295 min/month of information nobody acts on and should +not be adopted. If upstream compatibility is a shipped promise, this is the +cheapest way to hold it, and the current state — where *nothing* tests upstream — +means the promise is untested today. + +A cheaper variant if the answer is "nice to have": change `MAX_SKIP_DAYS` to 14 +and add a `plugins` subset so only the two most upstream-coupled plugins run per +digest. I did not default to that because a partial check that silently omits +plugins is the ambiguity this whole design is trying to remove. + +### `secrets` in `if:` — avoided by construction + +Per the explicit warning: the `secrets` context is **not** permitted in an `if:` +expression; using it invalidates the entire workflow file and the run dies as +`startup_failure` with zero jobs. `ci-budget.yml` therefore probes +`BILLING_READ_TOKEN` in a **`run:` step** (where secrets *are* available via +`env:`) and branches on `steps.creds.outputs.have_billing`. This mirrors the +existing pattern already proven in `plugin-e2e.yml`'s "Detect whether App +credentials are available" step. `upstream-compat.yml` uses no secrets at all — +the upstream image is public and the digest probe is anonymous. + +--- + +## Validation performed + +``` +YAML (python3 -c "import yaml; yaml.safe_load(open(...))") + OK workflows/ci-budget.yml + OK workflows/upstream-compat.yml + OK templates/ci-budget.caller.yml + +bash -n + OK scripts/upsert-record-issue.sh + OK scripts/upstream-digest.sh + +shellcheck CLEAN (both scripts, zero findings) +python3 -m py_compile OK scripts/ci-budget.py +``` + +**Live end-to-end runs against the real API** (not mocks): + +- `upstream-digest.sh` → resolved the real Docker Hub digest anonymously. +- `ci-budget.py` **without** a billing token (simulating `GITHUB_TOKEN`) → + rendered a complete record, billed minutes marked *not available* with the + reason. rc=0. +- `ci-budget.py` **with** the org token → tier 3 lit up: `454.0 min`, + `net $1.60`, `20.768 GB-hours`. rc=0. +- `ci-budget.py` against `share-chat` → `105.0 min` / `54.3 min` observed. rc=0. +- **Stale path**: asked for a workflow that has never run → rendered + `⚠️ never — no completed upstream-compat run in the last 30 days` and + **exited 2**, as designed. + +### Not validated + +The two workflows have **not been executed on GitHub Actions** — that needs a +commit to a repo, which this task explicitly forbids. YAML validity, script +correctness and every API call they make are verified; job wiring +(`needs`/`outputs`/matrix `fromJSON`) is reviewed but not run. The nested-A0 leg +in `upstream-compat.yml` is adapted from the proven `plugin-e2e.yml` body with +one real change — the plugin is checked out into `plugin/` and mounted at +`/plugin`, because this workflow runs in the *devkit's* workspace rather than the +consumer's. That path change is the highest-risk untested detail and should be +the first thing a trial `workflow_dispatch` exercises. + +--- + +## Install map + +| built here | goes to | +|---|---| +| `scripts/ci-budget.py` | devkit `e2e/scripts/ci-budget.py` | +| `scripts/upsert-record-issue.sh` | devkit `e2e/ci/upsert-record-issue.sh` | +| `scripts/upstream-digest.sh` | devkit `e2e/ci/upstream-digest.sh` | +| `workflows/ci-budget.yml` | devkit `.github/workflows/ci-budget.yml` | +| `workflows/upstream-compat.yml` | devkit `.github/workflows/upstream-compat.yml` | +| `templates/ci-budget.caller.yml` | devkit `templates/` + one line in `link-workflows` | + +`upstream-compat.yml` lives **only** in the devkit and needs no consumer change — +it checks the plugin repos out itself. Only `ci-budget` needs a caller +distributed to the four consumers. diff --git a/e2e/ci/ci-budget.py b/e2e/ci/ci-budget.py new file mode 100755 index 0000000..ce4534c --- /dev/null +++ b/e2e/ci/ci-budget.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +Build the per-plugin CI budget / e2e record. + +WHY THIS IS A SCRIPT AND NOT INLINE YAML + Same reason as e2e/ci/prune-artifacts.sh: it is called from the reusable + ci-budget.yml AND from the nightly upstream-compat.yml, and it is the kind + of logic (pagination, unit conversion, graceful degradation) that rots + instantly when copy-pasted into two workflows. + +WHAT THE GITHUB API ACTUALLY GIVES US (probed 2026-08-05, not assumed) + + 1. repos/{r}/actions/runs/{id}/timing → billable.UBUNTU.total_ms + Reachable with the workflow's own GITHUB_TOKEN (actions:read). + USELESS HERE: it returns 0 for every run on these repos, because they are + PUBLIC and public-repo Actions minutes are free, so GitHub reports zero + *billable* time. Verified on three real chat-comments runs: + {"billable":{"UBUNTU":{"total_ms":0,...}},"run_duration_ms":810000} + We therefore do NOT use it as the minute source. run_duration_ms is real + but is whole-run wall clock including queue/idle between jobs. + + 2. GET /orgs/{org}/settings/billing/actions → HTTP 410 "moved" + GET /orgs/{org}/settings/billing/shared-storage → HTTP 410 "moved" + Both are gone. gh additionally reports they'd want admin:org. Dead ends. + + 3. GET /organizations/{org}/settings/billing/usage?year=&month= + THE one that works. Needs an ORG-scoped token (read:org was enough for + the operator PAT; the endpoint is on the "enhanced billing platform"). + Returns one row per (day, repo, SKU): + {"date":"2026-08-04...","product":"actions","sku":"Actions Linux", + "quantity":78.0,"unitType":"Minutes","netAmount":0.0, + "repositoryName":"agent-zero-plugin-chat-comments"} + So MINUTES ARE REAL AND PER-REPO. netAmount is 0.0 because public repos + are free — the *cost* is genuinely zero, the *minutes* are not. + LIMITATION: a workflow's GITHUB_TOKEN is repo-scoped and has no billing + permission at all (there is no `permissions:` key that grants it), so this + tier only lights up when an org PAT is provided as a secret. Without it we + say so in the record rather than inventing a number. + + 4. repos/{r}/actions/runs/{id}/jobs → per-job started_at/completed_at. + Always reachable, no secret. Summing (completed-started) over jobs is our + ALWAYS-AVAILABLE minute proxy. It is runner wall clock, which is what + GitHub rounds up per job to bill, so it tracks tier-3 closely — it is a + proxy, and the record labels it as one. + +Output (stdout is the markdown record; --json-out writes the machine copy): + minutes_billed from (3) when available, else null + a stated reason + minutes_observed from (4), always + last e2e run: when, conclusion, duration, commit, link +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +API = "https://api.github.com" + + +def _get(url: str, token: str) -> object: + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "a0-devkit-ci-budget", + }, + ) + with urllib.request.urlopen(req, timeout=60) as r: + return json.load(r) + + +def _iso(s: str | None) -> datetime | None: + if not s: + return None + return datetime.fromisoformat(s.replace("Z", "+00:00")) + + +def _fmt_dur(seconds: float) -> str: + m, s = divmod(int(seconds), 60) + return f"{m}m {s:02d}s" + + +def _age(dt: datetime | None, now: datetime) -> str: + if dt is None: + return "never" + d = now - dt + if d < timedelta(hours=1): + return f"{int(d.total_seconds() // 60)}m ago" + if d < timedelta(days=1): + return f"{int(d.total_seconds() // 3600)}h ago" + return f"{d.days}d ago" + + +# ---------------------------------------------------------------- runs / jobs + +def collect_runs(repo: str, token: str, workflow: str, days: int) -> list[dict]: + """Runs of `workflow` in the last `days`. Repo-scoped: always available.""" + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + out: list[dict] = [] + page = 1 + while page <= 10: # hard cap: 1000 runs is far more than any window needs + url = ( + f"{API}/repos/{repo}/actions/runs" + f"?per_page=100&page={page}&exclude_pull_requests=false" + ) + try: + data = _get(url, token) + except urllib.error.HTTPError as e: # pragma: no cover - network + print(f"::warning::runs query failed: {e}", file=sys.stderr) + break + runs = data.get("workflow_runs", []) # type: ignore[union-attr] + if not runs: + break + stop = False + for r in runs: + created = _iso(r.get("created_at")) + if created and created < cutoff: + stop = True + continue + if workflow and r.get("name") != workflow: + continue + out.append(r) + if stop: + break + page += 1 + return out + + +def job_seconds(repo: str, run_id: int, token: str) -> float: + """Summed per-job runner wall clock. Our always-available minute proxy.""" + try: + data = _get(f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", token) + except urllib.error.HTTPError: # pragma: no cover - network + return 0.0 + total = 0.0 + for j in data.get("jobs", []): # type: ignore[union-attr] + a, b = _iso(j.get("started_at")), _iso(j.get("completed_at")) + if a and b and b > a: + total += (b - a).total_seconds() + return total + + +# ------------------------------------------------------------------- billing + +def billed_minutes(org: str, repo_name: str, token: str | None, days: int) -> dict: + """ + Tier-3 authoritative minutes from the enhanced-billing usage endpoint. + + Returns {"available": False, "reason": ...} when we have no org token, which + is the DEFAULT for a plugin repo running on GITHUB_TOKEN. We never guess. + """ + if not token: + return { + "available": False, + "reason": ( + "no org-scoped billing token supplied " + "(GITHUB_TOKEN is repo-scoped and cannot read org billing)" + ), + } + now = datetime.now(timezone.utc) + cutoff = now - timedelta(days=days) + months = {(now.year, now.month), (cutoff.year, cutoff.month)} + minutes = 0.0 + net = 0.0 + gross = 0.0 + gb_hours = 0.0 + seen = False + for year, month in sorted(months): + url = ( + f"{API}/organizations/{org}/settings/billing/usage" + f"?year={year}&month={month}" + ) + try: + data = _get(url, token) + except urllib.error.HTTPError as e: + return { + "available": False, + "reason": f"billing usage endpoint returned HTTP {e.code}", + } + for it in data.get("usageItems", []): # type: ignore[union-attr] + if it.get("repositoryName") != repo_name: + continue + d = _iso(it.get("date")) + if d and d < cutoff: + continue + seen = True + if it.get("unitType") == "Minutes": + minutes += float(it.get("quantity") or 0) + net += float(it.get("netAmount") or 0) + gross += float(it.get("grossAmount") or 0) + elif it.get("unitType") == "GigabyteHours": + gb_hours += float(it.get("quantity") or 0) + return { + "available": True, + "minutes": round(minutes, 1), + "storage_gb_hours": round(gb_hours, 3), + "net_usd": round(net, 4), + "gross_usd": round(gross, 4), + "note": ( + "netAmount is $0.00 because this repo is PUBLIC and public-repo " + "Actions minutes are free. The minutes are real; the bill is not." + ) + if seen and net == 0 + else "", + } + + +# -------------------------------------------------------------------- record + +def build_record(args: argparse.Namespace) -> dict: + now = datetime.now(timezone.utc) + org = args.repo.split("/")[0] + repo_name = args.repo.split("/")[1] + + runs = collect_runs(args.repo, args.token, args.workflow, args.days) + finished = [r for r in runs if r.get("status") == "completed"] + + observed = 0.0 + # Cost control: per-run /jobs is one API call each. Cap how many we walk; + # beyond the cap we report what we measured and say the window was capped. + walked = finished[: args.max_run_lookups] + for r in walked: + observed += job_seconds(args.repo, r["id"], args.token) + + last = finished[0] if finished else None + last_seconds = job_seconds(args.repo, last["id"], args.token) if last else 0.0 + + passes = sum(1 for r in finished if r.get("conclusion") == "success") + fails = sum(1 for r in finished if r.get("conclusion") == "failure") + + billing = billed_minutes(org, repo_name, args.billing_token, args.days) + + return { + "schema": "a0-devkit/ci-budget@1", + "generated_at": now.isoformat(), + "repo": args.repo, + "plugin": args.plugin, + "workflow": args.workflow, + "window_days": args.days, + "runs_total": len(finished), + "runs_measured": len(walked), + "runs_capped": len(finished) > len(walked), + "runs_success": passes, + "runs_failure": fails, + "minutes_observed": round(observed / 60, 1), + "minutes_billed": billing, + "last_run": None + if not last + else { + "id": last["id"], + "conclusion": last.get("conclusion"), + "event": last.get("event"), + "created_at": last.get("created_at"), + "age": _age(_iso(last.get("created_at")), now), + "duration": _fmt_dur(last_seconds), + "duration_seconds": round(last_seconds), + "sha": (last.get("head_sha") or "")[:8], + "url": last.get("html_url"), + }, + } + + +def render(rec: dict) -> str: + ico = {"success": "✅", "failure": "❌", "cancelled": "⚪", "timed_out": "⏱️"} + last = rec["last_run"] + b = rec["minutes_billed"] + + if b["available"]: + billed = f"**{b['minutes']} min** (net ${b['net_usd']:.2f})" + billed_note = b.get("note") or "" + else: + billed = "_not available_" + billed_note = b["reason"] + + # These two numbers are NOT the same measurement and must never be shown as + # if one validated the other. Measured live on chat_comments: 454 billed min + # vs 199.8 observed min over the same 30 days. The gap is not error — billing + # is REPO-WIDE (devkit-sync, unit, Dependabot, every workflow) and rounds each + # job up to a whole minute, while `observed` is filtered to one workflow. + scope_note = ( + f"Scopes differ by design: **billed** is repo-wide across *all* workflows " + f"and rounds each job up to a whole minute; **observed** is `{rec['workflow']}` " + f"only. Use billed for the invoice, observed for what the e2e suite costs." + ) + + lines = [ + f"## CI budget — `{rec['plugin']}`", + "", + f"Window: last **{rec['window_days']} days** · workflow `{rec['workflow']}` " + f"· generated {rec['generated_at'][:16].replace('T', ' ')} UTC", + "", + "| | |", + "|---|---|", + ] + if last: + lines += [ + f"| **e2e last ran** | [{last['created_at'][:16].replace('T', ' ')} UTC]({last['url']}) " + f"({last['age']}) |", + f"| **result** | {ico.get(last['conclusion'], '❔')} `{last['conclusion']}` " + f"on `{last['sha']}` (via `{last['event']}`) |", + f"| **that run took** | {last['duration']} |", + ] + else: + lines += [ + f"| **e2e last ran** | ⚠️ **never** — no completed `{rec['workflow']}` run " + f"in the last {rec['window_days']} days |", + "| **result** | — |", + "| **that run took** | — |", + ] + lines += [ + f"| **runs in window** | {rec['runs_total']} " + f"({rec['runs_success']} ✅ / {rec['runs_failure']} ❌) |", + f"| **minutes billed** _(whole repo, all workflows)_ | {billed} |", + f"| **minutes observed** _(`{rec['workflow']}` only, runner wall clock)_ | " + f"{rec['minutes_observed']} min over {rec['runs_measured']} run(s)" + + (" _(window capped)_" if rec["runs_capped"] else "") + + " |", + ] + if b["available"] and b.get("storage_gb_hours"): + lines.append( + f"| **artifact storage** _(whole repo)_ | {b['storage_gb_hours']} GB-hours |" + ) + lines += ["", f"> {scope_note}"] + if billed_note: + lines += ["", f"> {billed_note}"] + lines += [ + "", + "`minutes (observed)` is summed per-job runner wall clock from the " + "Actions API — a proxy, always available. `minutes (billed)` is the " + "authoritative org billing figure and only appears when an org-scoped " + "token is configured. The per-run `/timing` endpoint is deliberately " + "unused: it reports 0 billable ms on public repos.", + ] + return "\n".join(lines) + + +def main() -> int: + p = argparse.ArgumentParser(description="Generate the plugin CI budget record.") + p.add_argument("--repo", required=True, help="owner/name") + p.add_argument("--plugin", required=True) + p.add_argument("--workflow", default="plugin-e2e", help="workflow NAME to budget") + p.add_argument("--days", type=int, default=30) + p.add_argument("--max-run-lookups", type=int, default=40) + p.add_argument("--json-out", default="") + p.add_argument("--md-out", default="") + p.add_argument("--token", default=os.environ.get("GITHUB_TOKEN", "")) + p.add_argument( + "--billing-token", + default=os.environ.get("BILLING_TOKEN", "") or None, + help="org-scoped PAT for /organizations/{org}/settings/billing/usage", + ) + args = p.parse_args() + if not args.token: + print("::error::no GITHUB_TOKEN", file=sys.stderr) + return 1 + + rec = build_record(args) + md = render(rec) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as fh: + json.dump(rec, fh, indent=2) + fh.write("\n") + if args.md_out: + with open(args.md_out, "w", encoding="utf-8") as fh: + fh.write(md + "\n") + print(md) + + # Staleness is a FAILURE MODE, not a footnote. If the operator asked for + # "when last they ran", a record that quietly says "never" is the devkit-sync + # bug again. Exit non-zero so a human sees it. + if args.days and rec["last_run"] is None: + print( + f"::error::no completed {args.workflow} run in {args.days} days " + f"for {args.repo}", + file=sys.stderr, + ) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/e2e/ci/upsert-record-issue.sh b/e2e/ci/upsert-record-issue.sh new file mode 100755 index 0000000..99a9ee0 --- /dev/null +++ b/e2e/ci/upsert-record-issue.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Upsert a single long-lived GitHub issue whose BODY is the record. +# +# WHY AN ISSUE AND NOT A COMMITTED FILE +# A committed docs/CI-BUDGET.md would be rewritten on every run, so every e2e +# run would produce a commit (or a PR) whose only content is a changed +# timestamp. That is precisely the churn that got the nightly devkit-sync +# deleted — 189 runs, 29 useful bumps. An issue body is: +# * durable and permalinked (unlike a job summary, which dies with the run), +# * editable in place with zero commits, zero PRs, zero branch noise, +# * already where maintainers look, and pinnable per repo, +# * silent on edit (editing a body sends no notification), so it can update +# as often as we like without training anyone to ignore it. +# +# ANTI-STALENESS +# MODE=only-on-failure closes the issue when everything is green and (re)opens +# it when something is red — so an OPEN issue always means "act on this", and +# its absence is not ambiguous. The nightly's own last-run timestamp is written +# into the body either way, so "it silently stopped running" is visible rather +# than invisible. +# +# Env: +# GH_TOKEN token with issues:write on $REPO +# REPO owner/name +# TITLE exact issue title used as the upsert key +# BODY_FILE file whose contents become the issue body +# LABEL label to apply/create (default: ci-budget) +# MODE always | only-on-failure (default: always) +# STATUS ok | fail — consulted only when MODE=only-on-failure +# +# Never fails the caller: record-keeping must not turn a green run red. +set -euo pipefail + +: "${REPO:?set REPO}" +: "${TITLE:?set TITLE}" +: "${BODY_FILE:?set BODY_FILE}" +LABEL="${LABEL:-ci-budget}" +MODE="${MODE:-always}" +STATUS="${STATUS:-ok}" + +[ -f "$BODY_FILE" ] || { echo "::warning::$BODY_FILE missing — nothing to upsert"; exit 0; } + +# Ensure the label exists (idempotent; ignore "already exists"). +gh label create "$LABEL" --repo "$REPO" \ + --color 0e8a16 --description "Automated devkit record — do not close by hand" \ + >/dev/null 2>&1 || true + +# Find the existing record issue by exact title, open OR closed, so we reuse the +# same one forever instead of accumulating a new issue per run. +num=$(gh issue list --repo "$REPO" --label "$LABEL" --state all --limit 50 \ + --json number,title \ + --jq "[.[] | select(.title == \"$TITLE\")] | first | .number" 2>/dev/null || echo "") +[ "$num" = "null" ] && num="" + +if [ -z "$num" ]; then + if [ "$MODE" = "only-on-failure" ] && [ "$STATUS" = "ok" ]; then + echo "all green and no existing record issue — nothing to open" + exit 0 + fi + num=$(gh issue create --repo "$REPO" --title "$TITLE" \ + --body-file "$BODY_FILE" --label "$LABEL" \ + --json number --jq .number 2>/dev/null \ + || gh issue create --repo "$REPO" --title "$TITLE" \ + --body-file "$BODY_FILE" --label "$LABEL" | grep -oE '[0-9]+$') + echo "created record issue #$num" +else + gh issue edit "$num" --repo "$REPO" --body-file "$BODY_FILE" >/dev/null + echo "updated record issue #$num (body replaced, no comment, no notification)" +fi + +if [ "$MODE" = "only-on-failure" ]; then + if [ "$STATUS" = "ok" ]; then + gh issue close "$num" --repo "$REPO" \ + --comment "All plugins green against upstream — closing. This issue reopens automatically on the next red night." \ + >/dev/null 2>&1 || true + echo "green → record issue #$num closed" + else + gh issue reopen "$num" --repo "$REPO" >/dev/null 2>&1 || true + echo "red → record issue #$num open" + fi +fi + +echo "record: https://github.com/$REPO/issues/$num" diff --git a/e2e/ci/upstream-digest.sh b/e2e/ci/upstream-digest.sh new file mode 100755 index 0000000..7c5431e --- /dev/null +++ b/e2e/ci/upstream-digest.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Resolve the current manifest digest of an image on Docker Hub, anonymously. +# +# WHY THIS EXISTS — it is the whole cost argument for the nightly. +# A calendar-driven nightly re-runs 4 × ~14min e2e every night whether or not +# upstream changed. docker.io/agent0ai/agent-zero:latest does NOT change +# nightly. Gating the expensive fan-out on "did the digest actually move" +# turns ~1,700 runner-min/month into roughly the number of upstream releases +# × 56 min, plus a ~20s probe every night. Same detection latency, ~10% of the +# spend, and — unlike a calendar trigger — every expensive run corresponds to a +# real upstream change, so a red result is never ambiguous. +# +# Uses the anonymous pull token, so no Docker credentials are needed and this +# works on a fork PR. Prints the digest on stdout; writes GH outputs if +# GITHUB_OUTPUT is set. +# +# Env/args: +# IMAGE default docker.io/agent0ai/agent-zero:latest +set -euo pipefail + +IMAGE="${1:-${IMAGE:-docker.io/agent0ai/agent-zero:latest}}" + +ref="${IMAGE#docker.io/}" +repo="${ref%:*}" +tag="${ref##*:}" +[ "$tag" = "$repo" ] && tag=latest +case "$repo" in */*) ;; *) repo="library/$repo" ;; esac + +token=$(curl -fsS "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') + +digest=$(curl -fsSI -H "Authorization: Bearer ${token}" \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/${repo}/manifests/${tag}" \ + | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:"{print $2}') + +if [ -z "${digest:-}" ]; then + echo "::error::could not resolve digest for ${IMAGE}" >&2 + exit 1 +fi + +echo "$digest" +if [ -n "${GITHUB_OUTPUT:-}" ]; then + { + echo "digest=${digest}" + echo "digest_short=${digest#sha256:}" + echo "image=${IMAGE}" + } >> "$GITHUB_OUTPUT" +fi diff --git a/templates/ci-budget.caller.yml b/templates/ci-budget.caller.yml new file mode 100644 index 0000000..937b10d --- /dev/null +++ b/templates/ci-budget.caller.yml @@ -0,0 +1,37 @@ +# ci-budget — caller template. +# +# Copied into a consumer plugin repo at .github/workflows/ci-budget.yml by +# `make link-workflows`. Refreshes the plugin's CI budget / e2e record: +# a job summary, a ci-budget.json artifact, and one long-lived record issue +# updated in place (no commits, no PRs). +# +# Weekly, not per-PR: the record is a rolling 30-day view, so refreshing it on +# every PR would spend runner time to change a timestamp. It is also +# workflow_dispatch-able for an on-demand read. +name: ci-budget + +on: + schedule: + # Mondays 06:40 UTC. Off the hour (GitHub's :00 queues are the worst) and + # early enough that the record is fresh when the week starts. + - cron: "40 6 * * 1" + workflow_dispatch: + inputs: + window_days: + description: "Rolling window in days" + type: string + default: "30" + +permissions: + contents: read + actions: read + issues: write + +jobs: + budget: + uses: agent-zero-plugins/agent-zero-plugin-development-testkit/.github/workflows/ci-budget.yml@main + # Forwards BILLING_READ_TOKEN if the org has set it. Absent → the record + # honestly reports billed minutes as unavailable rather than guessing. + secrets: inherit + with: + window-days: ${{ fromJSON(inputs.window_days || '30') }}