diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 1da05c2..008cd10 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -4,16 +4,6 @@ "repo": "actions/setup-python", "version": "v7.0.0", "sha": "5fda3b95a4ea91299a34e894583c3862153e4b97" - }, - "github/gh-aw-actions/setup-cli@v0.87.2": { - "repo": "github/gh-aw-actions/setup-cli", - "version": "v0.87.2", - "sha": "b304200a0ef4b3998673bfc7945acb08ab8c88b7" - }, - "github/gh-aw-actions/setup@v0.87.2": { - "repo": "github/gh-aw-actions/setup", - "version": "v0.87.2", - "sha": "b304200a0ef4b3998673bfc7945acb08ab8c88b7" } }, "containers": { diff --git a/.github/graders/dependabot-release-train-updater-operational-value.sh b/.github/graders/dependabot-release-train-updater-operational-value.sh new file mode 100755 index 0000000..5488f4e --- /dev/null +++ b/.github/graders/dependabot-release-train-updater-operational-value.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +set -euo pipefail +export LC_ALL=C + +REPOSITORY=githubnext/central-agentic-ops +WORKFLOW_NAME="Dependabot / Release Train Updater" +MATURATION_SECONDS=1209600 + +tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/dependabot-release-train-value.XXXXXX") +trap 'rm -rf "$tmp_dir"' EXIT HUP INT TERM + +definition() { + cat <<'JSON' +{ + "schemaVersion": 4, "grader": "operational-value", + "repository": "githubnext/central-agentic-ops", "workflowName": "Dependabot / Release Train Updater", + "sourcePath": ".github/workflows/dependabot-release-train-updater.md", + "adoption": {"commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", "adoptedAt": "2026-08-18T17:54:55Z"}, + "operationalValue": "Resolve the dispatched target's matured dependency-update opportunities with validated merges.", + "evidence": { + "opportunity": "Dependency pull requests open in the dispatched target during the 30 days before the run.", + "assignment": "Bind targetRepo from workflow_dispatch inputs and freeze eligible pull-request numbers at the run creation time; key dependency-set::.", + "accepted": "An assigned pull request is merged by the evidence cutoff, changes a dependency manifest or lockfile, and satisfies every configured required status check.", + "repositories": ["githubnext/central-agentic-ops"], + "collection": "Query target pull requests and changed files once for assignment, then query immutable merge commits and required-check results through the capped cutoff.", + "maturation": "Fourteen days after the workflow run starts.", + "zeroRule": "Complete evidence for eligible assigned pull requests with no validated resolutions scores 0.", + "missingRule": "Missing target assignment, no eligible opportunity, inaccessible pull-request evidence, or unavailable required-check configuration scores null." + }, + "primaryMetric": {"id": "validated-resolution-share", "formula": "validated assigned dependency pull requests / eligible assigned dependency pull requests", "direction": "higher_is_better"}, + "baseline": {"mode": "attainment-only", "value": null, "evidenceCutoff": null, "provenance": []}, + "validationExamples": { + "targetAttained": {"valid": true, "eligible": 4, "validated": 4}, + "targetMissed": {"valid": true, "eligible": 4, "validated": 0}, + "missing": {"valid": false, "eligible": null, "validated": null}, + "malformed": {"valid": true, "eligible": 1, "validated": 2} + } +} +JSON +} + +metric() { + jq 'if .valid != true or (.eligible|type)!="number" or (.validated|type)!="number" + or .eligible<=0 or .validated<0 or .validated>.eligible then null + else ((.validated/.eligible)*1000000|round)/1000000 end' +} + +normalize_timestamp() { + jq -nr --arg value "$1" '($value|sub("\\.[0-9]+Z$";"Z")) as $v + | if ($v|test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) + and (try (($v|fromdateiso8601|todateiso8601)==$v) catch false) then $v else error("invalid") end' 2>/dev/null +} + +time_shift() { jq -nr --arg value "$1" --argjson seconds "$2" '$value|fromdateiso8601+$seconds|todateiso8601'; } +earlier() { jq -nr --arg left "$1" --arg right "$2" 'if ($left|fromdateiso8601)<($right|fromdateiso8601) then $left else $right end'; } + +emit_missing() { + jq -cn --arg key "$1" --argjson case "$2" --arg cutoff "$3" --arg matures "$4" --arg reason "$5" \ + '{value:null,opportunityKey:$key,case:$case,evidenceCutoff:$cutoff,maturesAt:$matures,provenance:[],diagnostics:{missingReason:$reason}}' +} + +dependency_file_regex='(^|/)(package(-lock)?\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lockb?|deno\.lock|requirements[^/]*\.txt|constraints[^/]*\.txt|Pipfile(\.lock)?|poetry\.lock|pyproject\.toml|uv\.lock|setup\.(py|cfg)|Gemfile(\.lock)?|go\.(mod|sum|work)|Cargo\.(toml|lock)|composer\.(json|lock)|mix\.(exs|lock)|pubspec\.(yaml|lock)|Package\.swift|Podfile(\.lock)?|packages?\.lock\.json|Directory\.Packages\.props|pom\.xml|build\.gradle(\.kts)?|gradle\.lockfile|MODULE\.bazel(\.lock)?|WORKSPACE|flake\.(nix|lock)|dependabot\.ya?ml)$' + +required_checks() { + target_repo=$1; branch=$2; output=$3 + if gh api "repos/$target_repo/branches/$branch/protection/required_status_checks" >"$tmp_dir/required-raw.json" 2>/dev/null; then + jq '[ + (.checks[]? | {context:.context,appId:(.app_id // null)}), + (.contexts[]? | {context:.,appId:null}) + ] | unique_by([.context,.appId])' "$tmp_dir/required-raw.json" >"$output" + return + fi + protected=$(gh api "repos/$target_repo/branches/$branch" --jq .protected 2>/dev/null) || return 1 + [[ $protected == false ]] || return 1 + printf '%s\n' '[]' >"$output" +} + +assign_case() { + request_file=$1 + target_repo=$(jq -r '.event.inputs.target_repo // empty' "$request_file") + [[ $target_repo =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || return 1 + created_at=$(jq -r .run.createdAt "$request_file") + window_start=$(time_shift "$created_at" -2592000) + gh api --paginate --method GET "repos/$target_repo/pulls" -f state=all -f sort=created -f direction=desc -f per_page=100 \ + | jq -s 'add // []' >"$tmp_dir/pulls.json" 2>/dev/null || return 1 + : >"$tmp_dir/opportunities.ndjson" + while IFS=$'\t' read -r number author created closed branch; do + [[ $created < $window_start || $created > $created_at ]] && continue + [[ -n $closed && $closed < $created_at ]] && continue + files="$tmp_dir/files-$number.json" + gh api --paginate "repos/$target_repo/pulls/$number/files?per_page=100" --jq '.[].filename' \ + | jq -Rsc 'split("\n")|map(select(length>0))' >"$files" 2>/dev/null || return 1 + is_dependency=$(jq --arg author "$author" --arg regex "$dependency_file_regex" \ + '($author|ascii_downcase|startswith("dependabot")) or any(.[];test($regex;"i"))' "$files") + [[ $is_dependency == true ]] || continue + required_file="$tmp_dir/required-$number.json" + required_checks "$target_repo" "$branch" "$required_file" || return 1 + jq -cn --argjson number "$number" --arg createdAt "$created" --arg baseRef "$branch" \ + --slurpfile required "$required_file" \ + '{number:$number,createdAt:$createdAt,baseRef:$baseRef,requiredChecks:$required[0]}' >>"$tmp_dir/opportunities.ndjson" + done < <(jq -r '.[]|[.number,(.user.login//""),.created_at,(.closed_at//""),.base.ref]|@tsv' "$tmp_dir/pulls.json") + opportunities=$(jq -s '.' "$tmp_dir/opportunities.ndjson") + jq -cn --arg targetRepo "$target_repo" --arg runId "$(jq -r .run.id "$request_file")" \ + --arg assignedAt "$created_at" --argjson opportunities "$opportunities" \ + '{targetRepo:$targetRepo,runId:$runId,assignedAt:$assignedAt,opportunities:$opportunities}' +} + +validated_pull() { + target_repo=$1; number=$2; cutoff=$3; required=$4 + gh api "repos/$target_repo/pulls/$number" >"$tmp_dir/pull-$number.json" 2>/dev/null || return 2 + merged_at=$(jq -r '.merged_at // empty' "$tmp_dir/pull-$number.json") + merge_sha=$(jq -r '.merge_commit_sha // empty' "$tmp_dir/pull-$number.json") + [[ -n $merged_at && -n $merge_sha ]] || return 1 + [[ $merged_at > $cutoff ]] && return 1 + [[ $(printf '%s\n' "$required" | jq -r 'type') == array ]] || return 2 + [[ $(printf '%s\n' "$required" | jq length) -gt 0 ]] || return 0 + gh api --paginate "repos/$target_repo/commits/$merge_sha/check-runs?per_page=100" --jq '.check_runs[]' | jq -s '.' >"$tmp_dir/checks-$number.json" 2>/dev/null || return 2 + gh api "repos/$target_repo/commits/$merge_sha/status" >"$tmp_dir/status-$number.json" 2>/dev/null || return 2 + jq -en --arg cutoff "$cutoff" --argjson required "$required" --slurpfile checks "$tmp_dir/checks-$number.json" --slurpfile statuses "$tmp_dir/status-$number.json" ' + [$checks[0][]|select(.conclusion=="success" and .completed_at <= $cutoff)|{context:.name,appId:(.app.id//null)}] as $checks + | [$statuses[0].statuses[]?|select(.state=="success" and .created_at <= $cutoff)|{context:.context,appId:null}] as $statuses + | all($required[] as $wanted; any(($checks+$statuses)[]; + .context==$wanted.context and ($wanted.appId==null or .appId==$wanted.appId)))' >/dev/null +} + +grade_run() { + request_file="$tmp_dir/request.json"; cat >"$request_file" + if ! jq -e '.schemaVersion==1 and (.run.id|type)=="string" and (.run.createdAt|type)=="string" and (.evidenceAt|type)=="string"' "$request_file" >/dev/null 2>&1; then + printf '%s\n' '{"value":null,"opportunityKey":"invalid-request","case":{"invalidRequest":true},"evidenceCutoff":"1970-01-01T00:00:00Z","maturesAt":"1970-01-01T00:00:00Z","provenance":[],"diagnostics":{"missingReason":"invalid request"}}'; return + fi + created_at=$(normalize_timestamp "$(jq -r .run.createdAt "$request_file")") || created_at=1970-01-01T00:00:00Z + evidence_at=$(normalize_timestamp "$(jq -r .evidenceAt "$request_file")") || evidence_at=$created_at + matures_at=$(time_shift "$created_at" "$MATURATION_SECONDS"); cutoff=$(earlier "$evidence_at" "$matures_at") + case_json=$(jq -c '.case' "$request_file") + if [[ $case_json == null ]]; then case_json=$(assign_case "$request_file") || case_json='{"assignmentMissing":true}'; fi + key="run:$(jq -r .run.id "$request_file")" + if [[ $(printf '%s\n' "$case_json"|jq -r '.assignmentMissing//false') == true ]]; then emit_missing "$key" "$case_json" "$cutoff" "$matures_at" assignment-unavailable; return; fi + target_repo=$(printf '%s\n' "$case_json"|jq -r .targetRepo); key="dependency-set:${target_repo}:$(printf '%s\n' "$case_json"|jq -r .runId)" + eligible=$(printf '%s\n' "$case_json"|jq '.opportunities|length') + [[ $eligible -gt 0 ]] || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" no-eligible-opportunities; return; } + validated=0; unavailable=false; : >"$tmp_dir/provenance.ndjson" + while IFS=$'\t' read -r number required; do + result=0; validated_pull "$target_repo" "$number" "$cutoff" "$required" || result=$? + [[ $result -eq 0 ]] && validated=$((validated+1)) + [[ $result -eq 2 ]] && unavailable=true + printf '{"repository":"%s","kind":"pull-request","ref":"%s"}\n' "$target_repo" "$number" >>"$tmp_dir/provenance.ndjson" + done < <(printf '%s\n' "$case_json"|jq -r '.opportunities[]|[.number,(.requiredChecks|tojson)]|@tsv') + [[ $unavailable == false ]] || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" required-check-evidence-unavailable; return; } + evidence=$(jq -cn --argjson eligible "$eligible" --argjson validated "$validated" '{valid:true,eligible:$eligible,validated:$validated}') + value=$(printf '%s\n' "$evidence"|metric); provenance=$(jq -s '.' "$tmp_dir/provenance.ndjson") + jq -cn --argjson value "$value" --arg key "$key" --argjson case "$case_json" --arg cutoff "$cutoff" --arg matures "$matures_at" --argjson provenance "$provenance" \ + '{value:$value,opportunityKey:$key,case:$case,evidenceCutoff:$cutoff,maturesAt:$matures,provenance:$provenance,diagnostics:{eligible:($case.opportunities|length),validated:($value*($case.opportunities|length))}}' +} + +case ${1:-} in + --definition) [[ $# -eq 1 ]] || exit 2; definition ;; + --metric) [[ $# -eq 1 ]] || exit 2; metric ;; + --grade-run) [[ $# -eq 1 ]] || exit 2; grade_run ;; + *) printf 'usage: %s --definition|--metric|--grade-run\n' "$0" >&2; exit 2 ;; +esac \ No newline at end of file diff --git a/.github/graders/optimization-ai-credit-auditor-operational-value.sh b/.github/graders/optimization-ai-credit-auditor-operational-value.sh new file mode 100755 index 0000000..923e72a --- /dev/null +++ b/.github/graders/optimization-ai-credit-auditor-operational-value.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export LC_ALL=C + +WORKFLOW_NAME="Optimization / AI Credit Auditor" +MATURATION_SECONDS=86400 + +tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/optimization-ai-credit-auditor-value.XXXXXX") +trap 'rm -rf "$tmp_dir"' EXIT HUP INT TERM + +definition() { + cat <<'JSON' +{ + "schemaVersion": 4, + "grader": "operational-value", + "repository": "githubnext/central-agentic-ops", + "workflowName": "Optimization / AI Credit Auditor", + "sourcePath": ".github/workflows/optimization-ai-credit-auditor.md", + "adoption": { + "commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", + "adoptedAt": "2026-08-18T17:54:55Z" + }, + "operationalValue": "Reproduce the dispatched target repository's completed agentic-workflow usage in a durable daily audit snapshot.", + "evidence": { + "opportunity": "One dispatched target repository and the 24-hour UTC period ending when its auditor run started, provided that period contains completed agentic-workflow runs.", + "assignment": "Bind targetRepo from workflow_dispatch inputs and auditDay from the run creation time; key target-day::. Replays retain this case.", + "accepted": "The durable repo-memory snapshot exactly matches retained completed-run totals and per-workflow aggregates for the assigned target and period.", + "repositories": ["githubnext/central-agentic-ops"], + "collection": "Read retained target workflow logs with gh aw and the assigned snapshot from the latest immutable repo-memory commit at the evidence cutoff.", + "maturation": "One day after the auditor run starts, matching the frozen daily observation contract.", + "zeroRule": "A readable snapshot that disagrees with complete retained run evidence scores 0.", + "missingRule": "Missing assignment, no completed-run opportunity, inaccessible logs, or an absent or malformed snapshot scores null." + }, + "primaryMetric": { + "id": "accurate-audit-day", + "formula": "1 when the assigned durable snapshot exactly reproduces eligible completed-run aggregates; 0 when it disagrees; null when either evidence side is unavailable.", + "direction": "higher_is_better" + }, + "baseline": { + "mode": "attainment-only", + "value": null, + "evidenceCutoff": null, + "provenance": [] + }, + "validationExamples": { + "targetAttained": {"valid": true, "eligible": true, "matched": true}, + "targetMissed": {"valid": true, "eligible": true, "matched": false}, + "missing": {"valid": false, "eligible": true, "matched": null}, + "malformed": {"valid": "yes", "eligible": true, "matched": true} + } +} +JSON +} + +metric() { + jq ' + if .valid != true or .eligible != true or (.matched | type) != "boolean" then null + elif .matched then 1 + else 0 + end + ' +} + +normalize_timestamp() { + jq -nr --arg value "$1" ' + ($value | sub("\\.[0-9]+Z$"; "Z")) as $timestamp + | if ($timestamp | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) + and (try (($timestamp | fromdateiso8601 | todateiso8601) == $timestamp) catch false) + then $timestamp else error("invalid timestamp") end + ' 2>/dev/null +} + +add_seconds() { + jq -nr --arg value "$1" --argjson seconds "$2" '$value | fromdateiso8601 + $seconds | todateiso8601' +} + +earlier_timestamp() { + jq -nr --arg left "$1" --arg right "$2" ' + if ($left | fromdateiso8601) < ($right | fromdateiso8601) then $left else $right end + ' +} + +emit_missing() { + opportunity_key=$1 + case_json=$2 + evidence_cutoff=$3 + matures_at=$4 + reason=$5 + jq -cn --arg key "$opportunity_key" --argjson case "$case_json" \ + --arg cutoff "$evidence_cutoff" --arg maturesAt "$matures_at" --arg reason "$reason" ' + {value: null, opportunityKey: $key, case: $case, evidenceCutoff: $cutoff, + maturesAt: $maturesAt, provenance: [], diagnostics: {missingReason: $reason}}' +} + +read_case() { + request_file=$1 + jq -c ' + if (.case | type) == "object" and (.case.targetRepo | type) == "string" then .case + elif (.event.inputs.target_repo | type) == "string" then { + targetRepo: .event.inputs.target_repo, + centralRepo: (.event.inputs.central_repo // .run.repository), + evidenceRepo: .run.repository, + auditDay: (.run.createdAt[0:10]), + windowStart: (.run.createdAt | fromdateiso8601 - 86400 | todateiso8601), + windowEnd: .run.createdAt + } + else {assignmentMissing: true} + end + ' "$request_file" +} + +collect_logs() { + target_repo=$1 + start_day=$2 + end_day=$3 + output=$4 + mkdir -p "$tmp_dir/logs" + gh aw logs --repo "$target_repo" --start-date "$start_day" --end-date "$end_day" \ + --count 10000 --json --output "$tmp_dir/logs" >"$output" 2>"$tmp_dir/logs.err" + jq -e '(.runs | type) == "array"' "$output" >/dev/null +} + +load_snapshot() { + evidence_repo=$1 + central_repo=$2 + target_repo=$3 + audit_day=$4 + cutoff=$5 + output=$6 + branch="memory/token-audit-${central_repo}-${target_repo}" + commits_file="$tmp_dir/commits.json" + gh api --paginate --method GET "repos/$evidence_repo/commits" \ + -f sha="$branch" -f until="$cutoff" -f per_page=100 | jq -s 'add // []' >"$commits_file" 2>/dev/null + commit_sha=$(jq -r 'sort_by(.commit.committer.date) | last | .sha // empty' "$commits_file") + [[ -n $commit_sha ]] || return 1 + gh api "repos/$evidence_repo/git/trees/$commit_sha?recursive=1" >"$tmp_dir/tree.json" 2>/dev/null + snapshot_name=$(printf '%s__%s__%s.json' "${target_repo%%/*}" "${target_repo#*/}" "$audit_day") + blob_sha=$(jq -r --arg name "$snapshot_name" ' + [.tree[]? | select(.type == "blob" and (.path | split("/") | last) == $name)] | first | .sha // empty + ' "$tmp_dir/tree.json") + [[ -n $blob_sha ]] || return 1 + gh api "repos/$evidence_repo/git/blobs/$blob_sha" --jq .content 2>/dev/null \ + | tr -d '\n' | base64 --decode >"$output" + jq -e '(.overall | type) == "object" and (.workflows | type) == "array" + and (.window_start | type) == "string" and (.window_end | type) == "string"' "$output" >/dev/null + printf '%s\n' "$commit_sha" +} + +compare_snapshot() { + runs_file=$1 + snapshot_file=$2 + window_start=$3 + window_end=$4 + jq --arg start "$window_start" --arg end "$window_end" ' + [.runs[] | select(.status == "completed" and .created_at >= $start and .created_at < $end)] + ' "$runs_file" >"$tmp_dir/completed.json" + completed_count=$(jq 'length' "$tmp_dir/completed.json") + [[ $completed_count -gt 0 ]] || return 2 + jq ' + def number: if type == "number" then . else 0 end; + def workflow: (.workflow_path // .workflow_name // "") | tostring; + { + overall: { + total_runs: length, + total_ai_credits: (map((.aic // 0) | number) | add // 0), + total_tokens: (map((.token_usage // 0) | number) | add // 0), + total_action_minutes: (map((.action_minutes // 0) | number) | add // 0) + }, + workflows: (group_by(workflow) | map({ + workflow_name: (.[0].workflow_name // (.[0] | workflow)), + workflow_path: (.[0] | workflow), run_count: length, + total_ai_credits: (map((.aic // 0) | number) | add // 0), + avg_ai_credits: ((map((.aic // 0) | number) | add // 0) / length), + total_tokens: (map((.token_usage // 0) | number) | add // 0), + avg_tokens: ((map((.token_usage // 0) | number) | add // 0) / length), + total_turns: (map((.turns // 0) | number) | add // 0), + avg_turns: ((map((.turns // 0) | number) | add // 0) / length), + total_action_minutes: (map((.action_minutes // 0) | number) | add // 0), + error_count: (map((.error_count // 0) | number) | add // 0), + warning_count: (map((.warning_count // 0) | number) | add // 0) + }) | sort_by(.workflow_path)) + } + ' "$tmp_dir/completed.json" >"$tmp_dir/expected.json" + jq -e --slurpfile expected "$tmp_dir/expected.json" ' + def close($left; $right): (($left | tonumber) - ($right | tonumber) | fabs) <= 0.000001; + .window_start == $start and .window_end == $end and .period_days == 1 + and .overall.total_runs == $expected[0].overall.total_runs + and close(.overall.total_ai_credits; $expected[0].overall.total_ai_credits) + and .overall.total_tokens == $expected[0].overall.total_tokens + and close(.overall.total_action_minutes; $expected[0].overall.total_action_minutes) + and ((.workflows | sort_by(.workflow_path)) as $actual + | $expected[0].workflows as $wanted + | ($actual | length) == ($wanted | length) + and all(range(0; $wanted | length); . as $index + | $actual[$index].workflow_path == $wanted[$index].workflow_path + and $actual[$index].run_count == $wanted[$index].run_count + and close($actual[$index].total_ai_credits; $wanted[$index].total_ai_credits) + and close($actual[$index].avg_ai_credits; $wanted[$index].avg_ai_credits) + and $actual[$index].total_tokens == $wanted[$index].total_tokens + and close($actual[$index].avg_tokens; $wanted[$index].avg_tokens) + and $actual[$index].total_turns == $wanted[$index].total_turns + and close($actual[$index].avg_turns; $wanted[$index].avg_turns) + and close($actual[$index].total_action_minutes; $wanted[$index].total_action_minutes) + and $actual[$index].error_count == $wanted[$index].error_count + and $actual[$index].warning_count == $wanted[$index].warning_count)) + ' "$snapshot_file" >/dev/null +} + +grade_run() { + request_file="$tmp_dir/request.json" + cat >"$request_file" + if ! jq -e '.schemaVersion == 1 and (.run.id | type) == "string" and (.run.createdAt | type) == "string" and (.evidenceAt | type) == "string"' "$request_file" >/dev/null 2>&1; then + printf '%s\n' '{"value":null,"opportunityKey":"invalid-request","case":{"invalidRequest":true},"evidenceCutoff":"1970-01-01T00:00:00Z","maturesAt":"1970-01-01T00:00:00Z","provenance":[],"diagnostics":{"missingReason":"invalid request"}}' + return + fi + created_at=$(normalize_timestamp "$(jq -r .run.createdAt "$request_file")") || created_at=1970-01-01T00:00:00Z + evidence_at=$(normalize_timestamp "$(jq -r .evidenceAt "$request_file")") || evidence_at=$created_at + matures_at=$(add_seconds "$created_at" "$MATURATION_SECONDS") + evidence_cutoff=$(earlier_timestamp "$evidence_at" "$matures_at") + case_json=$(read_case "$request_file") + if [[ $(printf '%s\n' "$case_json" | jq -r '.assignmentMissing // false') == true ]]; then + emit_missing "run:$(jq -r .run.id "$request_file")" "$case_json" "$evidence_cutoff" "$matures_at" assignment-unavailable + return + fi + target_repo=$(printf '%s\n' "$case_json" | jq -r .targetRepo) + central_repo=$(printf '%s\n' "$case_json" | jq -r .centralRepo) + evidence_repo=$(printf '%s\n' "$case_json" | jq -r '.evidenceRepo // empty') + audit_day=$(printf '%s\n' "$case_json" | jq -r .auditDay) + window_start=$(printf '%s\n' "$case_json" | jq -r .windowStart) + window_end=$(printf '%s\n' "$case_json" | jq -r .windowEnd) + opportunity_key="target-day:${target_repo}:${audit_day}" + if ! [[ $target_repo =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && $evidence_repo =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + emit_missing "$opportunity_key" "$case_json" "$evidence_cutoff" "$matures_at" invalid-target + return + fi + start_day=${window_start%%T*} + end_day=${window_end%%T*} + if ! collect_logs "$target_repo" "$start_day" "$end_day" "$tmp_dir/logs.json"; then + emit_missing "$opportunity_key" "$case_json" "$evidence_cutoff" "$matures_at" logs-unavailable + return + fi + if ! commit_sha=$(load_snapshot "$evidence_repo" "$central_repo" "$target_repo" "$audit_day" "$evidence_cutoff" "$tmp_dir/snapshot.json"); then + emit_missing "$opportunity_key" "$case_json" "$evidence_cutoff" "$matures_at" snapshot-unavailable + return + fi + matched=false + comparison_status=0 + compare_snapshot "$tmp_dir/logs.json" "$tmp_dir/snapshot.json" "$window_start" "$window_end" || comparison_status=$? + if [[ $comparison_status -eq 2 ]]; then + emit_missing "$opportunity_key" "$case_json" "$evidence_cutoff" "$matures_at" no-eligible-runs + return + elif [[ $comparison_status -eq 0 ]]; then + matched=true + fi + evidence=$(jq -cn --argjson matched "$matched" '{valid: true, eligible: true, matched: $matched}') + value=$(printf '%s\n' "$evidence" | metric) + jq -cn --argjson value "$value" --arg key "$opportunity_key" --argjson case "$case_json" \ + --arg cutoff "$evidence_cutoff" --arg maturesAt "$matures_at" --arg central "$evidence_repo" \ + --arg target "$target_repo" --arg commit "$commit_sha" ' + {value: $value, opportunityKey: $key, case: $case, evidenceCutoff: $cutoff, + maturesAt: $maturesAt, + provenance: [ + {repository: $central, kind: "repo-memory-commit", ref: $commit}, + {repository: $target, kind: "agentic-workflow-logs", ref: ($case.windowStart + ".." + $case.windowEnd)} + ], diagnostics: {matched: ($value == 1)}}' +} + +case ${1:-} in + --definition) [[ $# -eq 1 ]] || exit 2; definition ;; + --metric) [[ $# -eq 1 ]] || exit 2; metric ;; + --grade-run) [[ $# -eq 1 ]] || exit 2; grade_run ;; + *) printf 'usage: %s --definition|--metric|--grade-run\n' "$0" >&2; exit 2 ;; +esac \ No newline at end of file diff --git a/.github/graders/optimization-ai-credit-optimizer-operational-value.sh b/.github/graders/optimization-ai-credit-optimizer-operational-value.sh new file mode 100755 index 0000000..74a37d8 --- /dev/null +++ b/.github/graders/optimization-ai-credit-optimizer-operational-value.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +set -euo pipefail +export LC_ALL=C + +WORKFLOW_NAME="Optimization / AI Credit Optimizer" +MATURATION_SECONDS=604800 + +tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/optimization-ai-credit-optimizer-value.XXXXXX") +trap 'rm -rf "$tmp_dir"' EXIT HUP INT TERM + +definition() { + cat <<'JSON' +{ + "schemaVersion": 4, "grader": "operational-value", + "repository": "githubnext/central-agentic-ops", "workflowName": "Optimization / AI Credit Optimizer", + "sourcePath": ".github/workflows/optimization-ai-credit-optimizer.md", + "adoption": {"commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", "adoptedAt": "2026-08-18T17:54:55Z"}, + "operationalValue": "Lower the dispatched target's highest-AIC workflow cost without increasing its failure rate.", + "evidence": { + "opportunity": "The high-AIC target workflow selected by the optimizer run after its frozen eligibility and recent-optimization exclusions.", + "assignment": "Bind targetRepo from workflow_dispatch inputs and freeze the workflow_path recorded by this optimizer run; key target-workflow:::.", + "accepted": "During the seven days after assignment, the workflow's median successful-run AIC is lower than its pre-run median and its completed-run failure rate is no higher.", + "repositories": ["githubnext/central-agentic-ops"], + "collection": "Read retained gh aw run logs for the assigned target from seven days before the run through the capped evidence cutoff.", + "maturation": "Seven days after the optimizer run starts.", + "zeroRule": "Complete comparable evidence with no AIC reduction or a higher failure rate scores 0.", + "missingRule": "Missing assignment, no eligible workflow, inaccessible logs, or no successful and completed runs in either period scores null." + }, + "primaryMetric": {"id": "efficient-reliable-outcome", "formula": "1 when median successful-run AIC decreases and failure rate does not increase; otherwise 0", "direction": "higher_is_better"}, + "baseline": {"mode": "attainment-only", "value": null, "evidenceCutoff": null, "provenance": []}, + "validationExamples": { + "targetAttained": {"valid":true,"lowerAic":true,"reliabilityPreserved":true}, + "targetMissed": {"valid":true,"lowerAic":false,"reliabilityPreserved":true}, + "missing": {"valid":false,"lowerAic":null,"reliabilityPreserved":null}, + "malformed": {"valid":"yes","lowerAic":true,"reliabilityPreserved":true} + } +} +JSON +} + +metric() { jq 'if .valid!=true or (.lowerAic|type)!="boolean" or (.reliabilityPreserved|type)!="boolean" then null elif .lowerAic and .reliabilityPreserved then 1 else 0 end'; } +normalize_timestamp() { jq -nr --arg value "$1" '($value|sub("\\.[0-9]+Z$";"Z")) as $v|if ($v|test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) and (try (($v|fromdateiso8601|todateiso8601)==$v) catch false) then $v else error("invalid") end' 2>/dev/null; } +time_shift() { jq -nr --arg value "$1" --argjson seconds "$2" '$value|fromdateiso8601+$seconds|todateiso8601'; } +earlier() { jq -nr --arg left "$1" --arg right "$2" 'if ($left|fromdateiso8601)<($right|fromdateiso8601) then $left else $right end'; } +emit_missing() { jq -cn --arg key "$1" --argjson case "$2" --arg cutoff "$3" --arg matures "$4" --arg reason "$5" '{value:null,opportunityKey:$key,case:$case,evidenceCutoff:$cutoff,maturesAt:$matures,provenance:[],diagnostics:{missingReason:$reason}}'; } + +collect_logs() { + target_repo=$1; start_day=$2; end_day=$3; output=$4 + rm -rf "$tmp_dir/logs"; mkdir -p "$tmp_dir/logs" + gh aw logs --repo "$target_repo" --start-date "$start_day" --end-date "$end_day" --count 10000 --json --output "$tmp_dir/logs" >"$output" 2>"$tmp_dir/logs.err" + jq -e '(.runs|type)=="array"' "$output" >/dev/null +} + +assign_case() { + request_file=$1; target_repo=$(jq -r '.event.inputs.target_repo//empty' "$request_file") + [[ $target_repo =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || return 1 + run_id=$(jq -r .run.id "$request_file") + created_at=$(jq -r .run.createdAt "$request_file"); baseline_start=$(time_shift "$created_at" -604800) + log_prefix=$(printf '%s' "$target_repo" | sed 's|/|__|') + log_file="/tmp/gh-aw/repo-memory/default/${log_prefix}__optimization-log.json" + [[ -f $log_file ]] || return 1 + selected=$(jq -c --arg runId "$run_id" ' + [.[] | select((.optimizer_run_id | tostring) == $runId)] | last // null + ' "$log_file") + [[ $selected != null ]] || return 1 + workflow=$(printf '%s\n' "$selected" | jq -r '.workflow_path // empty') + [[ $workflow == .github/workflows/*.lock.yml ]] || return 1 + jq -cn --arg targetRepo "$target_repo" --arg assignedAt "$created_at" --arg baselineStart "$baseline_start" \ + --arg workflow "$workflow" --arg optimizerRunId "$run_id" \ + '{targetRepo:$targetRepo,assignedAt:$assignedAt,baselineStart:$baselineStart,workflow:$workflow,optimizerRunId:$optimizerRunId}' +} + +comparison() { + logs_file=$1; case_json=$2; cutoff=$3 + jq -c --argjson case "$case_json" --arg cutoff "$cutoff" ' + def completed: (.status=="completed" or (.conclusion|type)=="string"); + def median: sort as $v|($v|length) as $n|if $n==0 then null elif ($n%2)==1 then $v[($n/2|floor)] else (($v[$n/2-1]+$v[$n/2])/2) end; + [.runs[]|select((.workflow_path//.workflow_name)==$case.workflow and completed and .created_at >= $case.baselineStart and .created_at < $cutoff)] as $all + | [$all[]|select(.created_at < $case.assignedAt)] as $before + | [$all[]|select(.created_at >= $case.assignedAt)] as $after + | [$before[]|select(.conclusion=="success" and (.aic|type)=="number")|.aic] as $beforeSuccess + | [$after[]|select(.conclusion=="success" and (.aic|type)=="number")|.aic] as $afterSuccess + | if ($before|length)==0 or ($after|length)==0 or ($beforeSuccess|length)==0 or ($afterSuccess|length)==0 then {valid:false} + else ($beforeSuccess|median) as $beforeMedian|($afterSuccess|median) as $afterMedian + | (($before|map(select(.conclusion!="success"))|length)/($before|length)) as $beforeFailure + | (($after|map(select(.conclusion!="success"))|length)/($after|length)) as $afterFailure + | {valid:true,lowerAic:($afterMedian<$beforeMedian),reliabilityPreserved:($afterFailure<=$beforeFailure),beforeMedian:$beforeMedian,afterMedian:$afterMedian,beforeFailureRate:$beforeFailure,afterFailureRate:$afterFailure,runIds:(($before+$after)|map(.run_id|tostring)|unique)} end + ' "$logs_file" +} + +grade_run() { + request_file="$tmp_dir/request.json"; cat >"$request_file" + if ! jq -e '.schemaVersion==1 and (.run.id|type)=="string" and (.run.createdAt|type)=="string" and (.evidenceAt|type)=="string"' "$request_file" >/dev/null 2>&1; then + printf '%s\n' '{"value":null,"opportunityKey":"invalid-request","case":{"invalidRequest":true},"evidenceCutoff":"1970-01-01T00:00:00Z","maturesAt":"1970-01-01T00:00:00Z","provenance":[],"diagnostics":{"missingReason":"invalid request"}}'; return + fi + created_at=$(normalize_timestamp "$(jq -r .run.createdAt "$request_file")") || created_at=1970-01-01T00:00:00Z + evidence_at=$(normalize_timestamp "$(jq -r .evidenceAt "$request_file")") || evidence_at=$created_at + matures_at=$(time_shift "$created_at" "$MATURATION_SECONDS"); cutoff=$(earlier "$evidence_at" "$matures_at") + case_json=$(jq -c .case "$request_file"); [[ $case_json != null ]] || case_json=$(assign_case "$request_file") || case_json='{"assignmentMissing":true}' + key="run:$(jq -r .run.id "$request_file")" + [[ $(printf '%s\n' "$case_json"|jq -r '.assignmentMissing//false') != true ]] || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" assignment-unavailable; return; } + target_repo=$(printf '%s\n' "$case_json"|jq -r .targetRepo); workflow=$(printf '%s\n' "$case_json"|jq -r .workflow); optimizer_run_id=$(printf '%s\n' "$case_json"|jq -r '.optimizerRunId // empty'); key="target-workflow:${target_repo}:${workflow}:${optimizer_run_id}" + [[ $target_repo =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && $workflow == .github/workflows/*.lock.yml && $optimizer_run_id =~ ^[0-9]+$ ]] || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" invalid-assignment; return; } + collect_logs "$target_repo" "$(printf '%s\n' "$case_json"|jq -r .baselineStart|cut -dT -f1)" "${cutoff%%T*}" "$tmp_dir/all-logs.json" || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" logs-unavailable; return; } + evidence=$(comparison "$tmp_dir/all-logs.json" "$case_json" "$cutoff") + [[ $(printf '%s\n' "$evidence"|jq -r .valid) == true ]] || { emit_missing "$key" "$case_json" "$cutoff" "$matures_at" incomplete-comparable-evidence; return; } + value=$(printf '%s\n' "$evidence"|metric); provenance=$(printf '%s\n' "$evidence"|jq --arg repository "$target_repo" '[.runIds[]|{repository:$repository,kind:"actions-run",ref:.}]') + diagnostics=$(printf '%s\n' "$evidence"|jq 'del(.valid,.runIds,.lowerAic,.reliabilityPreserved)') + jq -cn --argjson value "$value" --arg key "$key" --argjson case "$case_json" --arg cutoff "$cutoff" --arg matures "$matures_at" --argjson provenance "$provenance" --argjson diagnostics "$diagnostics" \ + '{value:$value,opportunityKey:$key,case:$case,evidenceCutoff:$cutoff,maturesAt:$matures,provenance:$provenance,diagnostics:$diagnostics}' +} + +case ${1:-} in + --definition) [[ $# -eq 1 ]] || exit 2; definition ;; + --metric) [[ $# -eq 1 ]] || exit 2; metric ;; + --grade-run) [[ $# -eq 1 ]] || exit 2; grade_run ;; + *) printf 'usage: %s --definition|--metric|--grade-run\n' "$0" >&2; exit 2 ;; +esac \ No newline at end of file diff --git a/.github/ops-values/dependabot-release-train-updater.sh b/.github/ops-values/dependabot-release-train-updater.sh deleted file mode 100755 index e221936..0000000 --- a/.github/ops-values/dependabot-release-train-updater.sh +++ /dev/null @@ -1,441 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -readonly REPOSITORY="githubnext/central-agentic-ops" -readonly ADOPTION_COMMIT="35c7c3cbd319632f85784cce196e57c0f61db9a0" - -fail() { - printf 'error: %s\n' "$*" >&2 - exit 1 -} - -definition() { - jq -n \ - --arg repository "$REPOSITORY" \ - --arg adoption_commit "$ADOPTION_COMMIT" \ - '{ - schemaVersion: 3, - slug: "dependabot-release-train-updater", - sourcePath: ".github/workflows/dependabot-release-train-updater.md", - repository: $repository, - workflowName: "Dependabot / Release Train Updater", - adoption: { - commit: $adoption_commit, - adoptedAt: "2026-08-18T17:54:55Z", - baselineCommit: "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - baselineAt: "2026-08-18T12:20:21Z" - }, - evaluation: { - mode: "attainment-only" - }, - evidence: { - key: "dependabot-release-train-validated-resolutions-v1", - repositories: [$repository], - opportunity: "Matured dependency pull requests or security-alert opportunities in repositories named by immutable central workflow run display titles.", - filters: [ - "Dispatch targets are owner/repository names parsed from Dependabot / Release Train Updater Actions run display titles.", - "Opportunities are pull requests created in the window, matured for 14 days by observedAt, and classified from title, author, labels, and changed files.", - "Dependency opportunities are Dependabot-authored pull requests or pull requests changing dependency manifests or lockfiles.", - "Security opportunities have security labels, security title indicators, or Dependabot security indicators.", - "Validated resolutions are merged by windowEnd, change dependency manifests or lockfiles, and have successful evidence for every configured required check at the merge commit.", - "Unavailable required-check configuration or merge-commit check evidence cannot establish a validated resolution." - ], - collection: "Batch central Actions runs over the complete requested span, fetch each target pull-request population once for that span, classify locally, and fetch merge-commit check evidence only for eligible merged dependency candidates.", - window: { - durationDays: 30, - cadenceDays: 30, - maturationDays: 14 - } - }, - model: { - architecture: "Deterministic opportunity-normalized attainment shares", - recommendation: "Use the validated dependency resolution share as primary and the security-opportunity resolution share as a diagnostic; report missing when no eligible opportunity evidence exists.", - presentation: { - label: "Validated dependency resolution attainment", - betterLabel: "Higher validated resolution share" - } - }, - summary: { - nativeLabel: "Validated dependency resolution share" - }, - metrics: [ - { - id: "validated-resolution-share", - name: "Validated dependency resolution share", - role: "primary", - formula: "validatedResolutions / eligibleOpportunities", - direction: "increase", - presentation: { - name: "Validated dependency resolution share", - legendLabel: "Validated resolution", - transform: "identity" - } - }, - { - id: "security-resolution-share", - name: "Security-opportunity resolution share", - role: "diagnostic", - formula: "securityValidatedResolutions / securityEligibleOpportunities", - direction: "increase", - presentation: { - name: "Security-opportunity resolution share", - legendLabel: "Security resolution", - transform: "identity" - } - } - ], - validationExamples: { - targetAttained: { - status: "observed", - eligibleOpportunities: 4, - validatedResolutions: 4, - securityEligibleOpportunities: 2, - securityValidatedResolutions: 2 - }, - targetMissed: { - status: "observed", - eligibleOpportunities: 4, - validatedResolutions: 0, - securityEligibleOpportunities: 2, - securityValidatedResolutions: 0 - }, - missing: { - status: "missing", - eligibleOpportunities: null, - validatedResolutions: null, - securityEligibleOpportunities: null, - securityValidatedResolutions: null - }, - malformed: { - status: "observed", - eligibleOpportunities: 1, - validatedResolutions: 2, - securityEligibleOpportunities: "unknown", - securityValidatedResolutions: 0 - } - } - }' -} - -score_metric() { - metric_id=$1 - metric_evidence=$(cat) - printf '%s\n' "$metric_evidence" | jq -e . >/dev/null 2>&1 || { - printf 'null\n' - return - } - - case "$metric_id" in - validated-resolution-share) - printf '%s\n' "$metric_evidence" | jq ' - if .status != "observed" - or (.eligibleOpportunities | type) != "number" - or (.validatedResolutions | type) != "number" - or .eligibleOpportunities <= 0 - or .validatedResolutions < 0 - or .validatedResolutions > .eligibleOpportunities - or (.eligibleOpportunities | floor) != .eligibleOpportunities - or (.validatedResolutions | floor) != .validatedResolutions - then null - else ([1, (.validatedResolutions / .eligibleOpportunities)] | min) * 1000000 | round / 1000000 - end' - ;; - security-resolution-share) - printf '%s\n' "$metric_evidence" | jq ' - if .status != "observed" - or (.securityEligibleOpportunities | type) != "number" - or (.securityValidatedResolutions | type) != "number" - or .securityEligibleOpportunities <= 0 - or .securityValidatedResolutions < 0 - or .securityValidatedResolutions > .securityEligibleOpportunities - or (.securityEligibleOpportunities | floor) != .securityEligibleOpportunities - or (.securityValidatedResolutions | floor) != .securityValidatedResolutions - then null - else ([1, (.securityValidatedResolutions / .securityEligibleOpportunities)] | min) * 1000000 | round / 1000000 - end' - ;; - *) - fail "unknown metric: $metric_id" - ;; - esac -} - -dependency_path_filter='test("(^|/)(package(-lock)?\\.json|npm-shrinkwrap\\.json|yarn\\.lock|pnpm-lock\\.yaml|bun\\.lockb?|deno\\.lock|requirements[^/]*\\.txt|constraints[^/]*\\.txt|Pipfile(\\.lock)?|poetry\\.lock|pyproject\\.toml|uv\\.lock|setup\\.(py|cfg)|environment[^/]*\\.ya?ml|Gemfile(\\.lock)?|gems\\.lock(ed)?|go\\.(mod|sum|work)|Cargo\\.(toml|lock)|composer\\.(json|lock)|mix\\.(exs|lock)|pubspec\\.(yaml|lock)|Package\\.swift|Podfile(\\.lock)?|Cartfile(\\.resolved)?|packages?\\.lock\\.json|Directory\\.Packages\\.props|pom\\.xml|build\\.gradle(\\.kts)?|gradle\\.lockfile|dependencies\\.lock|MODULE\\.bazel(\\.lock)?|WORKSPACE|flake\\.(nix|lock)|renovate\\.json|dependabot\\.ya?ml)$"; "i")' - -api_json() { - output_file=$1 - shift - if gh api "$@" >"$output_file" 2>/dev/null; then - return 0 - fi - rm -f "$output_file" - return 1 -} - -collect_actions_runs() { - destination=$1 - span_start=$2 - span_end=$3 - endpoint="repos/$REPOSITORY/actions/runs?per_page=100&created=${span_start}..${span_end}" - gh api --paginate "$endpoint" --jq '.workflow_runs[]' 2>/dev/null | jq -s '.' >"$destination" -} - -collect_target_pulls() { - target=$1 - span_start=$2 - span_end=$3 - destination=$4 - query="repo:${target} is:pr created:${span_start}..${span_end}" - cursor=null - printf '[]\n' >"$destination" - - while :; do - page="$work_dir/graphql-page.json" - if ! gh api graphql \ - -f query='query($searchQuery: String!, $cursor: String) { search(query: $searchQuery, type: ISSUE, first: 100, after: $cursor) { pageInfo { hasNextPage endCursor } nodes { ... on PullRequest { number url title createdAt mergedAt baseRefName author { login } labels(first: 100) { nodes { name } } mergeCommit { oid } } } } }' \ - -f searchQuery="$query" \ - -F cursor="$cursor" >"$page" 2>/dev/null; then - return 1 - fi - jq -s '.[0] + [.[1].data.search.nodes[]]' "$destination" "$page" >"$destination.next" - mv "$destination.next" "$destination" - has_next=$(jq -r '.data.search.pageInfo.hasNextPage' "$page") - [[ $has_next == true ]] || break - cursor=$(jq -r '.data.search.pageInfo.endCursor' "$page") - done -} - -fetch_pull_files() { - target=$1 - number=$2 - destination=$3 - gh api --paginate "repos/$target/pulls/$number/files?per_page=100" --jq '.[].filename' 2>/dev/null | jq -Rsc 'split("\n") | map(select(length > 0))' >"$destination" -} - -checks_validate_resolution_uncached() { - target=$1 - branch=$2 - merge_commit=$3 - cache_key=$(printf '%s-%s' "$target" "$branch" | tr '/ :' '___') - required_file="$work_dir/required-$cache_key.json" - unavailable_file="$work_dir/required-$cache_key.unavailable" - - if [[ ! -f $required_file && ! -f $unavailable_file ]]; then - if ! api_json "$required_file" "repos/$target/branches/$branch/protection/required_status_checks"; then - : >"$unavailable_file" - fi - fi - [[ -f $required_file ]] || return 1 - - required_contexts=$(jq -c '[.checks[]?.context, .contexts[]?] | map(select(type == "string" and length > 0)) | unique' "$required_file") - [[ $(printf '%s\n' "$required_contexts" | jq 'length') -gt 0 ]] || return 1 - - checks_file="$work_dir/checks-$merge_commit.json" - statuses_file="$work_dir/statuses-$merge_commit.json" - gh api --paginate -H 'Accept: application/vnd.github+json' "repos/$target/commits/$merge_commit/check-runs?per_page=100" --jq '.check_runs[]' 2>/dev/null | jq -s '.' >"$checks_file" || return 1 - api_json "$statuses_file" "repos/$target/commits/$merge_commit/status" || return 1 - [[ $(jq 'length' "$checks_file") -gt 0 || $(jq '.statuses | length' "$statuses_file") -gt 0 ]] || return 1 - - jq -en \ - --argjson required "$required_contexts" \ - --slurpfile checks "$checks_file" \ - --slurpfile statuses "$statuses_file" ' - [$checks[0][] | select(.conclusion == "success") | .name] as $successfulChecks - | [$statuses[0].statuses[]? | select(.state == "success") | .context] as $successfulStatuses - | all($required[] as $context; (($successfulChecks + $successfulStatuses) | index($context)) != null)' >/dev/null -} - -checks_validate_resolution() { - target=$1 - branch=$2 - merge_commit=$3 - result_key=$(printf '%s-%s' "$target" "$merge_commit" | tr '/' '_') - result_file="$work_dir/validation-$result_key" - if [[ -f $result_file ]]; then - [[ $(cat "$result_file") == true ]] - return - fi - if checks_validate_resolution_uncached "$target" "$branch" "$merge_commit"; then - printf 'true\n' >"$result_file" - return 0 - fi - printf 'false\n' >"$result_file" - return 1 -} - -collect_batch() { - request_file="$work_dir/request.json" - cat >"$request_file" - jq -e ' - type == "array" and length > 0 - and all(.[]; - (.windowStart | type == "string" and fromdateiso8601) - and (.windowEnd | type == "string" and fromdateiso8601) - and (.observedAt | type == "string" and fromdateiso8601) - and (.windowStart | fromdateiso8601) < (.windowEnd | fromdateiso8601) - and (.windowEnd | fromdateiso8601) <= (.observedAt | fromdateiso8601))' "$request_file" >/dev/null \ - || fail "invalid batch request" - - span_start=$(jq -r 'map(.windowStart) | min' "$request_file") - span_end=$(jq -r 'map(.windowEnd) | max' "$request_file") - runs_file="$work_dir/actions-runs.json" - runs_available=true - if ! collect_actions_runs "$runs_file" "$span_start" "$span_end"; then - runs_available=false - printf '[]\n' >"$runs_file" - fi - - targets_file="$work_dir/targets.json" - jq --arg workflow "Dependabot / Release Train Updater" ' - [ .[] - | select(.name == $workflow or .workflow_name == $workflow) - | . as $run - | (($run.display_title // "") | split("\u00b7")) as $titleParts - | select(($titleParts | length) == 3) - | ($titleParts[1] | gsub("^\\s+|\\s+$"; "")) - | select(test("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")) - ] | unique' "$runs_file" >"$targets_file" - - target_records="$work_dir/target-records.json" - printf '[]\n' >"$target_records" - while IFS= read -r target; do - target_key=$(printf '%s' "$target" | tr '/' '_') - pulls_file="$work_dir/pulls-$target_key.json" - target_available=true - if ! collect_target_pulls "$target" "$span_start" "$span_end" "$pulls_file"; then - target_available=false - printf '[]\n' >"$pulls_file" - fi - jq -n --arg repository "$target" --argjson available "$target_available" --slurpfile pulls "$pulls_file" \ - '{repository: $repository, available: $available, pulls: $pulls[0]}' >"$work_dir/target-record.json" - jq -s '.[0] + [.[1]]' "$target_records" "$work_dir/target-record.json" >"$target_records.next" - mv "$target_records.next" "$target_records" - done < <(jq -r '.[]' "$targets_file") - - results_file="$work_dir/results.jsonl" - : >"$results_file" - window_count=$(jq 'length' "$request_file") - window_index=0 - while [[ $window_index -lt $window_count ]]; do - window_file="$work_dir/window-$window_index.json" - jq ".[$window_index]" "$request_file" >"$window_file" - window_start=$(jq -r '.windowStart' "$window_file") - window_end=$(jq -r '.windowEnd' "$window_file") - observed_at=$(jq -r '.observedAt' "$window_file") - cutoff_epoch=$(jq -nr --arg observed "$observed_at" '$observed | fromdateiso8601 - (14 * 86400)') - - opportunities_file="$work_dir/opportunities-$window_index.jsonl" - : >"$opportunities_file" - while IFS=$'\t' read -r target target_available number title created_at merged_at branch author merge_commit; do - [[ $target_available == true ]] || continue - created_epoch=$(jq -nr --arg value "$created_at" '$value | fromdateiso8601') - start_epoch=$(jq -nr --arg value "$window_start" '$value | fromdateiso8601') - end_epoch=$(jq -nr --arg value "$window_end" '$value | fromdateiso8601') - [[ $created_epoch -ge $start_epoch && $created_epoch -lt $end_epoch && $created_epoch -le $cutoff_epoch ]] || continue - - files_file="$work_dir/files-$(printf '%s-%s' "$target" "$number" | tr '/' '_').json" - files_unavailable="$files_file.unavailable" - if [[ ! -f $files_file && ! -f $files_unavailable ]] && ! fetch_pull_files "$target" "$number" "$files_file"; then - : >"$files_unavailable" - fi - if [[ -f $files_unavailable ]]; then - continue - fi - dependency_files=$(jq --arg filter "$dependency_path_filter" '[.[] | select(test("(^|/)(package(-lock)?\\.json|npm-shrinkwrap\\.json|yarn\\.lock|pnpm-lock\\.yaml|bun\\.lockb?|deno\\.lock|requirements[^/]*\\.txt|constraints[^/]*\\.txt|Pipfile(\\.lock)?|poetry\\.lock|pyproject\\.toml|uv\\.lock|setup\\.(py|cfg)|environment[^/]*\\.ya?ml|Gemfile(\\.lock)?|gems\\.lock(ed)?|go\\.(mod|sum|work)|Cargo\\.(toml|lock)|composer\\.(json|lock)|mix\\.(exs|lock)|pubspec\\.(yaml|lock)|Package\\.swift|Podfile(\\.lock)?|Cartfile(\\.resolved)?|packages?\\.lock\\.json|Directory\\.Packages\\.props|pom\\.xml|build\\.gradle(\\.kts)?|gradle\\.lockfile|dependencies\\.lock|MODULE\\.bazel(\\.lock)?|WORKSPACE|flake\\.(nix|lock)|renovate\\.json|dependabot\\.ya?ml)$"; "i"))]' "$files_file") - labels=$(jq -c --arg target "$target" --argjson number "$number" '.[] | select(.repository == $target) | .pulls[] | select(.number == $number) | [.labels.nodes[].name]' "$target_records") - is_dependabot=$(jq -nr --arg author "$author" '$author | ascii_downcase | startswith("dependabot")') - dependency_file_count=$(printf '%s\n' "$dependency_files" | jq 'length') - [[ $is_dependabot == true || $dependency_file_count -gt 0 ]] || continue - - is_security=$(jq -nr --arg title "$title" --arg author "$author" --argjson labels "$labels" ' - (($title | test("(^|[^a-z])(security|vulnerability|cve-[0-9]|ghsa-|dependabot alert)([^a-z]|$)"; "i")) - or any($labels[]; test("security|vulnerability|dependabot.*security"; "i")) - or (($author | ascii_downcase | startswith("dependabot")) and any($labels[]; test("security|vulnerability"; "i"))))') - validated=false - if [[ $merged_at != null && $merge_commit != null && $dependency_file_count -gt 0 ]]; then - merged_epoch=$(jq -nr --arg value "$merged_at" '$value | fromdateiso8601') - if [[ $merged_epoch -le $end_epoch ]] && checks_validate_resolution "$target" "$branch" "$merge_commit"; then - validated=true - fi - fi - jq -cn \ - --arg repository "$target" --argjson number "$number" --arg createdAt "$created_at" \ - --argjson security "$is_security" --argjson validated "$validated" --arg mergeCommit "$merge_commit" \ - '{repository: $repository, pullRequest: $number, createdAt: $createdAt, security: $security, validated: $validated, mergeCommit: (if $mergeCommit == "null" then null else $mergeCommit end)}' \ - >>"$opportunities_file" - done < <(jq -r '.[] | .repository as $repository | .available as $available | .pulls[] | [$repository, $available, .number, .title, .createdAt, (.mergedAt // "null"), .baseRefName, (.author.login // ""), (.mergeCommit.oid // "null")] | @tsv' "$target_records") - - jq -s '.' "$opportunities_file" >"$work_dir/opportunities-$window_index.json" - eligible=$(jq 'length' "$work_dir/opportunities-$window_index.json") - validated=$(jq '[.[] | select(.validated)] | length' "$work_dir/opportunities-$window_index.json") - security_eligible=$(jq '[.[] | select(.security)] | length' "$work_dir/opportunities-$window_index.json") - security_validated=$(jq '[.[] | select(.security and .validated)] | length' "$work_dir/opportunities-$window_index.json") - collection_status=observed - [[ $runs_available == true && $eligible -gt 0 ]] || collection_status=missing - - provenance_file="$work_dir/provenance-$window_index.json" - jq -n --arg repository "$REPOSITORY" --arg ref "$ADOPTION_COMMIT" '[{repository: $repository, kind: "frozen-contract", ref: $ref}]' >"$provenance_file" - jq --arg start "$window_start" --arg end "$window_end" --arg repository "$REPOSITORY" ' - [.[] | select(.created_at >= $start and .created_at < $end) | {repository: $repository, kind: "actions-run", ref: ("run:" + (.id | tostring))}]' "$runs_file" >"$work_dir/run-provenance.json" - jq -s '.[0] + .[1]' "$provenance_file" "$work_dir/run-provenance.json" >"$provenance_file.next" - mv "$provenance_file.next" "$provenance_file" - jq '[.[] | select(.mergeCommit != null) | {repository: .repository, kind: "merge-commit", ref: .mergeCommit}]' "$work_dir/opportunities-$window_index.json" >"$work_dir/merge-provenance.json" - jq -s '.[0] + .[1] | unique_by([.repository, .kind, .ref])' "$provenance_file" "$work_dir/merge-provenance.json" >"$provenance_file.next" - mv "$provenance_file.next" "$provenance_file" - - jq -n \ - --arg status "$collection_status" --arg windowStart "$window_start" --arg windowEnd "$window_end" --arg observedAt "$observed_at" \ - --argjson eligible "$eligible" --argjson validated "$validated" --argjson securityEligible "$security_eligible" --argjson securityValidated "$security_validated" \ - --slurpfile targets "$targets_file" --slurpfile opportunities "$work_dir/opportunities-$window_index.json" --slurpfile provenance "$provenance_file" ' - { - evidence: { - key: "dependabot-release-train-validated-resolutions-v1", - repositories: (["githubnext/central-agentic-ops"] + $targets[0] | unique), - opportunity: "Matured dependency pull requests or security-alert opportunities in dispatched targets", - filters: ["created in window", "matured 14 days", "Dependabot author or dependency file change", "validated merge with dependency file changes and successful configured required checks"], - collection: "Batched central Actions run discovery and one pull-request corpus per dispatched target over the complete requested span", - window: {durationDays: 30, cadenceDays: 30, maturationDays: 14, windowStart: $windowStart, windowEnd: $windowEnd, observedAt: $observedAt}, - status: $status, - eligibleOpportunities: (if $status == "observed" then $eligible else null end), - validatedResolutions: (if $status == "observed" then $validated else null end), - securityEligibleOpportunities: (if $status == "observed" then $securityEligible else null end), - securityValidatedResolutions: (if $status == "observed" then $securityValidated else null end), - targets: $targets[0], - opportunities: $opportunities[0] - }, - provenance: $provenance[0] - }' >>"$results_file" - window_index=$((window_index + 1)) - done - jq -s '.' "$results_file" -} - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -workspace_root=$(CDPATH= cd -- "$script_dir/../.." && pwd) -work_dir="$workspace_root/.aw-value-dependabot-release-train-updater.$$" -mkdir "$work_dir" -cleanup_dir=$work_dir -trap 'rm -rf "$cleanup_dir"' EXIT HUP INT TERM - -case ${1:-} in - --definition) - [[ $# -eq 1 ]] || fail "usage: $0 --definition" - definition - ;; - --collect-batch) - [[ $# -eq 1 ]] || fail "usage: $0 --collect-batch" - command -v gh >/dev/null 2>&1 || fail "gh is required for collection" - collect_batch - ;; - --metric) - [[ $# -eq 2 ]] || fail "usage: $0 --metric " - score_metric "$2" - ;; - '') - score_metric "validated-resolution-share" - ;; - *) - fail "usage: $0 [--definition|--collect-batch|--metric ]" - ;; -esac \ No newline at end of file diff --git a/.github/ops-values/optimization-ai-credit-auditor.sh b/.github/ops-values/optimization-ai-credit-auditor.sh deleted file mode 100755 index ee69077..0000000 --- a/.github/ops-values/optimization-ai-credit-auditor.sh +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repository='githubnext/central-agentic-ops' -workflow_slug='optimization-ai-credit-auditor' -workflow_file='optimization-ai-credit-auditor.lock.yml' - -definition() { - jq -cn ' - { - schemaVersion: 3, - slug: "optimization-ai-credit-auditor", - sourcePath: ".github/workflows/optimization-ai-credit-auditor.md", - repository: "githubnext/central-agentic-ops", - workflowName: "Optimization / AI Credit Auditor", - adoption: { - commit: "35c7c3cbd319632f85784cce196e57c0f61db9a0", - adoptedAt: "2026-08-18T17:54:55Z", - baselineCommit: "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - baselineAt: "2026-08-18T12:20:21Z" - }, - evaluation: {mode: "attainment-only"}, - evidence: { - key: "target-day-audit-aggregate-reproduction", - repositories: ["githubnext/central-agentic-ops"], - opportunity: "A target repository and UTC day containing at least one completed agentic workflow run, among targets immutably named by dispatched Optimization / AI Credit Auditor runs.", - filters: [ - "Discover targets only from central Actions run display_title values matching Token audit · owner/repo · mode.", - "Include only retained target runs with status completed and created_at within the target-day window.", - "Empty completed-run windows are ineligible.", - "A day is accurate only when the durable daily snapshot reproduces overall and per-workflow completed-run aggregates.", - "Absent or inaccessible retained logs and snapshots remain explicitly missing; numeric zero is accepted only from retained evidence." - ], - collection: "Batch central Actions discovery over the full request, fetch gh aw logs once per discovered target for the total date span, and read daily snapshots from immutable central repo-memory commits at or before observedAt.", - window: {durationDays: 1, cadenceDays: 1, maturationDays: 1} - }, - model: { - architecture: "Deterministic exact aggregate reproduction over paired retained run logs and durable daily snapshots", - recommendation: "Use the accurate audit-day share as primary; report completed-run and durable-history coverage separately so missing evidence is never averaged into accuracy.", - presentation: {label: "Accurate audit days", betterLabel: "Higher is better"} - }, - summary: {nativeLabel: "Accurate audit-day share"}, - metrics: [ - { - id: "accurate-audit-day-share", - name: "Accurate audit-day share", - role: "primary", - formula: "matched eligible target-days / eligible target-days with both retained completed-run logs and a readable durable snapshot", - direction: "increase", - presentation: {name: "Accurate audit-day share", legendLabel: "Accurate days", transform: "identity"} - }, - { - id: "completed-run-coverage", - name: "Completed-run coverage", - role: "diagnostic", - formula: "sum min(snapshot overall.total_runs, retained completed runs) / sum retained completed runs across paired eligible target-days", - direction: "increase", - presentation: {name: "Completed-run coverage", legendLabel: "Run coverage", transform: "identity"} - }, - { - id: "durable-history-coverage", - name: "Durable-history coverage", - role: "diagnostic", - formula: "eligible target-days with a readable durable snapshot / eligible target-days established from retained completed-run logs", - direction: "increase", - presentation: {name: "Durable-history coverage", legendLabel: "History coverage", transform: "identity"} - } - ], - validationExamples: { - targetAttained: { - status: "complete", - days: [{eligible: true, logsStatus: "available", snapshotStatus: "available", comparison: "matched", completedRuns: 4, snapshotRuns: 4}] - }, - targetMissed: { - status: "complete", - days: [ - {eligible: true, logsStatus: "available", snapshotStatus: "available", comparison: "mismatched", completedRuns: 4, snapshotRuns: 2}, - {eligible: true, logsStatus: "available", snapshotStatus: "missing", comparison: "missing", completedRuns: 3, snapshotRuns: null} - ] - }, - missing: {status: "missing", days: []}, - malformed: {status: "complete", days: [{eligible: "yes"}]} - } - }' -} - -score_metric() { - local metric_id=$1 - case $metric_id in - accurate-audit-day-share) - jq ' - if .status != "complete" or (.days | type) != "array" - or any(.days[]; (.eligible | type) != "boolean") - then null - else [.days[] | select(.eligible and .logsStatus == "available" and .snapshotStatus == "available")] - | if length == 0 or any(.[]; (.comparison != "matched" and .comparison != "mismatched")) then null - else (([.[] | select(.comparison == "matched")] | length) / length * 1000000 | round / 1000000) - end - end' - ;; - completed-run-coverage) - jq ' - if .status != "complete" or (.days | type) != "array" - or any(.days[]; (.eligible | type) != "boolean") - then null - else [.days[] | select(.eligible and .logsStatus == "available" and .snapshotStatus == "available")] - | if length == 0 - or any(.[]; (.completedRuns | type) != "number" or .completedRuns <= 0 - or (.snapshotRuns | type) != "number" or .snapshotRuns < 0) - then null - else (map(.completedRuns) | add) as $runs - | (map([.snapshotRuns, .completedRuns] | min) | add) / $runs - | . * 1000000 | round / 1000000 - end - end' - ;; - durable-history-coverage) - jq ' - if .status != "complete" or (.days | type) != "array" - or any(.days[]; (.eligible | type) != "boolean") - then null - else [.days[] | select(.eligible and .logsStatus == "available")] - | if length == 0 or any(.[]; (.snapshotStatus != "available" and .snapshotStatus != "missing")) then null - else (([.[] | select(.snapshotStatus == "available")] | length) / length * 1000000 | round / 1000000) - end - end' - ;; - *) - printf 'null\n' - ;; - esac -} - -empty_evidence() { - local request_file=$1 - local reason=$2 - jq -c --arg reason "$reason" ' - .[] | { - evidence: { - key: "target-day-audit-aggregate-reproduction", - repositories: ["githubnext/central-agentic-ops"], - opportunity: "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - filters: ["completed target runs", "exact durable aggregate reproduction", "empty run windows ineligible"], - collection: $reason, - window: {durationDays: 1, cadenceDays: 1, maturationDays: 1}, - status: "missing", - days: [] - }, - provenance: [{repository: "githubnext/central-agentic-ops", kind: "workflow-source-at-adoption", ref: "35c7c3cbd319632f85784cce196e57c0f61db9a0:.github/workflows/optimization-ai-credit-auditor.md"}], - commit: "35c7c3cbd319632f85784cce196e57c0f61db9a0" - }' "$request_file" | jq -s '.' -} - -collect_batch() { - local script_dir repo_root work_dir cleanup_command request_file - script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) - repo_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) - work_dir=$(mktemp -d "$repo_root/.aw-value-collector.XXXXXX") - printf -v cleanup_command 'rm -rf %q' "$work_dir" - trap "$cleanup_command" EXIT HUP INT TERM - request_file="$work_dir/request.json" - cat > "$request_file" - - if ! jq -e ' - type == "array" and all(.[]; - (.windowStart | type) == "string" and (.windowEnd | type) == "string" and (.observedAt | type) == "string" - and (try (.windowStart | fromdateiso8601) catch null) != null - and (try (.windowEnd | fromdateiso8601) catch null) != null - and (try (.observedAt | fromdateiso8601) catch null) != null) - ' "$request_file" >/dev/null 2>&1; then - empty_evidence "$request_file" "Invalid batch request; evidence was not collected." - return - fi - if [[ $(jq 'length' "$request_file") -eq 0 ]]; then - printf '[]\n' - return - fi - if ! command -v gh >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then - empty_evidence "$request_file" "Required collection command unavailable; evidence was not collected." - return - fi - - local actions_file min_dispatch max_observed - actions_file="$work_dir/actions.json" - min_dispatch=$(jq -r 'map(.windowEnd) | min' "$request_file") - max_observed=$(jq -r 'map(.observedAt) | max' "$request_file") - if ! gh api --paginate --method GET \ - "repos/$repository/actions/runs" \ - -f "created=$min_dispatch..$max_observed" -f event=workflow_dispatch -f per_page=100 \ - | jq -s '[.[].workflow_runs[]? | { - id, created_at, updated_at, status, conclusion, head_sha, - display_title, path, - target: (try (.display_title | capture("^Token audit · (?[^ ·]+/[^ ·]+) · ").target) catch null) - } | select(.path == ".github/workflows/optimization-ai-credit-auditor.lock.yml" and .target != null)]' > "$actions_file"; then - empty_evidence "$request_file" "Central Actions run discovery was inaccessible; evidence was not collected." - return - fi - - local targets_file - targets_file="$work_dir/targets.txt" - jq -r '.[].target' "$actions_file" | sort -u > "$targets_file" - - local logs_start logs_end target target_key logs_file - logs_start=$(jq -r 'map(.windowStart[0:10]) | min' "$request_file") - logs_end=$(jq -r 'map(.windowEnd[0:10]) | max' "$request_file") - while IFS= read -r target; do - [[ -n $target ]] || continue - target_key=$(printf '%s' "$target" | tr '/:' '___') - logs_file="$work_dir/logs-$target_key.json" - if gh aw logs --repo "$target" --start-date "$logs_start" --end-date "$logs_end" \ - --count 10000 --json --output "$work_dir/gh-aw-$target_key" > "$logs_file" 2> "$work_dir/logs-$target_key.err" \ - && jq -e '(.runs | type) == "array"' "$logs_file" >/dev/null 2>&1; then - jq -c '.runs' "$logs_file" > "$work_dir/runs-$target_key.json" - printf 'available\n' > "$work_dir/logs-status-$target_key" - else - printf '[]\n' > "$work_dir/runs-$target_key.json" - printf 'missing\n' > "$work_dir/logs-status-$target_key" - fi - done < "$targets_file" - - local results_file index window_start window_end observed_at dispatch_file - results_file="$work_dir/results.ndjson" - : > "$results_file" - index=0 - while [[ $index -lt $(jq 'length' "$request_file") ]]; do - window_start=$(jq -r ".[$index].windowStart" "$request_file") - window_end=$(jq -r ".[$index].windowEnd" "$request_file") - observed_at=$(jq -r ".[$index].observedAt" "$request_file") - dispatch_file="$work_dir/dispatch-$index.json" - jq --arg start "$window_end" --arg end "$observed_at" \ - '[.[] | select(.created_at > $start and .created_at <= $end)] | unique_by(.target)' \ - "$actions_file" > "$dispatch_file" - - local days_file provenance_file - days_file="$work_dir/days-$index.ndjson" - provenance_file="$work_dir/provenance-$index.ndjson" - : > "$days_file" - jq -c '.[] | {repository: "githubnext/central-agentic-ops", kind: "actions-run", ref: ("run:" + (.id | tostring) + "@" + .head_sha)}' \ - "$dispatch_file" > "$provenance_file" - - while IFS=$'\t' read -r target run_id run_sha dispatch_at; do - [[ -n $target ]] || continue - target_key=$(printf '%s' "$target" | tr '/:' '___') - local runs_file logs_status completed_file completed_count snapshot_date snapshot_name branch commits_file cutoff_sha tree_file blob_sha snapshot_file snapshot_status comparison snapshot_runs - runs_file="$work_dir/runs-$target_key.json" - logs_status=$(cat "$work_dir/logs-status-$target_key") - completed_file="$work_dir/completed-$index-$target_key.json" - if [[ $logs_status == available ]]; then - jq --arg start "$window_start" --arg end "$window_end" ' - [.[] | select(.status == "completed" and .created_at >= $start and .created_at < $end)]' \ - "$runs_file" > "$completed_file" - else - printf '[]\n' > "$completed_file" - fi - completed_count=$(jq 'length' "$completed_file") - snapshot_date=${dispatch_at%%T*} - snapshot_name=$(printf '%s__%s__%s.json' "${target%%/*}" "${target#*/}" "$snapshot_date") - branch="memory/token-audit-$target" - commits_file="$work_dir/commits-$target_key.json" - snapshot_status=missing - snapshot_runs=null - comparison=missing - cutoff_sha= - - if [[ ! -f $commits_file ]]; then - if gh api --paginate --method GET "repos/$repository/commits" \ - -f sha="$branch" -f until="$max_observed" -f per_page=100 \ - | jq -s 'add // []' > "$commits_file" 2>/dev/null; then - : - else - printf 'null\n' > "$commits_file" - fi - fi - if jq -e 'type == "array"' "$commits_file" >/dev/null 2>&1; then - cutoff_sha=$(jq -r --arg observed "$observed_at" \ - '[.[] | select(.commit.committer.date <= $observed)] | sort_by(.commit.committer.date) | last | .sha // empty' \ - "$commits_file") - if [[ -n $cutoff_sha ]]; then - tree_file="$work_dir/tree-$cutoff_sha.json" - if [[ ! -f $tree_file ]]; then - gh api "repos/$repository/git/trees/$cutoff_sha?recursive=1" > "$tree_file" 2>/dev/null || printf 'null\n' > "$tree_file" - fi - blob_sha=$(jq -r --arg name "$snapshot_name" ' - [.tree[]? | select(.type == "blob" and (.path | split("/") | last) == $name)] | first | .sha // empty' \ - "$tree_file" 2>/dev/null || true) - if [[ -n $blob_sha ]]; then - snapshot_file="$work_dir/blob-$blob_sha.json" - if [[ ! -f $snapshot_file ]]; then - gh api "repos/$repository/git/blobs/$blob_sha" --jq '.content' 2>/dev/null \ - | tr -d '\n' | base64 --decode > "$snapshot_file" 2>/dev/null || printf 'null\n' > "$snapshot_file" - fi - if jq -e '(.overall | type) == "object" and (.workflows | type) == "array"' "$snapshot_file" >/dev/null 2>&1; then - snapshot_status=available - snapshot_runs=$(jq '.overall.total_runs // null' "$snapshot_file") - else - snapshot_status=inaccessible - fi - fi - fi - else - snapshot_status=inaccessible - fi - - if [[ $logs_status == available && $completed_count -gt 0 && $snapshot_status == available ]]; then - local expected_file - expected_file="$work_dir/expected-$index-$target_key.json" - jq ' - def number: if type == "number" then . else 0 end; - def wf: (.workflow_name // "") | tostring; - { - overall: { - total_runs: length, - total_ai_credits: (map((.aic // 0) | number) | add // 0), - total_tokens: (map((.token_usage // 0) | number) | add // 0), - total_action_minutes: (map((.action_minutes // 0) | number) | add // 0) - }, - workflows: (group_by(wf) | map({ - workflow_name: (.[0] | wf), - run_count: length, - total_ai_credits: (map((.aic // 0) | number) | add // 0), - avg_ai_credits: ((map((.aic // 0) | number) | add // 0) / length), - total_tokens: (map((.token_usage // 0) | number) | add // 0), - avg_tokens: ((map((.token_usage // 0) | number) | add // 0) / length), - total_turns: (map((.turns // 0) | number) | add // 0), - avg_turns: ((map((.turns // 0) | number) | add // 0) / length), - total_action_minutes: (map((.action_minutes // 0) | number) | add // 0), - error_count: (map((.error_count // 0) | number) | add // 0), - warning_count: (map((.warning_count // 0) | number) | add // 0) - }) | sort_by(.workflow_name)) - }' "$completed_file" > "$expected_file" - if jq -e --slurpfile expected "$expected_file" ' - def close($a; $b): (($a | tonumber) - ($b | tonumber) | fabs) <= 0.000001; - (.overall.total_runs == $expected[0].overall.total_runs) - and close(.overall.total_ai_credits; $expected[0].overall.total_ai_credits) - and (.overall.total_tokens == $expected[0].overall.total_tokens) - and close(.overall.total_action_minutes; $expected[0].overall.total_action_minutes) - and ((.workflows | sort_by(.workflow_name)) as $actual - | ($expected[0].workflows) as $wanted - | ($actual | length) == ($wanted | length) - and all(range(0; $wanted | length); . as $i - | $actual[$i].workflow_name == $wanted[$i].workflow_name - and $actual[$i].run_count == $wanted[$i].run_count - and close($actual[$i].total_ai_credits; $wanted[$i].total_ai_credits) - and close($actual[$i].avg_ai_credits; $wanted[$i].avg_ai_credits) - and $actual[$i].total_tokens == $wanted[$i].total_tokens - and close($actual[$i].avg_tokens; $wanted[$i].avg_tokens) - and $actual[$i].total_turns == $wanted[$i].total_turns - and close($actual[$i].avg_turns; $wanted[$i].avg_turns) - and close($actual[$i].total_action_minutes; $wanted[$i].total_action_minutes) - and $actual[$i].error_count == $wanted[$i].error_count - and $actual[$i].warning_count == $wanted[$i].warning_count)) - ' "$snapshot_file" >/dev/null 2>&1; then comparison=matched; else comparison=mismatched; fi - fi - - jq -cn --arg target "$target" --arg date "$snapshot_date" --arg logs "$logs_status" \ - --arg snapshot "$snapshot_status" --arg comparison "$comparison" \ - --argjson completed "$completed_count" --argjson snapshot_runs "$snapshot_runs" ' - { - target: $target, - date: $date, - eligible: ($logs == "available" and $completed > 0), - logsStatus: $logs, - snapshotStatus: $snapshot, - comparison: $comparison, - completedRuns: $completed, - snapshotRuns: $snapshot_runs - }' >> "$days_file" - printf '{"repository":"%s","kind":"target-agentic-workflow-logs","ref":"actions-artifacts:%s:%s..%s"}\n' \ - "$target" "$target" "$window_start" "$window_end" >> "$provenance_file" - if [[ -n $cutoff_sha ]]; then - printf '{"repository":"%s","kind":"repo-memory-commit","ref":"%s"}\n' "$repository" "$cutoff_sha" >> "$provenance_file" - fi - done < <(jq -r '.[] | [.target, (.id | tostring), .head_sha, .created_at] | @tsv' "$dispatch_file") - - if [[ ! -s $provenance_file ]]; then - printf '{"repository":"%s","kind":"actions-workflow-query","ref":"%s:.github/workflows/%s"}\n' \ - "$repository" "35c7c3cbd319632f85784cce196e57c0f61db9a0" "$workflow_file" > "$provenance_file" - fi - jq -cn --slurpfile days <(jq -s '.' "$days_file") --slurpfile provenance <(jq -s 'unique_by(.repository, .kind, .ref)' "$provenance_file") ' - { - evidence: { - key: "target-day-audit-aggregate-reproduction", - repositories: (["githubnext/central-agentic-ops"] + [$days[0][].target] | unique), - opportunity: "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - filters: ["completed target runs", "exact durable aggregate reproduction", "empty run windows ineligible"], - collection: "Batched central Actions discovery, one gh aw logs fetch per target, immutable repo-memory snapshot lookup", - window: {durationDays: 1, cadenceDays: 1, maturationDays: 1}, - status: "complete", - days: $days[0] - }, - provenance: $provenance[0] - }' >> "$results_file" - index=$((index + 1)) - done - jq -s '.' "$results_file" -} - -case ${1:-} in - --definition) - [[ $# -eq 1 ]] || exit 2 - definition - ;; - --collect-batch) - [[ $# -eq 1 ]] || exit 2 - collect_batch - ;; - --metric) - [[ $# -eq 2 ]] || exit 2 - score_metric "$2" - ;; - '') - score_metric 'accurate-audit-day-share' - ;; - *) - exit 2 - ;; -esac \ No newline at end of file diff --git a/.github/ops-values/optimization-ai-credit-optimizer.sh b/.github/ops-values/optimization-ai-credit-optimizer.sh deleted file mode 100755 index 9ee1100..0000000 --- a/.github/ops-values/optimization-ai-credit-optimizer.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -readonly REPOSITORY="githubnext/central-agentic-ops" -readonly WORKFLOW_SLUG="optimization-ai-credit-optimizer" -readonly WORKFLOW_NAME="Optimization / AI Credit Optimizer" - -fail() { - printf 'error: %s\n' "$*" >&2 - exit 1 -} - -definition() { - jq -n ' - { - schemaVersion: 3, - slug: "optimization-ai-credit-optimizer", - sourcePath: ".github/workflows/optimization-ai-credit-optimizer.md", - repository: "githubnext/central-agentic-ops", - workflowName: "Optimization / AI Credit Optimizer", - adoption: { - commit: "35c7c3cbd319632f85784cce196e57c0f61db9a0", - adoptedAt: "2026-08-18T17:54:55Z", - baselineCommit: "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - baselineAt: "2026-08-18T12:20:21Z" - }, - evaluation: {mode: "attainment-only"}, - evidence: { - key: "same-workflow-half-window-aic-and-reliability", - repositories: ["githubnext/central-agentic-ops"], - opportunity: "A dispatched repository target in a seven-day window for which the completed first-half runs identify a highest-total-AIC workflow and that same workflow has completed and successful-run evidence in both halves.", - filters: [ - "Targets are parsed from immutable display_title values of completed Optimization / AI Credit Optimizer Actions runs in the central repository.", - "The target workflow is the workflow with highest total AIC among completed first-half runs, with workflow name ascending as the stable tie-break.", - "The selected workflow must have at least one completed run and at least one successful run in each half.", - "AIC medians use successful runs only; failure rates use all completed runs and classify every conclusion other than success as non-successful.", - "Second-half median AIC must be strictly lower and second-half failure rate must be no greater than first-half failure rate." - ], - collection: "Batch central Actions runs for all windows, deduplicate dispatched targets, invoke gh aw logs once per target for the total requested span, and derive each midpoint comparison locally from immutable target run IDs.", - window: {durationDays: 7, cadenceDays: 7, maturationDays: 7} - }, - model: { - architecture: "One eligible opportunity per dispatched target-window, scored on two independently reported attainment dimensions.", - recommendation: "Use efficient-and-reliable opportunity share as primary; retain lower-AIC share and reliability-preserved share as diagnostics. Reject recommendation and issue counts because they measure workflow output rather than repository outcome.", - presentation: { - label: "Efficient and reliable opportunities", - betterLabel: "Higher is better" - } - }, - summary: { - nativeLabel: "Share of comparable target-windows with lower median successful-run AIC and preserved failure rate" - }, - metrics: [ - { - id: "efficient-reliable-share", - name: "Efficient-and-reliable opportunity share", - role: "primary", - formula: "efficientReliableOpportunities / comparableOpportunities", - direction: "increase", - presentation: {name: "Efficient and reliable", legendLabel: "Efficient + reliable", transform: "identity"} - }, - { - id: "lower-aic-share", - name: "Lower-AIC opportunity share", - role: "diagnostic", - formula: "lowerAicOpportunities / comparableOpportunities", - direction: "increase", - presentation: {name: "Lower median AIC", legendLabel: "Lower AIC", transform: "identity"} - }, - { - id: "reliability-preserved-share", - name: "Reliability-preserved opportunity share", - role: "diagnostic", - formula: "reliabilityPreservedOpportunities / comparableOpportunities", - direction: "increase", - presentation: {name: "Reliability preserved", legendLabel: "Reliability preserved", transform: "identity"} - } - ], - validationExamples: { - targetAttained: { - status: "complete", comparableOpportunities: 1, - efficientReliableOpportunities: 1, lowerAicOpportunities: 1, - reliabilityPreservedOpportunities: 1 - }, - targetMissed: { - status: "complete", comparableOpportunities: 1, - efficientReliableOpportunities: 0, lowerAicOpportunities: 0, - reliabilityPreservedOpportunities: 0 - }, - missing: { - status: "missing", comparableOpportunities: 0, - efficientReliableOpportunities: null, lowerAicOpportunities: null, - reliabilityPreservedOpportunities: null - }, - malformed: {status: "complete", comparableOpportunities: "one"} - } - }' -} - -score_metric() { - local metric_id=$1 - local numerator_field - local evidence - case "$metric_id" in - efficient-reliable-share) numerator_field=efficientReliableOpportunities ;; - lower-aic-share) numerator_field=lowerAicOpportunities ;; - reliability-preserved-share) numerator_field=reliabilityPreservedOpportunities ;; - *) fail "unknown metric: $metric_id" ;; - esac - - evidence=$(cat) - printf '%s\n' "$evidence" | jq -e . >/dev/null 2>&1 || { printf 'null\n'; return; } - printf '%s\n' "$evidence" | jq --arg numerator "$numerator_field" ' - if type != "object" - or .status != "complete" - or (.comparableOpportunities | type) != "number" - or .comparableOpportunities <= 0 - or (.[$numerator] | type) != "number" - or .[$numerator] < 0 - or .[$numerator] > .comparableOpportunities - then null - else ((.[$numerator] / .comparableOpportunities) * 1000000 | round) / 1000000 - end' -} - -iso_day() { - printf '%s\n' "${1%%T*}" -} - -collect_batch() { - local request repo_root temp_dir cleanup_command min_start max_end start_day end_day log_end_day - request=$(cat) - printf '%s\n' "$request" | jq -e ' - type == "array" and all(.[]; - (.windowStart | type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) - and (.windowEnd | type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) - and (.observedAt | type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) - and ((.windowStart | fromdateiso8601) < (.windowEnd | fromdateiso8601)) - and ((.windowEnd | fromdateiso8601) <= (.observedAt | fromdateiso8601)) - )' >/dev/null || fail "invalid batch request" - - if [[ $(printf '%s\n' "$request" | jq 'length') -eq 0 ]]; then - printf '[]\n' - return - fi - - repo_root=$(cd "$(dirname "$0")/../.." && pwd) - temp_dir=$(mktemp -d "$repo_root/.aw-value-${WORKFLOW_SLUG}.XXXXXX") - printf -v cleanup_command 'rm -rf %q' "$temp_dir" - trap "$cleanup_command" EXIT - - min_start=$(printf '%s\n' "$request" | jq -r 'map(.windowStart) | min') - max_end=$(printf '%s\n' "$request" | jq -r 'map(.windowEnd) | max') - start_day=$(iso_day "$min_start") - end_day=$(iso_day "$max_end") - log_end_day=$(jq -nr --arg end "$max_end" '$end | fromdateiso8601 + 86400 | todateiso8601 | split("T")[0]') - - gh api --paginate --method GET \ - -H 'Accept: application/vnd.github+json' \ - "repos/$REPOSITORY/actions/runs?per_page=100&created=${start_day}..${end_day}" \ - --jq '.workflow_runs[] | {id, name, path, created_at, display_title, status, conclusion}' \ - | jq -s '.' >"$temp_dir/central-runs.json" - - jq --arg workflowName "$WORKFLOW_NAME" --arg workflowSlug "$WORKFLOW_SLUG" '[.[] - | select(.name == $workflowName or (.path | type == "string" and contains($workflowSlug))) - | select(.id != null and (.created_at | type == "string") and (.display_title | type == "string")) - | . + (try (.display_title | capture("(?[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)")) catch {}) - | select(.target != null)]' "$temp_dir/central-runs.json" >"$temp_dir/dispatches.json" - - local targets_file target target_index=0 - targets_file="$temp_dir/targets.txt" - printf '[]\n' >"$temp_dir/all-runs.json" - jq -r '.[].target' "$temp_dir/dispatches.json" | LC_ALL=C sort -u >"$targets_file" - while IFS= read -r target; do - [[ -n "$target" ]] || continue - target_index=$((target_index + 1)) - mkdir -p "$temp_dir/logs-$target_index" - gh aw logs --repo "$target" --start-date "$start_day" --end-date "$log_end_day" \ - --count 10000 --json --output "$temp_dir/logs-$target_index" \ - >"$temp_dir/logs-$target_index/stdout.json" - jq -s 'map(select(type == "object" or type == "array"))' \ - "$temp_dir/logs-$target_index/stdout.json" \ - "$temp_dir/logs-$target_index/summary.json" 2>/dev/null \ - >"$temp_dir/logs-$target_index/input.json" || \ - jq -s 'map(select(type == "object" or type == "array"))' \ - "$temp_dir/logs-$target_index/stdout.json" >"$temp_dir/logs-$target_index/input.json" - - jq --arg repository "$target" ' - def pick($names): . as $object | first($names[] as $name | $object[$name] | select(. != null)) // null; - [.. | objects - | . as $run - | (pick(["run_id", "runId", "database_id", "databaseId", "id"])) as $runId - | (pick(["workflow_name", "workflowName", "workflow", "workflow_id", "workflowId", "name"])) as $workflow - | (pick(["created_at", "createdAt", "started_at", "startedAt", "timestamp"])) as $createdAt - | (pick(["status"])) as $status - | (pick(["conclusion", "result"])) as $conclusion - | (pick(["aic", "total_aic", "totalAic", "ai_credits", "aiCredits", "estimated_aic", "estimatedAic", "credits", "cost"])) as $aic - | select(($runId | type) == "number" or ($runId | type) == "string") - | select(($workflow | type) == "string" and ($createdAt | type) == "string") - | {repository: $repository, runId: ($runId | tostring), workflow: $workflow, - createdAt: $createdAt, status: $status, conclusion: $conclusion, - aic: (if ($aic | type) == "number" then $aic - elif ($aic | type) == "string" then (try ($aic | tonumber) catch null) - else null end)}] - | unique_by(.runId)' "$temp_dir/logs-$target_index/input.json" \ - >"$temp_dir/logs-$target_index/runs.json" - jq -s 'add | unique_by(.repository + "#" + .runId)' \ - "$temp_dir/all-runs.json" "$temp_dir/logs-$target_index/runs.json" \ - >"$temp_dir/all-runs.next.json" - mv "$temp_dir/all-runs.next.json" "$temp_dir/all-runs.json" - done <"$targets_file" - - jq -n --argjson requests "$request" \ - --slurpfile dispatches "$temp_dir/dispatches.json" \ - --slurpfile runs "$temp_dir/all-runs.json" ' - def median: - sort as $values | ($values | length) as $length - | if $length == 0 then null - elif ($length % 2) == 1 then $values[($length / 2 | floor)] - else (($values[$length / 2 - 1] + $values[$length / 2]) / 2) end; - def completed: (.status == "completed" or (.conclusion | type) == "string"); - def target_result($target; $start; $midpoint; $end): - ($runs[0] | map(select(.repository == $target and completed - and (.createdAt | fromdateiso8601) >= $start - and (.createdAt | fromdateiso8601) < $end))) as $completedRuns - | ($completedRuns | map(select((.createdAt | fromdateiso8601) < $midpoint))) as $firstRuns - | ($completedRuns | map(select((.createdAt | fromdateiso8601) >= $midpoint))) as $secondRuns - | (if ($firstRuns | length) > 0 and all($firstRuns[]; (.aic | type) == "number") - then ($firstRuns | group_by(.workflow) - | map({workflow: .[0].workflow, totalAic: (map(.aic) | add)}) - | sort_by([-.totalAic, .workflow]) | first) - else null end) as $selection - | if $selection == null then - {target: $target, status: "missing", reason: "no-identifiable-first-half-workflow", - runIds: ($completedRuns | map(.runId) | unique)} - else - ($firstRuns | map(select(.workflow == $selection.workflow))) as $firstSelected - | ($secondRuns | map(select(.workflow == $selection.workflow))) as $secondSelected - | ($firstSelected | map(select(.conclusion == "success" and (.aic | type) == "number"))) as $firstSuccess - | ($secondSelected | map(select(.conclusion == "success" and (.aic | type) == "number"))) as $secondSuccess - | if ($firstSelected | length) == 0 or ($secondSelected | length) == 0 - or ($firstSuccess | length) == 0 or ($secondSuccess | length) == 0 then - {target: $target, status: "missing", workflow: $selection.workflow, - reason: "incomplete-comparable-run-evidence", - runIds: (($firstSelected + $secondSelected) | map(.runId) | unique)} - else - ($firstSuccess | map(.aic) | median) as $firstMedian - | ($secondSuccess | map(.aic) | median) as $secondMedian - | (($firstSelected | map(select(.conclusion != "success")) | length) / ($firstSelected | length)) as $firstFailureRate - | (($secondSelected | map(select(.conclusion != "success")) | length) / ($secondSelected | length)) as $secondFailureRate - | {target: $target, status: "complete", workflow: $selection.workflow, - firstHalf: {completedRuns: ($firstSelected | length), successfulRuns: ($firstSuccess | length), - medianAic: $firstMedian, failureRate: $firstFailureRate}, - secondHalf: {completedRuns: ($secondSelected | length), successfulRuns: ($secondSuccess | length), - medianAic: $secondMedian, failureRate: $secondFailureRate}, - lowerAic: ($secondMedian < $firstMedian), - reliabilityPreserved: ($secondFailureRate <= $firstFailureRate), - efficientReliable: (($secondMedian < $firstMedian) and ($secondFailureRate <= $firstFailureRate)), - runIds: (($firstSelected + $secondSelected) | map(.runId) | unique)} - end - end; - $requests | map(. as $window - | ($window.windowStart | fromdateiso8601) as $start - | ($window.windowEnd | fromdateiso8601) as $end - | ($start + (($end - $start) / 2)) as $midpoint - | ($dispatches[0] | map(select((.created_at | fromdateiso8601) >= ($window.windowStart | fromdateiso8601) - and (.created_at | fromdateiso8601) < ($window.windowEnd | fromdateiso8601)))) as $windowDispatches - | ($windowDispatches | map(.target) | unique) as $targets - | ($targets | map(target_result(.; $start; $midpoint; $end))) as $targetResults - | ($targetResults | map(select(.status == "complete"))) as $comparable - | { - evidence: { - key: "same-workflow-half-window-aic-and-reliability", - repositories: ($targets | if length == 0 then ["githubnext/central-agentic-ops"] else . end), - opportunity: "Dispatched target-window with a comparable highest-first-half-AIC workflow", - filters: ["completed runs", "successful-run AIC medians", "same workflow in both halves"], - collection: "Central immutable dispatch runs and one gh aw logs JSON download per target over the batch span", - window: {durationDays: 7, cadenceDays: 7, maturationDays: 7, - windowStart: $window.windowStart, windowEnd: $window.windowEnd, observedAt: $window.observedAt}, - status: (if ($comparable | length) > 0 then "complete" else "missing" end), - comparableOpportunities: ($comparable | length), - efficientReliableOpportunities: (if ($comparable | length) > 0 then ($comparable | map(select(.efficientReliable)) | length) else null end), - lowerAicOpportunities: (if ($comparable | length) > 0 then ($comparable | map(select(.lowerAic)) | length) else null end), - reliabilityPreservedOpportunities: (if ($comparable | length) > 0 then ($comparable | map(select(.reliabilityPreserved)) | length) else null end), - targets: $targets, - targetResults: $targetResults, - reason: (if ($comparable | length) > 0 then null elif ($targets | length) == 0 then "no-dispatched-target" else "no-comparable-matured-runs" end) - }, - provenance: (($windowDispatches | map({repository: "githubnext/central-agentic-ops", kind: "actions-run", ref: (.id | tostring)})) - + ($targetResults | map(. as $result | $result.runIds[]? | {repository: $result.target, kind: "actions-run", ref: tostring}))) - } - | if (.provenance | length) == 0 then - .provenance = [{repository: "githubnext/central-agentic-ops", kind: "commit", ref: "35c7c3cbd319632f85784cce196e57c0f61db9a0"}] - else . end - )' -} - -case ${1:-} in - --definition) - [[ $# -eq 1 ]] || fail "--definition takes no arguments" - definition - ;; - --metric) - [[ $# -eq 2 ]] || fail "usage: $0 --metric " - score_metric "$2" - ;; - --collect-batch) - [[ $# -eq 1 ]] || fail "--collect-batch takes no arguments" - collect_batch - ;; - "") - score_metric "efficient-reliable-share" - ;; - *) - fail "usage: $0 [--definition | --metric | --collect-batch]" - ;; -esac \ No newline at end of file diff --git a/.github/scripts/pages-report/operational-values.mjs b/.github/scripts/pages-report/operational-values.mjs new file mode 100644 index 0000000..0e70ce8 --- /dev/null +++ b/.github/scripts/pages-report/operational-values.mjs @@ -0,0 +1,272 @@ +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +const workerIds = new Set([ + "dependabot-release-train-updater", + "optimization-ai-credit-auditor", + "optimization-ai-credit-optimizer", +]); + +function downloadAgentArtifact(repository, runId, destination) { + return runCommand("gh", ["run", "download", String(runId), "--repo", repository, "--name", "agent", "--dir", destination]); +} + +function runCommand(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout = []; + const stderr = []; + child.stdout.on("data", (chunk) => stdout.push(chunk)); + child.stderr.on("data", (chunk) => stderr.push(chunk)); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code === 0 && !signal) { + resolve(Buffer.concat(stdout).toString("utf8")); + return; + } + reject(new Error(Buffer.concat(stderr).toString("utf8").trim() || `${command} exited with ${signal || code}`)); + }); + }); +} + +async function findResultsFile(directory) { + const entries = await readdir(directory, { recursive: true }); + const matches = entries.filter((entry) => entry.endsWith("grader_results.json")); + if (matches.length !== 1) return null; + return path.join(directory, matches[0]); +} + +async function mapWithConcurrency(values, concurrency, mapper) { + const results = new Array(values.length); + let nextIndex = 0; + async function worker() { + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await mapper(values[index]); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, worker)); + return results; +} + +function normalizeResult(selected, result, source = "run") { + const value = Number.isFinite(result.value) && result.value >= 0 && result.value <= 1 ? result.value : null; + return { + schemaVersion: 1, + repository: selected.repository, + workflowId: selected.workflowId, + runId: selected.runId, + runUrl: `https://github.com/${selected.repository}/actions/runs/${selected.runId}`, + status: result.status || "unavailable", + value, + baselineValue: Number.isFinite(result.baselineValue) ? result.baselineValue : null, + deltaFromBaseline: Number.isFinite(result.deltaFromBaseline) ? result.deltaFromBaseline : null, + evaluatorDigest: result.implementation?.digest || null, + observation: result.observation || null, + observationSource: source, + diagnostics: result.diagnostics || {}, + error: result.error || null, + }; +} + +function recordKey(record) { + return `${record.repository}:${record.runId}`; +} + +function observationTime(record) { + return Date.parse(record.observation?.evidenceAt || record.run?.createdAt || ""); +} + +function mergeRecords(cachedRecords, currentRecords, cutoff) { + const records = new Map(); + for (const record of [...cachedRecords, ...currentRecords]) { + const key = recordKey(record); + const existing = records.get(key); + if (existing?.observationSource === "regrade" && record.observationSource !== "regrade") continue; + if (existing?.observation && !record.observation) continue; + records.set(key, record); + } + return [...records.values()].filter((record) => { + const observedAt = observationTime(record); + return !Number.isFinite(observedAt) || observedAt >= cutoff; + }); +} + +function regradeDue(record, evidenceAt) { + const maturesAt = Date.parse(record.observation?.maturesAt || ""); + const unavailableReplay = record.observationSource === "regrade" + && (record.status === "unavailable" || record.value === null); + return record.observation + && (record.observation.mature !== true || unavailableReplay) + && Number.isFinite(maturesAt) + && maturesAt <= Date.parse(evidenceAt); +} + +async function regradeSupported() { + try { + await runGhAw(["graders", "operational-value", "--help"]); + return true; + } catch { + return false; + } +} + +function runGhAw(args, options = {}) { + const executable = process.env.REPORT_GH_AW_BIN; + return executable + ? runCommand(executable, args, options) + : runCommand("gh", ["aw", ...args], options); +} + +async function prepareTrustedCheckout(record, temporaryRoot) { + const repository = record.observation?.subject?.repository || record.repository; + const sha = record.observation?.subject?.sha; + if (!repository || !sha || !/^[0-9a-f]{40}$/i.test(sha)) { + throw new Error("operational-value observation has no trusted repository commit"); + } + const checkout = path.join(temporaryRoot, "checkouts", `${repository.replace("/", "-")}-${sha}`); + await mkdir(path.dirname(checkout), { recursive: true }); + await runCommand("gh", ["repo", "clone", repository, checkout, "--", "--filter=blob:none", "--no-checkout", "--depth=1"]); + try { + await runCommand("git", ["-C", checkout, "cat-file", "-e", `${sha}^{commit}`]); + } catch { + await runCommand("git", ["-C", checkout, "fetch", "--depth=1", "origin", sha]); + } + return checkout; +} + +async function regradeRecord(record, evidenceAt, checkout) { + const output = await runGhAw([ + "graders", "operational-value", String(record.runId), + "--repo", record.repository, + "--evidence-at", evidenceAt, + "--json", + ], { cwd: checkout }); + const artifact = JSON.parse(output); + const matches = (artifact.results || []).filter((result) => result.id === "operational-value" && result.source === "operational-value"); + if (artifact.version !== 1 || matches.length !== 1) throw new Error("unsupported operational-value regrade result"); + return { + ...normalizeResult(record, matches[0], "regrade"), + originalEvidenceAt: artifact.regrade?.originalEvidenceAt || record.observation?.evidenceAt || null, + regradedAt: evidenceAt, + }; +} + +(async () => { + const inventoryPath = process.env.REPORT_DEPLOYED_WORKFLOWS; + const outputPath = path.resolve(process.env.REPORT_OPERATIONAL_VALUES || "_inventory/operational-values.json"); + const cachePath = process.env.REPORT_VALUE_CACHE ? path.resolve(process.env.REPORT_VALUE_CACHE) : null; + const requestedConcurrency = Number(process.env.REPORT_VALUE_CONCURRENCY || 3); + const concurrency = Number.isInteger(requestedConcurrency) && requestedConcurrency > 0 + ? Math.min(requestedConcurrency, 8) + : 3; + if (!inventoryPath) throw new Error("REPORT_DEPLOYED_WORKFLOWS is required"); + + const generatedAt = new Date().toISOString(); + const inventory = JSON.parse(await readFile(inventoryPath, "utf8")); + const selectedRuns = []; + const seen = new Set(); + for (const workflow of inventory.workflows || []) { + const workflowId = workflow.path?.split("/").at(-1)?.replace(/\.lock\.yml$/, ""); + if (!workerIds.has(workflowId)) continue; + const runRecords = new Map((workflow.runHealth?.runRecords || []).map((run) => [Number(run.runId), run])); + for (const runId of workflow.runHealth?.runIds || []) { + const key = `${workflow.repository}:${runId}`; + if (seen.has(key)) continue; + seen.add(key); + selectedRuns.push({ + repository: workflow.repository, + runId: Number(runId), + workflowId, + run: runRecords.get(Number(runId)) || null, + }); + } + } + + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), "pages-operational-values-")); + try { + const currentRecords = await mapWithConcurrency(selectedRuns, concurrency, async (selected) => { + const destination = path.join(temporaryRoot, `${selected.repository.replace("/", "-")}-${selected.runId}`); + await mkdir(destination, { recursive: true }); + try { + await downloadAgentArtifact(selected.repository, selected.runId, destination); + const resultsPath = await findResultsFile(destination); + if (!resultsPath) return { ...selected, status: "unavailable", reason: "grader results not found" }; + const artifact = JSON.parse(await readFile(resultsPath, "utf8")); + if (artifact.version !== 1 || !Array.isArray(artifact.results)) { + return { ...selected, status: "unavailable", reason: "unsupported grader results" }; + } + const matches = artifact.results.filter((result) => result.id === "operational-value" && result.source === "operational-value"); + if (matches.length !== 1) return { ...selected, status: "unavailable", reason: "operational-value result not found" }; + return normalizeResult(selected, matches[0]); + } catch (error) { + console.warn(`Operational value unavailable for ${selected.repository} run ${selected.runId}: ${error.message}`); + return { ...selected, status: "unavailable", reason: error.message }; + } + }); + + let cachedRecords = []; + if (cachePath) { + try { + const cached = JSON.parse(await readFile(cachePath, "utf8")); + if (cached.schemaVersion === 1 && Array.isArray(cached.records)) cachedRecords = cached.records; + } catch (error) { + if (error.code !== "ENOENT") console.warn(`Ignoring operational-value cache: ${error.message}`); + } + } + const retentionCutoff = Date.parse(generatedAt) - 90 * 24 * 60 * 60 * 1000; + let records = mergeRecords(cachedRecords, currentRecords, retentionCutoff); + const replayAvailable = await regradeSupported(); + const dueRecords = records.filter((record) => regradeDue(record, generatedAt)); + if (replayAvailable && dueRecords.length) { + const checkouts = new Map(); + const replayed = await mapWithConcurrency(dueRecords, concurrency, async (record) => { + const checkoutKey = `${record.observation.subject?.repository || record.repository}:${record.observation.subject?.sha || ""}`; + if (!checkouts.has(checkoutKey)) { + checkouts.set(checkoutKey, prepareTrustedCheckout(record, temporaryRoot)); + } + try { + return await regradeRecord(record, generatedAt, await checkouts.get(checkoutKey)); + } catch (error) { + console.warn(`Operational-value regrade unavailable for ${record.repository} run ${record.runId}: ${error.message}`); + return { ...record, regradeAttemptedAt: generatedAt, regradeError: error.message }; + } + }); + const replayedByRun = new Map(replayed.map((record) => [recordKey(record), record])); + records = records.map((record) => replayedByRun.get(recordKey(record)) || record); + } + + const output = { + schemaVersion: 1, + generatedAt, + windowStart: inventory.runHealth?.windowStart || null, + windowHours: inventory.runHealth?.windowHours || null, + selectedRuns: selectedRuns.length, + observedRuns: records.filter((record) => record.observation).length, + matureRuns: records.filter((record) => record.observation?.mature).length, + regradedRuns: records.filter((record) => record.observationSource === "regrade").length, + pendingRegrades: records.filter((record) => regradeDue(record, generatedAt)).length, + regradeAvailable: replayAvailable, + records, + }; + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`); + if (cachePath) { + await mkdir(path.dirname(cachePath), { recursive: true }); + await writeFile(cachePath, `${JSON.stringify(output, null, 2)}\n`); + } + console.log(`Collected ${output.observedRuns} operational-value observations from ${output.selectedRuns} worker runs`); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/.github/scripts/pages-report/report.mjs b/.github/scripts/pages-report/report.mjs index e400c59..9d289df 100644 --- a/.github/scripts/pages-report/report.mjs +++ b/.github/scripts/pages-report/report.mjs @@ -1,4 +1,4 @@ -import { copyFile, mkdir, readdir, writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; @@ -9,9 +9,9 @@ const token = process.env.GITHUB_TOKEN; const pagesToken = process.env.REPORT_PAGES_TOKEN || token; const outputDirectory = process.env.REPORT_OUTPUT || "_site"; const inventoryPath = process.env.REPORT_INVENTORY; -const valueReportRoot = process.env.REPORT_VALUE_ROOT || ".github/value"; const deployedWorkflowsPath = process.env.REPORT_DEPLOYED_WORKFLOWS || "_inventory/deployed-workflows.json"; const aicUsagePath = process.env.REPORT_AIC_USAGE || "_inventory/aic-usage.json"; +const operationalValuesPath = process.env.REPORT_OPERATIONAL_VALUES || "_inventory/operational-values.json"; if (!repository || !token || !inventoryPath) { throw new Error("GITHUB_REPOSITORY, GITHUB_TOKEN, and REPORT_INVENTORY are required"); @@ -27,6 +27,9 @@ const deployedInventory = existsSync(deployedWorkflowsPath) const aicUsage = existsSync(aicUsagePath) ? JSON.parse(readFileSync(aicUsagePath, "utf8")) : { schemaVersion: 1, available: false, complete: false, repositories: [], runs: [] }; +const operationalValues = existsSync(operationalValuesPath) + ? JSON.parse(readFileSync(operationalValuesPath, "utf8")) + : { schemaVersion: 1, selectedRuns: 0, observedRuns: 0, records: [] }; const allowedRepositories = new Set((process.env.REPORT_ALLOWED_REPOS || "").split(",") .map((value) => value.trim().toLowerCase()).filter(Boolean)); if (inventory.schemaVersion !== 1 || !Array.isArray(inventory.workflows) || !Array.isArray(inventory.bundles)) { @@ -37,30 +40,37 @@ const standaloneDefinitions = inventory.standalone; const workflowDefinitionById = new Map(inventory.workflows.map((workflow) => [workflow.id, workflow])); const workerDefinitions = bundleDefinitions.flatMap((bundle) => bundle.workers.map((worker) => ({ ...worker, bundleId: bundle.id, bundleName: bundle.name }))); const workerIds = new Set(workerDefinitions.map((worker) => worker.id)); - -async function loadValueTimelines() { - const timelines = new Map(); - let paths = []; - try { - paths = await readdir(valueReportRoot, { recursive: true }); - } catch (error) { - if (error.code !== "ENOENT") throw error; - } - for (const relativePath of paths.filter((candidate) => candidate.endsWith("-timeline.json"))) { - const timelinePath = path.join(valueReportRoot, relativePath); - const timeline = JSON.parse(readFileSync(timelinePath, "utf8")); - if (timeline.schemaVersion !== 2 || !workerIds.has(timeline.workflowSlug) || !Array.isArray(timeline.snapshots) || timeline.snapshots.length < 2) continue; - timelines.set(timeline.workflowSlug, { - timeline, - timelinePath, - svgPath: timelinePath.replace(/-timeline\.json$/, "-timeline.svg"), - definitionsPath: timelinePath.replace(/-timeline\.json$/, "-definitions.md"), - }); +if (operationalValues.schemaVersion !== 1 || !Array.isArray(operationalValues.records)) { + throw new Error(`Unsupported or invalid operational-value observations: ${operationalValuesPath}`); +} +function comparableValueObservations(records) { + const valid = records + .filter((record) => record.observation + && record.status === "pass" + && Number.isFinite(record.value) + && typeof record.evaluatorDigest === "string" + && record.evaluatorDigest.length > 0 + && typeof record.observation.opportunityKey === "string" + && record.observation.opportunityKey.length > 0); + const latestEvaluator = valid.reduce((latest, record) => { + const assignedAt = Date.parse(record.observation.subject?.createdAt || record.originalEvidenceAt || record.observation.evidenceAt || ""); + return !latest || assignedAt > latest.assignedAt ? { digest: record.evaluatorDigest, assignedAt } : latest; + }, null); + if (!latestEvaluator) return []; + const opportunities = new Map(); + for (const record of valid.filter((candidate) => candidate.evaluatorDigest === latestEvaluator.digest)) { + const key = `${record.repository}:${record.observation.opportunityKey}`; + const existing = opportunities.get(key); + const observedAt = Date.parse(record.observation.evidenceAt || ""); + const existingAt = Date.parse(existing?.observation?.evidenceAt || ""); + if (!existing || observedAt >= existingAt) opportunities.set(key, record); } - return timelines; + return [...opportunities.values()] + .sort((left, right) => String(left.observation.evidenceAt).localeCompare(String(right.observation.evidenceAt))); } - -const valueTimelines = await loadValueTimelines(); +const valueObservations = new Map([...workerIds].map((workerId) => [workerId, + comparableValueObservations(operationalValues.records.filter((record) => record.workflowId === workerId)), +])); const reportDefinitions = [ ...bundleDefinitions, ...standaloneDefinitions.map((workflow) => ({ ...workflow, workers: [], missingWorkers: [] })), @@ -1268,42 +1278,33 @@ for (const [repositoryName, workflows] of deployedByRepository) { })); } -function presentedMetric(value, transform) { - if (!Number.isFinite(value)) return null; - return transform === "complement" ? 1 - value : value; -} - function formatGoalMeasure(value) { return value === null ? "Not observed" : new Intl.NumberFormat("en", { style: "percent", maximumFractionDigits: 1 }).format(value); } -function valueReportContent(worker, artifact, assetName) { - if (!artifact) { +function valueReportContent(worker, observations) { + if (!observations?.length) { return `

${escapeHtml(worker.name)}

${escapeHtml(worker.id)}

Not evaluated
-
${octicon("graph")}

No evaluation observations yet

Publish the canonical .github/value/${escapeHtml(worker.id)}/${escapeHtml(worker.id)}-timeline.json and sibling SVG artifacts to show this worker's value trend.

-
Metric details unavailable
+
${octicon("graph")}

No workflow observations yet

Operational value will appear after this worker publishes a valid grader_results.json.

+
Run evidence unavailable
`; } - const timeline = artifact.timeline; - const metrics = timeline.metricReview.metrics; - const primaryMetric = metrics.find((metric) => metric.role === "primary"); - const latestSnapshot = timeline.snapshots.at(-1); - const latestPrimary = primaryMetric ? presentedMetric(latestSnapshot.metrics[primaryMetric.id], primaryMetric.presentation?.transform) : null; - const mode = timeline.evaluationMode || "baseline-comparable"; - const metricRows = metrics.map((metric) => { - const latestValue = presentedMetric(latestSnapshot.metrics[metric.id], metric.presentation?.transform); - return `${escapeHtml(metric.presentation?.name || metric.name)}${escapeHtml(metric.role)}${escapeHtml(formatGoalMeasure(latestValue))}`; + const latest = observations.at(-1); + const mature = observations.filter((record) => record.observation.mature); + const matureAverage = mature.length ? mature.reduce((sum, record) => sum + record.value, 0) / mature.length : null; + const observationRows = [...observations].reverse().map((record) => { + const observation = record.observation; + const target = observation.case?.targetRepo || observation.subject?.repository || record.repository; + return `${escapeHtml(target)}${escapeHtml(observation.opportunityKey)}${escapeHtml(formatGoalMeasure(record.value))}${observation.mature ? "Mature" : "As of run"}`; }).join("\n"); - const observationRows = [...timeline.snapshots].reverse().map((snapshot) => `${metrics.map((metric) => `${escapeHtml(formatGoalMeasure(presentedMetric(snapshot.metrics[metric.id], metric.presentation?.transform)))}`).join("")}`).join("\n"); return `
-

${escapeHtml(timeline.workflowName || worker.name)}

${escapeHtml(timeline.summary?.nativeLabel || primaryMetric?.name || "Operational value attainment")}

${escapeHtml(formatGoalMeasure(latestPrimary))}${mode === "attainment-only" ? "Latest attainment" : "Latest goal measure"}
-
${escapeHtml(timeline.workflowName || worker.name)} value-function metrics over time
+

${escapeHtml(worker.name)}

Run-scoped attainment from the workflow's frozen operational-value evaluator.

${escapeHtml(formatGoalMeasure(latest.value))}Latest observation
+
Latest
${escapeHtml(formatGoalMeasure(latest.value))}
Mature average
${escapeHtml(formatGoalMeasure(matureAverage))}
Opportunities
${observations.length}
Evaluator
${escapeHtml(latest.evaluatorDigest?.slice(0, 12) || "Unavailable")}
- View metric details + View run evidence
-

Latest measures

${mode === "attainment-only" ? "Post-adoption attainment; no comparable pre-adoption baseline is available." : "Baseline-comparable measures before and after adoption."}

${metricRows}
MeasureRoleLatest value
-

Dated observations

${metrics.map((metric) => ``).join("")}${observationRows}
Observed${escapeHtml(metric.presentation?.name || metric.name)}
+

Workflow observations

Only the latest evaluator version is aggregated. Repeated runs for one opportunity are collapsed; missing, failed, and null grader results are excluded rather than scored as zero.

${observationRows}
ObservedTargetOpportunityValueEvidence
`; @@ -1314,21 +1315,11 @@ for (const bundle of bundleDefinitions) { const navigation = ``; const sections = []; for (const worker of bundle.workers) { - const artifact = valueTimelines.get(worker.id); - const assetName = `${worker.id}-timeline.svg`; - if (artifact) { - try { - await copyFile(artifact.svgPath, path.join(outputDirectory, "insights", "assets", assetName)); - } catch (error) { - if (error.code !== "ENOENT") throw error; - valueTimelines.delete(worker.id); - } - } - sections.push(valueReportContent(worker, valueTimelines.get(worker.id), assetName)); + sections.push(valueReportContent(worker, valueObservations.get(worker.id))); } await writeFile(path.join(outputDirectory, "insights", `${bundle.id}.html`), layout({ title: bundle.name, - description: `Worker operational-value measurements from the ${bundle.name} value functions.`, + description: `Worker operational-value observations from actual ${bundle.name} workflow runs.`, content: `${bundleTabs(bundle, "insights")}${sections.join("\n")}`, nested: true, navigation, @@ -1678,14 +1669,18 @@ footer a { min-height: 24px; display: inline-flex; align-items: center; } .value-score strong, .value-score span { display: block; } .value-score strong { font-size: 1.5rem; font-variant-numeric: tabular-nums; } .value-score span { color: var(--muted); font-size: .6875rem; } -.value-chart { height: 400px; padding: 12px 16px; overflow: hidden; border-bottom: 1px solid var(--border); background: var(--canvas-subtle); } -.value-chart img { width: 100%; height: 100%; display: block; object-fit: contain; object-position: center; } +.value-chart { min-height: 180px; display: grid; align-items: center; padding: 24px 16px; overflow: hidden; border-bottom: 1px solid var(--border); background: var(--canvas-subtle); } +.value-chart dl { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 1px; margin: 0; overflow: hidden; border: 1px solid var(--border); border-radius: 6px; background: var(--border); } +.value-chart dl > div { min-width: 0; padding: 18px; background: var(--canvas); } +.value-chart dt { color: var(--muted); font-size: .75rem; font-weight: 600; text-transform: uppercase; } +.value-chart dd { margin: 4px 0 0; overflow: hidden; font-size: 1.375rem; font-weight: 600; text-overflow: ellipsis; white-space: nowrap; } +.value-chart dd code { font-size: .875rem; } .value-details-disclosure > summary, .value-details-unavailable { min-height: 44px; display: flex; align-items: center; padding: 10px 16px; color: var(--fg); font-size: .75rem; font-weight: 600; } .value-details-disclosure > summary { cursor: pointer; } .value-details-disclosure > summary:hover { background: var(--canvas-subtle); } .value-details-disclosure[open] > summary { border-bottom: 1px solid var(--border); } .value-details-unavailable { color: var(--muted); } -.value-details { display: grid; grid-template-columns: minmax(260px, .75fr) minmax(0, 1.25fr); gap: 0; } +.value-details { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0; } .value-details > section { min-width: 0; padding: 16px; } .value-details > section + section { border-left: 1px solid var(--border); } .value-details h3 { margin: 0 0 4px; font-size: .875rem; } diff --git a/.github/skills/agentic-workflows/SKILL.md b/.github/skills/agentic-workflows/SKILL.md index 4109b7d..e9fb5f1 100644 --- a/.github/skills/agentic-workflows/SKILL.md +++ b/.github/skills/agentic-workflows/SKILL.md @@ -34,6 +34,7 @@ Load these files from `github/gh-aw` (they are not available locally). - `.github/aw/deployment-status.md` - `.github/aw/designer-mappings.md` - `.github/aw/designer.md` +- `.github/aw/drive-memory.md` - `.github/aw/enclaves.md` - `.github/aw/evals.md` - `.github/aw/experiments.md` @@ -103,5 +104,6 @@ After loading the matching workflow prompt or skill, follow it directly: - Choose workflow architecture and patterns: `.github/aw/patterns.md` - Optimize token usage and cost: `.github/aw/token-optimization.md` - Design long-running multi-agent research workflows: `.github/aw/multi-agent-research.md` +- Add skills or agent plugins requested by the user (`skills:` / `plugins:` frontmatter, never on-the-fly installs): `.github/aw/skills.md` When the task involves OTEL, OTLP, traces, observability backends, or telemetry-driven analysis, also read and follow `skills/otel-queries/SKILL.md` after loading the matching workflow prompt or skill. diff --git a/.github/skills/create-ops-bundle/SKILL.md b/.github/skills/create-ops-bundle/SKILL.md index c4c0fb7..eb30ddb 100644 --- a/.github/skills/create-ops-bundle/SKILL.md +++ b/.github/skills/create-ops-bundle/SKILL.md @@ -22,7 +22,7 @@ Turn an operational idea into a complete bundle of GitHub Agentic Workflows. A b 4. Ask only for decisions that cannot be inferred safely. If the strategy is broad, split it into workers by independently dispatchable responsibility, not by implementation step. 5. Create the orchestrator and every worker under `.github/workflows/` in the same change. 6. Compile and validate all new source workflows. Repair failures before finishing. -7. When an adopted worker already has a frozen ops-value function, preserve it under `.github/ops-values/`. Ops-value authoring remains a separate post-adoption maintenance task. +7. When an adopted worker already has an operational-value evaluator, preserve it under `.github/graders/` and keep its `graders.operational-value` registration. Evaluator design remains a separate post-adoption maintenance task. ## Bundle Contract @@ -78,12 +78,11 @@ Use a dedicated `target/` checkout when the worker must inspect a target reposit ### Worker Value -Measure operational value per worker because workers have independently dispatchable responsibilities and outcomes. The catalog keeps frozen ops-value functions locally but does not currently package them or the experimental authoring skill. +Measure operational value per worker because workers have independently dispatchable responsibilities and outcomes. gh-aw freezes each registered evaluator into the compiled workflow and publishes its observation with the workflow run artifacts. - Design from the worker's adoption-time intent and pre-adoption evidence. Never derive a measure from the orchestrator's dispatch activity or from post-adoption results. -- Keep the canonical function at `.github/ops-values/.sh`. -- Keep package manifests workflow-only while ops-value distribution is experimental. -- Treat function creation and report generation as post-adoption work; never create placeholder commits, evidence, scores, or reports while authoring an unadopted bundle. +- Keep the canonical evaluator at `.github/graders/-operational-value.sh` and register it under `graders.operational-value.run`. +- Treat evaluator creation as post-adoption work; never create placeholder commits, evidence, scores, or reports while authoring an unadopted bundle. - If the bundle is new in the current change, finish workflow validation and report the pending per-worker value follow-up explicitly. - A worker may be baseline-comparable, attainment-only, or not measurable. Preserve that independently determined classification rather than forcing every worker into the same bundle-level model. @@ -117,7 +116,7 @@ Before finishing: 6. Confirm the orchestrator has a `Completion` section that preserves the exact standard report contract from `shared/control.md`; bundle-specific reporting must be additive. 7. Confirm worker concurrency is keyed by `github.workflow` and `inputs.target_repo` with stale runs cancelled. 8. Check permissions, tools, network hosts, safe-output limits, credits, timeouts, and dispatch maximums against actual need. -9. Confirm every existing frozen ops-value function remains under `.github/ops-values/`, or explicitly identify each new worker whose value design is pending adoption. +9. Confirm every existing operational-value evaluator remains under `.github/graders/` and registered by its worker, or explicitly identify each new worker whose value design is pending adoption. 10. Run `gh aw compile ` for every new orchestrator and worker. Then run the repository's narrowest relevant tests or validation command if one exists. 11. Review the generated diff for accidental lockfile churn, secret exposure, unsafe live defaults, fabricated value evidence, and deviations from the nearest bundle that are not justified by the strategy. diff --git a/.github/skills/operational-value-designer/SKILL.md b/.github/skills/operational-value-designer/SKILL.md new file mode 100644 index 0000000..3bacb60 --- /dev/null +++ b/.github/skills/operational-value-designer/SKILL.md @@ -0,0 +1,99 @@ +--- +name: operational-value-designer +description: "Design and verify a deterministic operational-value grader for a GitHub Agentic Workflow. Use for per-run operational value, evidence attribution, maturation, baselines, and operational-value evaluators. Usage: /operational-value-designer OWNER/REPO WORKFLOW-NAME." +argument-hint: "OWNER/REPO WORKFLOW-NAME" +allowed-tools: bash jq gh +metadata: + version: "1.0.0" +--- + +# Operational Value Grader + +Design one deterministic `operational-value` grader that reports absolute operational attainment for each workflow run. + +Operational value is the degree to which the workflow's intended repository outcome is attained for the opportunity assigned to a run, demonstrated by accepted repository evidence under a frozen contract. It is not execution quality, output volume, safe-output creation, or an agent's assessment. + +## Output + +Create one executable evaluator at: + +```text +.github/graders/WORKFLOW-NAME-operational-value.sh +``` + +Configure the workflow: + +```yaml +graders: + operational-value: + run: .github/graders/WORKFLOW-NAME-operational-value.sh +``` + +The grader's primary operational value (`value`) is absolute attainment in `[0,1]`. A comparable frozen baseline may be reported separately as `baselineValue`; gh-aw derives `deltaFromBaseline`. Never define the primary operational value as a difference from baseline. + +## Design Procedure + +1. Validate `OWNER/REPO` and resolve `.github/workflows/WORKFLOW-NAME.md`. Do not infer inputs from the workspace or remotes. +2. Recover adoption-time intent from the workflow's first commit and first parent. Use only adoption-time workflow content and pre-adoption evidence to choose opportunities, accepted evidence, formulas, targets, or a baseline. +3. Define how every workflow run binds to one operational case: + - produce a stable `opportunityKey`; + - prevent overlapping ownership where possible; + - preserve repeated keys when duplicate runs target the same opportunity so downstream analysis can cluster or deduplicate them; + - treat reruns with the same GitHub run ID as the same subject. +4. Freeze accepted evidence, evidence repositories, matching rules, zero-versus-missing behavior, and `maturesAt` computation. + - Declare only the workflow permission scopes required to collect that evidence. The evaluator receives `GH_TOKEN` with the agent job's declared permissions; gh-aw does not add evidence permissions automatically. +5. Choose exactly one direct primary metric in `[0,1]`. Higher must always mean greater attainment. Keep trace graders and activity counts separate. +6. If comparable pre-adoption evidence exists, score it with the same metric and freeze it under `baseline`. Otherwise use `attainment-only` with a null baseline value. +7. Implement the evaluator interface below and run: + + ```bash + .github/skills/operational-value-designer/scripts/verify-operational-value-evaluator.sh .github/graders/WORKFLOW-NAME-operational-value.sh + gh aw compile .github/workflows/WORKFLOW-NAME.md + ``` +8. For a packaged workflow, install the package into a clean consumer and confirm the evaluator exists at the registered `.github/graders/` path before declaring the grader distributable. A source checkout compiling successfully does not prove package transport. + +## Evaluator Interface + +The evaluator uses Bash 3.2-compatible Bash plus `jq` and supports: + +- `--definition`: print the frozen schema-version 4 contract. +- `--metric`: read one evidence object on stdin and print a deterministic number in `[0,1]` or `null`. +- `--grade-run`: read a run request on stdin and print one operational-value observation. + +`--grade-run` receives a schema-version 1 request containing the complete `run` subject, `evidenceAt`, an optional replay `case`, the workflow `event`, and grader `config`. It returns `value`, `opportunityKey`, replayable `case`, `evidenceCutoff`, `maturesAt`, `provenance`, and optional diagnostics. + +The function must cap `evidenceCutoff` at the earlier of `evidenceAt` and `maturesAt`. A run is never intrinsically pending: the operational value is an as-of observation and may be recomputed until maturity. After maturity, the cap makes the result stable. + +## Regrade a Historical Run + +Recompute a run at an explicit evidence time with the same evaluator used by the original run: + +```bash +gh aw graders operational-value RUN-ID \ + --evidence-at 2026-08-30T12:00:00.000Z \ + --json +``` + +Add `--repo [HOST/]OWNER/REPO` to select the GitHub host for the current repository checkout. The command verifies the archived evaluator against both digest records and the evaluator at the recorded commit before executing it. + +## Definition Contract + +`--definition` must contain: + +- `schemaVersion: 4` and `grader: "operational-value"`; +- repository, workflow name, source path, and adoption commit/time; +- operational-value statement; +- evidence opportunity, assignment, accepted evidence, repositories, collection, maturation, zero rule, and missing rule; +- one primary metric with formula and validation examples; +- baseline mode, value, cutoff, and provenance. + +For `baseline-comparable`, baseline value must be in `[0,1]` and have immutable provenance. For `attainment-only`, baseline value and cutoff must be null. + +## Interpretation Rules + +- `value` answers "what operational value did this run attain for its assigned opportunity?" +- `deltaFromBaseline` answers "how far is this observation above or below the frozen pre-adoption reference?" +- Neither establishes that the workflow caused the outcome. +- Compare runs only under the same evaluator digest and evidence horizon. +- Identify a replayed observation by `(runId, evaluatorDigest, evidenceAt)`. +- Do not treat repeated observations of one run, duplicate opportunity keys, or overlapping state windows as independent samples. \ No newline at end of file diff --git a/.github/skills/operational-value-designer/scripts/operational-value-evaluator-path.sh b/.github/skills/operational-value-designer/scripts/operational-value-evaluator-path.sh new file mode 100755 index 0000000..b43f0ca --- /dev/null +++ b/.github/skills/operational-value-designer/scripts/operational-value-evaluator-path.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +[[ $# -eq 1 ]] || fail "usage: operational-value-evaluator-path.sh WORKFLOW-NAME" + +workflow_name=$1 +[[ $workflow_name =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] \ + || fail "workflow name must contain lowercase letters, numbers, and single hyphens" + +printf '.github/graders/%s-operational-value.sh\n' "$workflow_name" \ No newline at end of file diff --git a/.github/skills/operational-value-designer/scripts/verify-operational-value-evaluator.sh b/.github/skills/operational-value-designer/scripts/verify-operational-value-evaluator.sh new file mode 100755 index 0000000..f4dd9fc --- /dev/null +++ b/.github/skills/operational-value-designer/scripts/verify-operational-value-evaluator.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +[[ $# -eq 1 ]] || fail "usage: verify-operational-value-evaluator.sh " + +evaluator=$1 +[[ -f $evaluator ]] || fail "operational-value evaluator not found: $evaluator" +[[ -x $evaluator ]] || fail "operational-value evaluator is not executable: $evaluator" +command -v jq >/dev/null 2>&1 || fail "jq is required" +bash -n "$evaluator" + +definition=$("$evaluator" --definition) +printf '%s\n' "$definition" | jq -e ' + .schemaVersion == 4 + and .grader == "operational-value" + and (.repository | type == "string" and test("^[^/]+/[^/]+$")) + and (.workflowName | type == "string" and length > 0) + and (.sourcePath | type == "string" and startswith(".github/workflows/") and endswith(".md")) + and (.adoption.commit | type == "string" and test("^[0-9a-f]{40}$")) + and (.adoption.adoptedAt | type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T")) + and (.operationalValue | type == "string" and length > 0) + and (.evidence.opportunity | type == "string" and length > 0) + and (.evidence.assignment | type == "string" and length > 0) + and (.evidence.accepted | type == "string" and length > 0) + and (.evidence.repositories | type == "array" and length > 0) + and (all(.evidence.repositories[]; type == "string" and test("^[^/]+/[^/]+$"))) + and (.evidence.collection | type == "string" and length > 0) + and (.evidence.maturation | type == "string" and length > 0) + and (.evidence.zeroRule | type == "string" and length > 0) + and (.evidence.missingRule | type == "string" and length > 0) + and (.primaryMetric.id | type == "string" and length > 0) + and (.primaryMetric.formula | type == "string" and length > 0) + and (.primaryMetric.direction == "higher_is_better") + and (.validationExamples | has("targetAttained") and has("targetMissed") and has("missing") and has("malformed")) + and (.baseline.mode == "baseline-comparable" or .baseline.mode == "attainment-only") + and (if .baseline.mode == "baseline-comparable" then + (.baseline.value | type == "number" and . >= 0 and . <= 1) + and (.baseline.evidenceCutoff | type == "string" and length > 0) + and (.baseline.provenance | type == "array" and length > 0) + else + .baseline.value == null and .baseline.evidenceCutoff == null + end) +' >/dev/null || fail "operational-value evaluator definition is invalid" + +for example_name in targetAttained targetMissed missing malformed; do + evidence=$(printf '%s\n' "$definition" | jq -c --arg name "$example_name" '.validationExamples[$name]') + result=$(printf '%s\n' "$evidence" | "$evaluator" --metric) + printf '%s\n' "$result" | jq -e '. == null or (type == "number" and . >= 0 and . <= 1)' >/dev/null \ + || fail "--metric returned an invalid score for $example_name" + case $example_name in + targetAttained) target_attained=$result ;; + targetMissed) target_missed=$result ;; + missing|malformed) [[ $result == null ]] || fail "--metric must return null for $example_name" ;; + esac +done + +jq -en --argjson attained "$target_attained" --argjson missed "$target_missed" \ + '$attained != null and $missed != null and $attained > $missed' >/dev/null \ + || fail "targetAttained must score higher than targetMissed" + +repository=$(printf '%s\n' "$definition" | jq -r .repository) +workflow_name=$(printf '%s\n' "$definition" | jq -r .workflowName) +adoption_commit=$(printf '%s\n' "$definition" | jq -r .adoption.commit) +created_at=$(printf '%s\n' "$definition" | jq -r .adoption.adoptedAt) +evidence_at=2099-01-01T00:00:00Z +request=$(jq -cn --arg repository "$repository" --arg workflow "$workflow_name" \ + --arg sha "$adoption_commit" --arg createdAt "$created_at" --arg evidenceAt "$evidence_at" ' + {schemaVersion:1,run:{id:"1",attempt:1,repository:$repository,workflow:$workflow, + ref:"refs/heads/main",sha:$sha,eventName:"workflow_dispatch",createdAt:$createdAt}, + evidenceAt:$evidenceAt,case:null,event:{},config:{verification:true}}') +grade_run=$(printf '%s\n' "$request" | "$evaluator" --grade-run) +printf '%s\n' "$grade_run" | jq -e --arg evidenceAt "$evidence_at" ' + def timestamp: type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{3})?Z$"); + def epoch: sub("\\.[0-9]{3}Z$"; "Z") | fromdateiso8601; + type == "object" + and (.value == null or (.value | type == "number" and isfinite and . >= 0 and . <= 1)) + and (.opportunityKey | type == "string" and length > 0) + and (.case | type == "object") + and (.evidenceCutoff | timestamp) and (.maturesAt | timestamp) + and ((.evidenceCutoff | epoch) <= ($evidenceAt | epoch)) + and ((.evidenceCutoff | epoch) <= (.maturesAt | epoch)) + and (.provenance | type == "array") + and (if .value == null then true else (.provenance | length > 0) end) + and (all(.provenance[]; type == "object" + and (.repository | type == "string" and length > 0) + and (.kind | type == "string" and length > 0) + and (.ref | type == "string" and length > 0))) + and ((has("diagnostics") | not) or (.diagnostics | type == "object")) + and ((has("message") | not) or (.message | type == "string")) +' >/dev/null || fail "--grade-run returned an invalid operational-value observation" + +printf 'verified %s\n' "$evaluator" \ No newline at end of file diff --git a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-definitions.md b/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-definitions.md deleted file mode 100644 index eb54363..0000000 --- a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-definitions.md +++ /dev/null @@ -1,46 +0,0 @@ -# What Dependabot / Release Train Updater measures - -This page explains the chart in plain language. It defines what was measured; it does not decide whether the workflow caused the observed changes. - -![Dependabot / Release Train Updater outcome measures after adoption](dependabot-release-train-updater-timeline.svg) - -## How to read the chart - -- Observations begin at workflow adoption on `2026-08-18`. -- No comparable pre-adoption evidence is available, so the chart shows attainment rather than improvement. -- Each dot is one immutable observation. Missing evidence is omitted, never treated as zero. -- Workflow runs show execution activity only. They do not prove repository value. - -## What was measured - -### Validated dependency resolution share - -- **What it tells you:** Validated dependency resolution share. This is a `primary` measure. -- **Normalized scoring formula:** `validatedResolutions / eligibleOpportunities` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -### Security-opportunity resolution share - -- **What it tells you:** Security-opportunity resolution share. This is a `diagnostic` measure. -- **Normalized scoring formula:** `securityValidatedResolutions / securityEligibleOpportunities` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -## Evidence rules - -- **Repository:** `githubnext/central-agentic-ops` -- **Evidence population:** Matured dependency pull requests or security-alert opportunities in repositories named by immutable central workflow run display titles. -- **Collection:** Batch central Actions runs over the complete requested span, fetch each target pull-request population once for that span, classify locally, and fetch merge-commit check evidence only for eligible merged dependency candidates. -- **Observation window:** 30 days, sampled every 30 days -- **Maturation delay:** 14 days -- **Filters:** `Dispatch targets are owner/repository names parsed from Dependabot / Release Train Updater Actions run display titles.`; `Opportunities are pull requests created in the window, matured for 14 days by observedAt, and classified from title, author, labels, and changed files.`; `Dependency opportunities are Dependabot-authored pull requests or pull requests changing dependency manifests or lockfiles.`; `Security opportunities have security labels, security title indicators, or Dependabot security indicators.`; `Validated resolutions are merged by windowEnd, change dependency manifests or lockfiles, and have successful evidence for every configured required check at the merge commit.`; `Unavailable required-check configuration or merge-commit check evidence cannot establish a validated resolution.` - -The frozen definitions and formulas are applied to every post-adoption observation. The structured evidence, exact snapshots, provenance, and normalized scores are in [dependabot-release-train-updater-timeline.json](dependabot-release-train-updater-timeline.json). - -## Important limitation - -This report can show whether the intended outcome is attained after adoption. It cannot estimate change from pre-adoption conditions or attribute attainment to the workflow. - -Value-function SHA-256: `3ae460a857f942ecda114ccaa697da705096c235d8dc43c30fe5b0f218ff6519` - diff --git a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-evidence-archive.json b/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-evidence-archive.json deleted file mode 100644 index 8da8497..0000000 --- a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-evidence-archive.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "schemaVersion": 1, - "repository": "githubnext/central-agentic-ops", - "workflowSlug": "dependabot-release-train-updater", - "functions": { - "3ae460a857f942ecda114ccaa697da705096c235d8dc43c30fe5b0f218ff6519": { - "valueFunctionSha256": "3ae460a857f942ecda114ccaa697da705096c235d8dc43c30fe5b0f218ff6519", - "snapshots": [] - } - } -} diff --git a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.json b/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.json deleted file mode 100644 index bef6817..0000000 --- a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.json +++ /dev/null @@ -1,284 +0,0 @@ -{ - "repository": "githubnext/central-agentic-ops", - "workflowName": "Dependabot / Release Train Updater", - "adoptionAt": "2026-08-18T17:54:55Z", - "evaluationMode": "attainment-only", - "window": { - "startAt": "2026-08-18T17:54:55Z", - "endAt": "2026-08-20T18:33:18Z" - }, - "evidenceCoverage": { - "completeFromAdoption": true, - "hasPreAdoptionBaseline": false, - "baselineDescription": "No comparable pre-adoption baseline is available; observations begin at adoption.", - "reason": "Comparable historical outcome evidence cannot be reconstructed." - }, - "runs": [], - "caveat": "The evidence measures post-adoption attainment only; without a comparable pre-adoption baseline it cannot establish improvement or causation.", - "schemaVersion": 2, - "definitionSchemaVersion": 3, - "generatedAt": "2026-08-20T18:33:20Z", - "workflowSlug": "dependabot-release-train-updater", - "sourcePath": ".github/workflows/dependabot-release-train-updater.md", - "valueFunction": { - "path": ".github/ops-values/dependabot-release-train-updater.sh", - "sha256": "3ae460a857f942ecda114ccaa697da705096c235d8dc43c30fe5b0f218ff6519", - "definition": { - "schemaVersion": 3, - "slug": "dependabot-release-train-updater", - "sourcePath": ".github/workflows/dependabot-release-train-updater.md", - "repository": "githubnext/central-agentic-ops", - "workflowName": "Dependabot / Release Train Updater", - "adoption": { - "commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", - "adoptedAt": "2026-08-18T17:54:55Z", - "baselineCommit": "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - "baselineAt": "2026-08-18T12:20:21Z" - }, - "evaluation": { - "mode": "attainment-only" - }, - "evidence": { - "key": "dependabot-release-train-validated-resolutions-v1", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Matured dependency pull requests or security-alert opportunities in repositories named by immutable central workflow run display titles.", - "filters": [ - "Dispatch targets are owner/repository names parsed from Dependabot / Release Train Updater Actions run display titles.", - "Opportunities are pull requests created in the window, matured for 14 days by observedAt, and classified from title, author, labels, and changed files.", - "Dependency opportunities are Dependabot-authored pull requests or pull requests changing dependency manifests or lockfiles.", - "Security opportunities have security labels, security title indicators, or Dependabot security indicators.", - "Validated resolutions are merged by windowEnd, change dependency manifests or lockfiles, and have successful evidence for every configured required check at the merge commit.", - "Unavailable required-check configuration or merge-commit check evidence cannot establish a validated resolution." - ], - "collection": "Batch central Actions runs over the complete requested span, fetch each target pull-request population once for that span, classify locally, and fetch merge-commit check evidence only for eligible merged dependency candidates.", - "window": { - "durationDays": 30, - "cadenceDays": 30, - "maturationDays": 14 - } - }, - "model": { - "architecture": "Deterministic opportunity-normalized attainment shares", - "recommendation": "Use the validated dependency resolution share as primary and the security-opportunity resolution share as a diagnostic; report missing when no eligible opportunity evidence exists.", - "presentation": { - "label": "Validated dependency resolution attainment", - "betterLabel": "Higher validated resolution share" - } - }, - "summary": { - "nativeLabel": "Validated dependency resolution share" - }, - "metrics": [ - { - "id": "validated-resolution-share", - "name": "Validated dependency resolution share", - "role": "primary", - "formula": "validatedResolutions / eligibleOpportunities", - "direction": "increase", - "presentation": { - "name": "Validated dependency resolution share", - "legendLabel": "Validated resolution", - "transform": "identity" - } - }, - { - "id": "security-resolution-share", - "name": "Security-opportunity resolution share", - "role": "diagnostic", - "formula": "securityValidatedResolutions / securityEligibleOpportunities", - "direction": "increase", - "presentation": { - "name": "Security-opportunity resolution share", - "legendLabel": "Security resolution", - "transform": "identity" - } - } - ], - "validationExamples": { - "targetAttained": { - "status": "observed", - "eligibleOpportunities": 4, - "validatedResolutions": 4, - "securityEligibleOpportunities": 2, - "securityValidatedResolutions": 2 - }, - "targetMissed": { - "status": "observed", - "eligibleOpportunities": 4, - "validatedResolutions": 0, - "securityEligibleOpportunities": 2, - "securityValidatedResolutions": 0 - }, - "missing": { - "status": "missing", - "eligibleOpportunities": null, - "validatedResolutions": null, - "securityEligibleOpportunities": null, - "securityValidatedResolutions": null - }, - "malformed": { - "status": "observed", - "eligibleOpportunities": 1, - "validatedResolutions": 2, - "securityEligibleOpportunities": "unknown", - "securityValidatedResolutions": 0 - } - } - } - }, - "metricReview": { - "architecture": "Deterministic opportunity-normalized attainment shares", - "recommendation": "Use the validated dependency resolution share as primary and the security-opportunity resolution share as a diagnostic; report missing when no eligible opportunity evidence exists.", - "presentation": { - "label": "Validated dependency resolution attainment", - "betterLabel": "Higher validated resolution share" - }, - "metrics": [ - { - "id": "validated-resolution-share", - "name": "Validated dependency resolution share", - "role": "primary", - "formula": "validatedResolutions / eligibleOpportunities", - "direction": "increase", - "presentation": { - "name": "Validated dependency resolution share", - "legendLabel": "Validated resolution", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - }, - { - "id": "security-resolution-share", - "name": "Security-opportunity resolution share", - "role": "diagnostic", - "formula": "securityValidatedResolutions / securityEligibleOpportunities", - "direction": "increase", - "presentation": { - "name": "Security-opportunity resolution share", - "legendLabel": "Security resolution", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - } - ] - }, - "summary": { - "nativeLabel": "Validated dependency resolution share" - }, - "snapshots": [ - { - "observedAt": "2026-08-18T17:54:55Z", - "window": { - "startAt": "2026-07-05T17:54:55Z", - "endAt": "2026-08-04T17:54:55Z" - }, - "evidence": { - "key": "dependabot-release-train-validated-resolutions-v1", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Matured dependency pull requests or security-alert opportunities in dispatched targets", - "filters": [ - "created in window", - "matured 14 days", - "Dependabot author or dependency file change", - "validated merge with dependency file changes and successful configured required checks" - ], - "collection": "Batched central Actions run discovery and one pull-request corpus per dispatched target over the complete requested span", - "window": { - "durationDays": 30, - "cadenceDays": 30, - "maturationDays": 14, - "windowStart": "2026-07-05T17:54:55Z", - "windowEnd": "2026-08-04T17:54:55Z", - "observedAt": "2026-08-18T17:54:55Z" - }, - "status": "missing", - "eligibleOpportunities": null, - "validatedResolutions": null, - "securityEligibleOpportunities": null, - "securityValidatedResolutions": null, - "targets": [], - "opportunities": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-run", - "ref": "run:30544560426" - }, - { - "repository": "githubnext/central-agentic-ops", - "kind": "frozen-contract", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0" - } - ], - "metrics": { - "validated-resolution-share": null, - "security-resolution-share": null - } - }, - { - "observedAt": "2026-08-20T18:33:18Z", - "window": { - "startAt": "2026-07-07T18:33:18Z", - "endAt": "2026-08-06T18:33:18Z" - }, - "evidence": { - "key": "dependabot-release-train-validated-resolutions-v1", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Matured dependency pull requests or security-alert opportunities in dispatched targets", - "filters": [ - "created in window", - "matured 14 days", - "Dependabot author or dependency file change", - "validated merge with dependency file changes and successful configured required checks" - ], - "collection": "Batched central Actions run discovery and one pull-request corpus per dispatched target over the complete requested span", - "window": { - "durationDays": 30, - "cadenceDays": 30, - "maturationDays": 14, - "windowStart": "2026-07-07T18:33:18Z", - "windowEnd": "2026-08-06T18:33:18Z", - "observedAt": "2026-08-20T18:33:18Z" - }, - "status": "missing", - "eligibleOpportunities": null, - "validatedResolutions": null, - "securityEligibleOpportunities": null, - "securityValidatedResolutions": null, - "targets": [], - "opportunities": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-run", - "ref": "run:30544560426" - }, - { - "repository": "githubnext/central-agentic-ops", - "kind": "frozen-contract", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0" - } - ], - "metrics": { - "validated-resolution-share": null, - "security-resolution-share": null - } - } - ] -} diff --git a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.svg b/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.svg deleted file mode 100644 index 2bd0699..0000000 --- a/.github/value/dependabot-release-train-updater/dependabot-release-train-updater-timeline.svg +++ /dev/null @@ -1,25 +0,0 @@ - -Dependabot / Release Train Updater workflow attainment timeline -Goal-oriented repository outcome metrics after workflow adoption, with workflow run conclusions over time. - - -Dependabot / Release Train Updater attainment over time -Post-adoption attainment -1 -0.75 -0.5 -0.25 -0 -Aug 18 -Aug 20 -Goal measure - - - -Validated resolution -Security resolution -Workflow adopted -Workflow runs - -SuccessFailureOther - diff --git a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-definitions.md b/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-definitions.md deleted file mode 100644 index 15aca8f..0000000 --- a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-definitions.md +++ /dev/null @@ -1,53 +0,0 @@ -# What Optimization / AI Credit Auditor measures - -This page explains the chart in plain language. It defines what was measured; it does not decide whether the workflow caused the observed changes. - -![Optimization / AI Credit Auditor outcome measures after adoption](optimization-ai-credit-auditor-timeline.svg) - -## How to read the chart - -- Observations begin at workflow adoption on `2026-08-18`. -- No comparable pre-adoption evidence is available, so the chart shows attainment rather than improvement. -- Each dot is one immutable observation. Missing evidence is omitted, never treated as zero. -- Workflow runs show execution activity only. They do not prove repository value. - -## What was measured - -### Accurate audit-day share - -- **What it tells you:** Accurate audit-day share. This is a `primary` measure. -- **Normalized scoring formula:** `matched eligible target-days / eligible target-days with both retained completed-run logs and a readable durable snapshot` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -### Completed-run coverage - -- **What it tells you:** Completed-run coverage. This is a `diagnostic` measure. -- **Normalized scoring formula:** `sum min(snapshot overall.total_runs, retained completed runs) / sum retained completed runs across paired eligible target-days` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -### Durable-history coverage - -- **What it tells you:** Durable-history coverage. This is a `diagnostic` measure. -- **Normalized scoring formula:** `eligible target-days with a readable durable snapshot / eligible target-days established from retained completed-run logs` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -## Evidence rules - -- **Repository:** `githubnext/central-agentic-ops` -- **Evidence population:** A target repository and UTC day containing at least one completed agentic workflow run, among targets immutably named by dispatched Optimization / AI Credit Auditor runs. -- **Collection:** Batch central Actions discovery over the full request, fetch gh aw logs once per discovered target for the total date span, and read daily snapshots from immutable central repo-memory commits at or before observedAt. -- **Observation window:** 1 days, sampled every 1 days -- **Maturation delay:** 1 days -- **Filters:** `Discover targets only from central Actions run display_title values matching Token audit · owner/repo · mode.`; `Include only retained target runs with status completed and created_at within the target-day window.`; `Empty completed-run windows are ineligible.`; `A day is accurate only when the durable daily snapshot reproduces overall and per-workflow completed-run aggregates.`; `Absent or inaccessible retained logs and snapshots remain explicitly missing; numeric zero is accepted only from retained evidence.` - -The frozen definitions and formulas are applied to every post-adoption observation. The structured evidence, exact snapshots, provenance, and normalized scores are in [optimization-ai-credit-auditor-timeline.json](optimization-ai-credit-auditor-timeline.json). - -## Important limitation - -This report can show whether the intended outcome is attained after adoption. It cannot estimate change from pre-adoption conditions or attribute attainment to the workflow. - -Value-function SHA-256: `87e7dcc0ed2e8d64e93a5023f40f3476f3677314b1711d140113a488d930a5b1` - diff --git a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-evidence-archive.json b/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-evidence-archive.json deleted file mode 100644 index 7b11ff5..0000000 --- a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-evidence-archive.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "schemaVersion": 1, - "repository": "githubnext/central-agentic-ops", - "workflowSlug": "optimization-ai-credit-auditor", - "functions": { - "87e7dcc0ed2e8d64e93a5023f40f3476f3677314b1711d140113a488d930a5b1": { - "valueFunctionSha256": "87e7dcc0ed2e8d64e93a5023f40f3476f3677314b1711d140113a488d930a5b1", - "snapshots": [] - } - } -} diff --git a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.json b/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.json deleted file mode 100644 index d433a18..0000000 --- a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.json +++ /dev/null @@ -1,382 +0,0 @@ -{ - "repository": "githubnext/central-agentic-ops", - "workflowName": "Optimization / AI Credit Auditor", - "adoptionAt": "2026-08-18T17:54:55Z", - "evaluationMode": "attainment-only", - "window": { - "startAt": "2026-08-18T17:54:55Z", - "endAt": "2026-08-20T18:33:18Z" - }, - "evidenceCoverage": { - "completeFromAdoption": true, - "hasPreAdoptionBaseline": false, - "baselineDescription": "No comparable pre-adoption baseline is available; observations begin at adoption.", - "reason": "Comparable historical outcome evidence cannot be reconstructed." - }, - "runs": [], - "caveat": "The evidence measures post-adoption attainment only; without a comparable pre-adoption baseline it cannot establish improvement or causation.", - "schemaVersion": 2, - "definitionSchemaVersion": 3, - "generatedAt": "2026-08-20T18:33:22Z", - "workflowSlug": "optimization-ai-credit-auditor", - "sourcePath": ".github/workflows/optimization-ai-credit-auditor.md", - "valueFunction": { - "path": ".github/ops-values/optimization-ai-credit-auditor.sh", - "sha256": "87e7dcc0ed2e8d64e93a5023f40f3476f3677314b1711d140113a488d930a5b1", - "definition": { - "schemaVersion": 3, - "slug": "optimization-ai-credit-auditor", - "sourcePath": ".github/workflows/optimization-ai-credit-auditor.md", - "repository": "githubnext/central-agentic-ops", - "workflowName": "Optimization / AI Credit Auditor", - "adoption": { - "commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", - "adoptedAt": "2026-08-18T17:54:55Z", - "baselineCommit": "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - "baselineAt": "2026-08-18T12:20:21Z" - }, - "evaluation": { - "mode": "attainment-only" - }, - "evidence": { - "key": "target-day-audit-aggregate-reproduction", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "A target repository and UTC day containing at least one completed agentic workflow run, among targets immutably named by dispatched Optimization / AI Credit Auditor runs.", - "filters": [ - "Discover targets only from central Actions run display_title values matching Token audit · owner/repo · mode.", - "Include only retained target runs with status completed and created_at within the target-day window.", - "Empty completed-run windows are ineligible.", - "A day is accurate only when the durable daily snapshot reproduces overall and per-workflow completed-run aggregates.", - "Absent or inaccessible retained logs and snapshots remain explicitly missing; numeric zero is accepted only from retained evidence." - ], - "collection": "Batch central Actions discovery over the full request, fetch gh aw logs once per discovered target for the total date span, and read daily snapshots from immutable central repo-memory commits at or before observedAt.", - "window": { - "durationDays": 1, - "cadenceDays": 1, - "maturationDays": 1 - } - }, - "model": { - "architecture": "Deterministic exact aggregate reproduction over paired retained run logs and durable daily snapshots", - "recommendation": "Use the accurate audit-day share as primary; report completed-run and durable-history coverage separately so missing evidence is never averaged into accuracy.", - "presentation": { - "label": "Accurate audit days", - "betterLabel": "Higher is better" - } - }, - "summary": { - "nativeLabel": "Accurate audit-day share" - }, - "metrics": [ - { - "id": "accurate-audit-day-share", - "name": "Accurate audit-day share", - "role": "primary", - "formula": "matched eligible target-days / eligible target-days with both retained completed-run logs and a readable durable snapshot", - "direction": "increase", - "presentation": { - "name": "Accurate audit-day share", - "legendLabel": "Accurate days", - "transform": "identity" - } - }, - { - "id": "completed-run-coverage", - "name": "Completed-run coverage", - "role": "diagnostic", - "formula": "sum min(snapshot overall.total_runs, retained completed runs) / sum retained completed runs across paired eligible target-days", - "direction": "increase", - "presentation": { - "name": "Completed-run coverage", - "legendLabel": "Run coverage", - "transform": "identity" - } - }, - { - "id": "durable-history-coverage", - "name": "Durable-history coverage", - "role": "diagnostic", - "formula": "eligible target-days with a readable durable snapshot / eligible target-days established from retained completed-run logs", - "direction": "increase", - "presentation": { - "name": "Durable-history coverage", - "legendLabel": "History coverage", - "transform": "identity" - } - } - ], - "validationExamples": { - "targetAttained": { - "status": "complete", - "days": [ - { - "eligible": true, - "logsStatus": "available", - "snapshotStatus": "available", - "comparison": "matched", - "completedRuns": 4, - "snapshotRuns": 4 - } - ] - }, - "targetMissed": { - "status": "complete", - "days": [ - { - "eligible": true, - "logsStatus": "available", - "snapshotStatus": "available", - "comparison": "mismatched", - "completedRuns": 4, - "snapshotRuns": 2 - }, - { - "eligible": true, - "logsStatus": "available", - "snapshotStatus": "missing", - "comparison": "missing", - "completedRuns": 3, - "snapshotRuns": null - } - ] - }, - "missing": { - "status": "missing", - "days": [] - }, - "malformed": { - "status": "complete", - "days": [ - { - "eligible": "yes" - } - ] - } - } - } - }, - "metricReview": { - "architecture": "Deterministic exact aggregate reproduction over paired retained run logs and durable daily snapshots", - "recommendation": "Use the accurate audit-day share as primary; report completed-run and durable-history coverage separately so missing evidence is never averaged into accuracy.", - "presentation": { - "label": "Accurate audit days", - "betterLabel": "Higher is better" - }, - "metrics": [ - { - "id": "accurate-audit-day-share", - "name": "Accurate audit-day share", - "role": "primary", - "formula": "matched eligible target-days / eligible target-days with both retained completed-run logs and a readable durable snapshot", - "direction": "increase", - "presentation": { - "name": "Accurate audit-day share", - "legendLabel": "Accurate days", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - }, - { - "id": "completed-run-coverage", - "name": "Completed-run coverage", - "role": "diagnostic", - "formula": "sum min(snapshot overall.total_runs, retained completed runs) / sum retained completed runs across paired eligible target-days", - "direction": "increase", - "presentation": { - "name": "Completed-run coverage", - "legendLabel": "Run coverage", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - }, - { - "id": "durable-history-coverage", - "name": "Durable-history coverage", - "role": "diagnostic", - "formula": "eligible target-days with a readable durable snapshot / eligible target-days established from retained completed-run logs", - "direction": "increase", - "presentation": { - "name": "Durable-history coverage", - "legendLabel": "History coverage", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - } - ] - }, - "summary": { - "nativeLabel": "Accurate audit-day share" - }, - "snapshots": [ - { - "observedAt": "2026-08-18T17:54:55Z", - "window": { - "startAt": "2026-08-16T17:54:55Z", - "endAt": "2026-08-17T17:54:55Z" - }, - "evidence": { - "key": "target-day-audit-aggregate-reproduction", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - "filters": [ - "completed target runs", - "exact durable aggregate reproduction", - "empty run windows ineligible" - ], - "collection": "Batched central Actions discovery, one gh aw logs fetch per target, immutable repo-memory snapshot lookup", - "window": { - "durationDays": 1, - "cadenceDays": 1, - "maturationDays": 1 - }, - "status": "complete", - "days": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-workflow-query", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0:.github/workflows/optimization-ai-credit-auditor.lock.yml" - } - ], - "metrics": { - "accurate-audit-day-share": null, - "completed-run-coverage": null, - "durable-history-coverage": null - } - }, - { - "observedAt": "2026-08-19T17:54:55Z", - "window": { - "startAt": "2026-08-17T17:54:55Z", - "endAt": "2026-08-18T17:54:55Z" - }, - "evidence": { - "key": "target-day-audit-aggregate-reproduction", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - "filters": [ - "completed target runs", - "exact durable aggregate reproduction", - "empty run windows ineligible" - ], - "collection": "Batched central Actions discovery, one gh aw logs fetch per target, immutable repo-memory snapshot lookup", - "window": { - "durationDays": 1, - "cadenceDays": 1, - "maturationDays": 1 - }, - "status": "complete", - "days": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-workflow-query", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0:.github/workflows/optimization-ai-credit-auditor.lock.yml" - } - ], - "metrics": { - "accurate-audit-day-share": null, - "completed-run-coverage": null, - "durable-history-coverage": null - } - }, - { - "observedAt": "2026-08-20T17:54:55Z", - "window": { - "startAt": "2026-08-18T17:54:55Z", - "endAt": "2026-08-19T17:54:55Z" - }, - "evidence": { - "key": "target-day-audit-aggregate-reproduction", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - "filters": [ - "completed target runs", - "exact durable aggregate reproduction", - "empty run windows ineligible" - ], - "collection": "Batched central Actions discovery, one gh aw logs fetch per target, immutable repo-memory snapshot lookup", - "window": { - "durationDays": 1, - "cadenceDays": 1, - "maturationDays": 1 - }, - "status": "complete", - "days": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-workflow-query", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0:.github/workflows/optimization-ai-credit-auditor.lock.yml" - } - ], - "metrics": { - "accurate-audit-day-share": null, - "completed-run-coverage": null, - "durable-history-coverage": null - } - }, - { - "observedAt": "2026-08-20T18:33:18Z", - "window": { - "startAt": "2026-08-18T18:33:18Z", - "endAt": "2026-08-19T18:33:18Z" - }, - "evidence": { - "key": "target-day-audit-aggregate-reproduction", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Target-days containing completed agentic workflow runs for immutably dispatched auditor targets", - "filters": [ - "completed target runs", - "exact durable aggregate reproduction", - "empty run windows ineligible" - ], - "collection": "Batched central Actions discovery, one gh aw logs fetch per target, immutable repo-memory snapshot lookup", - "window": { - "durationDays": 1, - "cadenceDays": 1, - "maturationDays": 1 - }, - "status": "complete", - "days": [] - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "actions-workflow-query", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0:.github/workflows/optimization-ai-credit-auditor.lock.yml" - } - ], - "metrics": { - "accurate-audit-day-share": null, - "completed-run-coverage": null, - "durable-history-coverage": null - } - } - ] -} diff --git a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.svg b/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.svg deleted file mode 100644 index 94dc726..0000000 --- a/.github/value/optimization-ai-credit-auditor/optimization-ai-credit-auditor-timeline.svg +++ /dev/null @@ -1,29 +0,0 @@ - -Optimization / AI Credit Auditor workflow attainment timeline -Goal-oriented repository outcome metrics after workflow adoption, with workflow run conclusions over time. - - -Optimization / AI Credit Auditor attainment over time -Post-adoption attainment -1 -0.75 -0.5 -0.25 -0 -Aug 18 -Aug 19 -Aug 20 -Aug 20 -Goal measure - - - - -Accurate days -Run coverage -History coverage -Workflow adopted -Workflow runs - -SuccessFailureOther - diff --git a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-definitions.md b/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-definitions.md deleted file mode 100644 index d1d423b..0000000 --- a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-definitions.md +++ /dev/null @@ -1,53 +0,0 @@ -# What Optimization / AI Credit Optimizer measures - -This page explains the chart in plain language. It defines what was measured; it does not decide whether the workflow caused the observed changes. - -![Optimization / AI Credit Optimizer outcome measures after adoption](optimization-ai-credit-optimizer-timeline.svg) - -## How to read the chart - -- Observations begin at workflow adoption on `2026-08-18`. -- No comparable pre-adoption evidence is available, so the chart shows attainment rather than improvement. -- Each dot is one immutable observation. Missing evidence is omitted, never treated as zero. -- Workflow runs show execution activity only. They do not prove repository value. - -## What was measured - -### Efficient and reliable - -- **What it tells you:** Efficient-and-reliable opportunity share. This is a `primary` measure. -- **Normalized scoring formula:** `efficientReliableOpportunities / comparableOpportunities` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -### Lower median AIC - -- **What it tells you:** Lower-AIC opportunity share. This is a `diagnostic` measure. -- **Normalized scoring formula:** `lowerAicOpportunities / comparableOpportunities` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -### Reliability preserved - -- **What it tells you:** Reliability-preserved opportunity share. This is a `diagnostic` measure. -- **Normalized scoring formula:** `reliabilityPreservedOpportunities / comparableOpportunities` -- **Goal:** Higher values are better. -- **Chart display:** The chart shows the normalized score directly. - -## Evidence rules - -- **Repository:** `githubnext/central-agentic-ops` -- **Evidence population:** A dispatched repository target in a seven-day window for which the completed first-half runs identify a highest-total-AIC workflow and that same workflow has completed and successful-run evidence in both halves. -- **Collection:** Batch central Actions runs for all windows, deduplicate dispatched targets, invoke gh aw logs once per target for the total requested span, and derive each midpoint comparison locally from immutable target run IDs. -- **Observation window:** 7 days, sampled every 7 days -- **Maturation delay:** 7 days -- **Filters:** `Targets are parsed from immutable display_title values of completed Optimization / AI Credit Optimizer Actions runs in the central repository.`; `The target workflow is the workflow with highest total AIC among completed first-half runs, with workflow name ascending as the stable tie-break.`; `The selected workflow must have at least one completed run and at least one successful run in each half.`; `AIC medians use successful runs only; failure rates use all completed runs and classify every conclusion other than success as non-successful.`; `Second-half median AIC must be strictly lower and second-half failure rate must be no greater than first-half failure rate.` - -The frozen definitions and formulas are applied to every post-adoption observation. The structured evidence, exact snapshots, provenance, and normalized scores are in [optimization-ai-credit-optimizer-timeline.json](optimization-ai-credit-optimizer-timeline.json). - -## Important limitation - -This report can show whether the intended outcome is attained after adoption. It cannot estimate change from pre-adoption conditions or attribute attainment to the workflow. - -Value-function SHA-256: `31fd953cba856c7a825c5cfec3bdac30b9c6d16b4fc3d1dd42abd7cb52fc949d` - diff --git a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-evidence-archive.json b/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-evidence-archive.json deleted file mode 100644 index 93c2360..0000000 --- a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-evidence-archive.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "schemaVersion": 1, - "repository": "githubnext/central-agentic-ops", - "workflowSlug": "optimization-ai-credit-optimizer", - "functions": { - "31fd953cba856c7a825c5cfec3bdac30b9c6d16b4fc3d1dd42abd7cb52fc949d": { - "valueFunctionSha256": "31fd953cba856c7a825c5cfec3bdac30b9c6d16b4fc3d1dd42abd7cb52fc949d", - "snapshots": [] - } - } -} diff --git a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.json b/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.json deleted file mode 100644 index a69507a..0000000 --- a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.json +++ /dev/null @@ -1,301 +0,0 @@ -{ - "repository": "githubnext/central-agentic-ops", - "workflowName": "Optimization / AI Credit Optimizer", - "adoptionAt": "2026-08-18T17:54:55Z", - "evaluationMode": "attainment-only", - "window": { - "startAt": "2026-08-18T17:54:55Z", - "endAt": "2026-08-20T18:33:18Z" - }, - "evidenceCoverage": { - "completeFromAdoption": true, - "hasPreAdoptionBaseline": false, - "baselineDescription": "No comparable pre-adoption baseline is available; observations begin at adoption.", - "reason": "Comparable historical outcome evidence cannot be reconstructed." - }, - "runs": [], - "caveat": "The evidence measures post-adoption attainment only; without a comparable pre-adoption baseline it cannot establish improvement or causation.", - "schemaVersion": 2, - "definitionSchemaVersion": 3, - "generatedAt": "2026-08-20T18:33:25Z", - "workflowSlug": "optimization-ai-credit-optimizer", - "sourcePath": ".github/workflows/optimization-ai-credit-optimizer.md", - "valueFunction": { - "path": ".github/ops-values/optimization-ai-credit-optimizer.sh", - "sha256": "31fd953cba856c7a825c5cfec3bdac30b9c6d16b4fc3d1dd42abd7cb52fc949d", - "definition": { - "schemaVersion": 3, - "slug": "optimization-ai-credit-optimizer", - "sourcePath": ".github/workflows/optimization-ai-credit-optimizer.md", - "repository": "githubnext/central-agentic-ops", - "workflowName": "Optimization / AI Credit Optimizer", - "adoption": { - "commit": "35c7c3cbd319632f85784cce196e57c0f61db9a0", - "adoptedAt": "2026-08-18T17:54:55Z", - "baselineCommit": "ed9921bfd3aa8f95f9cc8dd30f87d0dbca97a42b", - "baselineAt": "2026-08-18T12:20:21Z" - }, - "evaluation": { - "mode": "attainment-only" - }, - "evidence": { - "key": "same-workflow-half-window-aic-and-reliability", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "A dispatched repository target in a seven-day window for which the completed first-half runs identify a highest-total-AIC workflow and that same workflow has completed and successful-run evidence in both halves.", - "filters": [ - "Targets are parsed from immutable display_title values of completed Optimization / AI Credit Optimizer Actions runs in the central repository.", - "The target workflow is the workflow with highest total AIC among completed first-half runs, with workflow name ascending as the stable tie-break.", - "The selected workflow must have at least one completed run and at least one successful run in each half.", - "AIC medians use successful runs only; failure rates use all completed runs and classify every conclusion other than success as non-successful.", - "Second-half median AIC must be strictly lower and second-half failure rate must be no greater than first-half failure rate." - ], - "collection": "Batch central Actions runs for all windows, deduplicate dispatched targets, invoke gh aw logs once per target for the total requested span, and derive each midpoint comparison locally from immutable target run IDs.", - "window": { - "durationDays": 7, - "cadenceDays": 7, - "maturationDays": 7 - } - }, - "model": { - "architecture": "One eligible opportunity per dispatched target-window, scored on two independently reported attainment dimensions.", - "recommendation": "Use efficient-and-reliable opportunity share as primary; retain lower-AIC share and reliability-preserved share as diagnostics. Reject recommendation and issue counts because they measure workflow output rather than repository outcome.", - "presentation": { - "label": "Efficient and reliable opportunities", - "betterLabel": "Higher is better" - } - }, - "summary": { - "nativeLabel": "Share of comparable target-windows with lower median successful-run AIC and preserved failure rate" - }, - "metrics": [ - { - "id": "efficient-reliable-share", - "name": "Efficient-and-reliable opportunity share", - "role": "primary", - "formula": "efficientReliableOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Efficient and reliable", - "legendLabel": "Efficient + reliable", - "transform": "identity" - } - }, - { - "id": "lower-aic-share", - "name": "Lower-AIC opportunity share", - "role": "diagnostic", - "formula": "lowerAicOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Lower median AIC", - "legendLabel": "Lower AIC", - "transform": "identity" - } - }, - { - "id": "reliability-preserved-share", - "name": "Reliability-preserved opportunity share", - "role": "diagnostic", - "formula": "reliabilityPreservedOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Reliability preserved", - "legendLabel": "Reliability preserved", - "transform": "identity" - } - } - ], - "validationExamples": { - "targetAttained": { - "status": "complete", - "comparableOpportunities": 1, - "efficientReliableOpportunities": 1, - "lowerAicOpportunities": 1, - "reliabilityPreservedOpportunities": 1 - }, - "targetMissed": { - "status": "complete", - "comparableOpportunities": 1, - "efficientReliableOpportunities": 0, - "lowerAicOpportunities": 0, - "reliabilityPreservedOpportunities": 0 - }, - "missing": { - "status": "missing", - "comparableOpportunities": 0, - "efficientReliableOpportunities": null, - "lowerAicOpportunities": null, - "reliabilityPreservedOpportunities": null - }, - "malformed": { - "status": "complete", - "comparableOpportunities": "one" - } - } - } - }, - "metricReview": { - "architecture": "One eligible opportunity per dispatched target-window, scored on two independently reported attainment dimensions.", - "recommendation": "Use efficient-and-reliable opportunity share as primary; retain lower-AIC share and reliability-preserved share as diagnostics. Reject recommendation and issue counts because they measure workflow output rather than repository outcome.", - "presentation": { - "label": "Efficient and reliable opportunities", - "betterLabel": "Higher is better" - }, - "metrics": [ - { - "id": "efficient-reliable-share", - "name": "Efficient-and-reliable opportunity share", - "role": "primary", - "formula": "efficientReliableOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Efficient and reliable", - "legendLabel": "Efficient + reliable", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - }, - { - "id": "lower-aic-share", - "name": "Lower-AIC opportunity share", - "role": "diagnostic", - "formula": "lowerAicOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Lower median AIC", - "legendLabel": "Lower AIC", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - }, - { - "id": "reliability-preserved-share", - "name": "Reliability-preserved opportunity share", - "role": "diagnostic", - "formula": "reliabilityPreservedOpportunities / comparableOpportunities", - "direction": "increase", - "presentation": { - "name": "Reliability preserved", - "legendLabel": "Reliability preserved", - "transform": "identity" - }, - "status": "unevaluated", - "beforeValue": null, - "afterValue": null, - "attainmentValue": null, - "improvement": null - } - ] - }, - "summary": { - "nativeLabel": "Share of comparable target-windows with lower median successful-run AIC and preserved failure rate" - }, - "snapshots": [ - { - "observedAt": "2026-08-18T17:54:55Z", - "window": { - "startAt": "2026-08-04T17:54:55Z", - "endAt": "2026-08-11T17:54:55Z" - }, - "evidence": { - "key": "same-workflow-half-window-aic-and-reliability", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Dispatched target-window with a comparable highest-first-half-AIC workflow", - "filters": [ - "completed runs", - "successful-run AIC medians", - "same workflow in both halves" - ], - "collection": "Central immutable dispatch runs and one gh aw logs JSON download per target over the batch span", - "window": { - "durationDays": 7, - "cadenceDays": 7, - "maturationDays": 7, - "windowStart": "2026-08-04T17:54:55Z", - "windowEnd": "2026-08-11T17:54:55Z", - "observedAt": "2026-08-18T17:54:55Z" - }, - "status": "missing", - "comparableOpportunities": 0, - "efficientReliableOpportunities": null, - "lowerAicOpportunities": null, - "reliabilityPreservedOpportunities": null, - "targets": [], - "targetResults": [], - "reason": "no-dispatched-target" - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "commit", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0" - } - ], - "metrics": { - "efficient-reliable-share": null, - "lower-aic-share": null, - "reliability-preserved-share": null - } - }, - { - "observedAt": "2026-08-20T18:33:18Z", - "window": { - "startAt": "2026-08-06T18:33:18Z", - "endAt": "2026-08-13T18:33:18Z" - }, - "evidence": { - "key": "same-workflow-half-window-aic-and-reliability", - "repositories": [ - "githubnext/central-agentic-ops" - ], - "opportunity": "Dispatched target-window with a comparable highest-first-half-AIC workflow", - "filters": [ - "completed runs", - "successful-run AIC medians", - "same workflow in both halves" - ], - "collection": "Central immutable dispatch runs and one gh aw logs JSON download per target over the batch span", - "window": { - "durationDays": 7, - "cadenceDays": 7, - "maturationDays": 7, - "windowStart": "2026-08-06T18:33:18Z", - "windowEnd": "2026-08-13T18:33:18Z", - "observedAt": "2026-08-20T18:33:18Z" - }, - "status": "missing", - "comparableOpportunities": 0, - "efficientReliableOpportunities": null, - "lowerAicOpportunities": null, - "reliabilityPreservedOpportunities": null, - "targets": [], - "targetResults": [], - "reason": "no-dispatched-target" - }, - "provenance": [ - { - "repository": "githubnext/central-agentic-ops", - "kind": "commit", - "ref": "35c7c3cbd319632f85784cce196e57c0f61db9a0" - } - ], - "metrics": { - "efficient-reliable-share": null, - "lower-aic-share": null, - "reliability-preserved-share": null - } - } - ] -} diff --git a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.svg b/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.svg deleted file mode 100644 index d92fcf6..0000000 --- a/.github/value/optimization-ai-credit-optimizer/optimization-ai-credit-optimizer-timeline.svg +++ /dev/null @@ -1,27 +0,0 @@ - -Optimization / AI Credit Optimizer workflow attainment timeline -Goal-oriented repository outcome metrics after workflow adoption, with workflow run conclusions over time. - - -Optimization / AI Credit Optimizer attainment over time -Post-adoption attainment -1 -0.75 -0.5 -0.25 -0 -Aug 18 -Aug 20 -Goal measure - - - - -Efficient + reliable -Lower AIC -Reliability preserved -Workflow adopted -Workflow runs - -SuccessFailureOther - diff --git a/.github/workflows/agentics-maintenance.yml b/.github/workflows/agentics-maintenance.yml index 1acdfed..26537e6 100644 --- a/.github/workflows/agentics-maintenance.yml +++ b/.github/workflows/agentics-maintenance.yml @@ -1,4 +1,4 @@ -# This file was automatically generated by pkg/workflow/maintenance_workflow.go (v0.87.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md +# This file was automatically generated by pkg/workflow/maintenance_workflow.go. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ # / _ \ | | (_) @@ -38,6 +38,11 @@ name: Agentic Maintenance on: schedule: - cron: "37 */12 * * *" # Every 12 hours (based on minimum expires: 3 days) + push: + branches: + - main + paths: + - '.github/workflows/*.md' workflow_dispatch: inputs: operation: @@ -93,8 +98,16 @@ jobs: permissions: discussions: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -114,8 +127,16 @@ jobs: permissions: issues: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -135,8 +156,16 @@ jobs: permissions: pull-requests: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -157,8 +186,16 @@ jobs: permissions: actions: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -189,7 +226,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -205,17 +242,21 @@ jobs: const { main } = require(path.join(actionsDir, 'check_team_member.cjs')); await main(); - - name: Install gh-aw - uses: github/gh-aw-actions/setup-cli@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - version: v0.87.2 + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build - name: Run operation uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_AW_OPERATION: ${{ inputs.operation }} - GH_AW_CMD_PREFIX: gh aw + GH_AW_CMD_PREFIX: ./gh-aw with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -239,8 +280,16 @@ jobs: contents: write pull-requests: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -291,7 +340,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -341,7 +390,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -357,15 +406,19 @@ jobs: const { main } = require(path.join(actionsDir, 'check_team_member.cjs')); await main(); - - name: Install gh-aw - uses: github/gh-aw-actions/setup-cli@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - version: v0.87.2 + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build - name: Create missing labels uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_CMD_PREFIX: gh aw + GH_AW_CMD_PREFIX: ./gh-aw with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -391,7 +444,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -407,10 +460,14 @@ jobs: const { main } = require(path.join(actionsDir, 'check_team_member.cjs')); await main(); - - name: Install gh-aw - uses: github/gh-aw-actions/setup-cli@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - version: v0.87.2 + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build - name: Restore activity report logs cache id: activity_report_logs_cache @@ -426,7 +483,7 @@ jobs: shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_AW_CMD_PREFIX: gh aw + GH_AW_CMD_PREFIX: ./gh-aw run: | ${GH_AW_CMD_PREFIX} logs \ --repo "$GITHUB_REPOSITORY" \ @@ -498,7 +555,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -514,10 +571,14 @@ jobs: const { main } = require(path.join(actionsDir, 'check_team_member.cjs')); await main(); - - name: Install gh-aw - uses: github/gh-aw-actions/setup-cli@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - version: v0.87.2 + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build - name: Restore forecast report logs cache id: forecast_report_logs_cache @@ -536,7 +597,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEBUG: "*" - GH_AW_CMD_PREFIX: gh aw + GH_AW_CMD_PREFIX: ./gh-aw run: | mkdir -p ./.cache/gh-aw/forecast set +e @@ -593,8 +654,16 @@ jobs: permissions: issues: write steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -635,7 +704,7 @@ jobs: persist-credentials: false - name: Setup Scripts - uses: github/gh-aw-actions/setup@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + uses: ./actions/setup with: destination: ${{ runner.temp }}/gh-aw/actions @@ -651,15 +720,19 @@ jobs: const { main } = require(path.join(actionsDir, 'check_team_member.cjs')); await main(); - - name: Install gh-aw - uses: github/gh-aw-actions/setup-cli@b304200a0ef4b3998673bfc7945acb08ab8c88b7 # v0.87.2 + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - version: v0.87.2 + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build - name: Validate workflows and file issue on findings uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_CMD_PREFIX: gh aw + GH_AW_CMD_PREFIX: ./gh-aw with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -669,3 +742,110 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require(path.join(actionsDir, 'run_validate_workflows.cjs')); await main(); + + compile-workflows: + if: ${{ (!(github.event.repository.fork)) && (github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' || inputs.operation == '') }} + runs-on: ubuntu-slim + concurrency: + group: ${{ github.workflow }}-compile-workflows-${{ github.repository }} + cancel-in-progress: true + permissions: + contents: read + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Build gh-aw + run: make build + + - name: Pre-compile validation + run: | + ./gh-aw compile --validate --no-emit --verbose + echo "✓ Pre-compile validation passed" + + - name: Compile workflows + run: | + ./gh-aw compile --validate --verbose + echo "✓ All workflows compiled successfully" + + - name: Setup Scripts + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + + - name: Check for out-of-sync workflows and create issue or pull request if needed + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_workflow_recompile_needed.cjs')); + await main(); + + secret-validation: + if: ${{ (!(github.event.repository.fork)) && github.event_name != 'push' && (github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' || inputs.operation == '') }} + runs-on: ubuntu-slim + permissions: + contents: read + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: '22' + + - name: Setup Scripts + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + + - name: Validate Secrets + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + # GitHub tokens + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + GH_AW_PROJECT_GITHUB_TOKEN: ${{ secrets.GH_AW_PROJECT_GITHUB_TOKEN }} + GH_AW_COPILOT_TOKEN: ${{ secrets.GH_AW_COPILOT_TOKEN }} + GH_AW_COPILOT_ORG_BILLING: "true" + # AI Engine API keys + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + # Integration tokens + NOTION_API_TOKEN: ${{ secrets.NOTION_API_TOKEN }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'validate_secrets.cjs')); + await main(); + + - name: Upload secret validation report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: secret-validation-report + path: secret-validation-report.md + retention-days: 30 + if-no-files-found: warn diff --git a/.github/workflows/dependabot-release-train-updater.md b/.github/workflows/dependabot-release-train-updater.md index 28c7aff..5f2f338 100644 --- a/.github/workflows/dependabot-release-train-updater.md +++ b/.github/workflows/dependabot-release-train-updater.md @@ -67,7 +67,9 @@ imports: permissions: contents: read actions: read + checks: read security-events: read + statuses: read vulnerability-alerts: read pull-requests: read issues: read @@ -121,6 +123,10 @@ tools: web-fetch: cache-memory: true +graders: + operational-value: + run: .github/graders/dependabot-release-train-updater-operational-value.sh + safe-outputs: staged: ${{ inputs.preview_only == 'true' }} create-pull-request: diff --git a/.github/workflows/optimization-ai-credit-auditor.md b/.github/workflows/optimization-ai-credit-auditor.md index c6a62d6..c954b87 100644 --- a/.github/workflows/optimization-ai-credit-auditor.md +++ b/.github/workflows/optimization-ai-credit-auditor.md @@ -75,6 +75,10 @@ concurrency: group: "${{ github.workflow }}-${{ inputs.target_repo }}" cancel-in-progress: true +graders: + operational-value: + run: .github/graders/optimization-ai-credit-auditor-operational-value.sh + tracker-id: optimization-ai-credit-auditor tools: @@ -133,6 +137,8 @@ steps: mkdir -p /tmp/gh-aw/token-audit PARTS_DIR=/tmp/gh-aw/token-audit/log-parts mkdir -p "$PARTS_DIR" + WINDOW_END=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at) + WINDOW_START=$(date -u -d "$WINDOW_END - 24 hours" +%Y-%m-%dT%H:%M:%SZ) # Fetch logs per workflow to avoid repo-wide pagination truncation in # high-CI-volume repositories. @@ -148,7 +154,7 @@ steps: PART_FILE="$PARTS_DIR/$SAFE_WORKFLOW_ID.json" PART_EXIT=0 gh aw logs "$WORKFLOW_ID" \ - --start-date -1d \ + --start-date -2d \ --json \ -c 100 \ > "$PART_FILE" || PART_EXIT=$? @@ -169,9 +175,12 @@ steps: done if [ "$FOUND_WORKFLOW" -eq 1 ] && ls "$PARTS_DIR"/*.json >/dev/null 2>&1; then - jq -s ' - (map(.runs // []) | add // [] | unique_by(.run_id)) as $runs | + jq -s --arg windowStart "$WINDOW_START" --arg windowEnd "$WINDOW_END" ' + (map(.runs // []) | add // [] | unique_by(.run_id) + | map(select(.created_at >= $windowStart and .created_at < $windowEnd))) as $runs | { + window_start: $windowStart, + window_end: $windowEnd, summary: { total_runs: ($runs | length), total_tokens: ($runs | map(.token_usage // 0) | add // 0), @@ -186,7 +195,9 @@ steps: if [ "$FOUND_WORKFLOW" -eq 0 ]; then echo "⚠️ No agentic workflow sources found under target/.github/workflows" fi - echo '{"runs":[],"summary":{}}' > /tmp/gh-aw/token-audit/workflow-logs.json + jq -cn --arg windowStart "$WINDOW_START" --arg windowEnd "$WINDOW_END" \ + '{window_start:$windowStart,window_end:$windowEnd,runs:[],summary:{}}' \ + > /tmp/gh-aw/token-audit/workflow-logs.json fi source: githubnext/central-agentic-ops/.github/workflows/optimization-ai-credit-auditor.md@main @@ -253,7 +264,7 @@ Previous snapshots live at `/tmp/gh-aw/repo-memory/default/`. For local runs, ea Write a Python script to `/tmp/gh-aw/token-audit/process_audit.py` and run it. The script must: -1. Load `/tmp/gh-aw/token-audit/workflow-logs.json` and extract `.runs`. +1. Load `/tmp/gh-aw/token-audit/workflow-logs.json`; preserve its `window_start` and `window_end`, and extract `.runs`. 2. Filter to `status == "completed"` runs only. 3. Group by `workflow_path` (falling back to `workflow_name` only when the path is absent) and compute per-workflow aggregates. Preserve both fields so distinct workflows with the same display name never merge: - `run_count`, `total_ai_credits`, `avg_ai_credits`, `total_tokens`, `avg_tokens`, `total_turns`, `avg_turns`, `total_action_minutes`, `error_count`, `warning_count` @@ -264,7 +275,9 @@ Write a Python script to `/tmp/gh-aw/token-audit/process_audit.py` and run it. T ```json { "date": "YYYY-MM-DD", - "period_days": 30, + "period_days": 1, + "window_start": "ISO-8601", + "window_end": "ISO-8601", "overall": { "total_runs": N, "total_ai_credits": F, diff --git a/.github/workflows/optimization-ai-credit-optimizer.md b/.github/workflows/optimization-ai-credit-optimizer.md index 848f112..d582137 100644 --- a/.github/workflows/optimization-ai-credit-optimizer.md +++ b/.github/workflows/optimization-ai-credit-optimizer.md @@ -66,6 +66,10 @@ concurrency: group: "${{ github.workflow }}-${{ inputs.target_repo }}" cancel-in-progress: true +graders: + operational-value: + run: .github/graders/optimization-ai-credit-optimizer-operational-value.sh + tracker-id: optimization-ai-credit-optimizer tools: @@ -97,6 +101,7 @@ steps: - name: Download recent agentic workflow logs env: GH_TOKEN: ${{ steps.github-mcp-app-token.outputs.token || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GH_REPO: ${{ inputs.target_repo }} run: | set -euo pipefail mkdir -p /tmp/gh-aw/token-audit @@ -106,7 +111,7 @@ steps: echo "📥 Downloading agentic workflow logs (last 7 days)..." FOUND_WORKFLOW=0 - for workflow in .github/workflows/*.md; do + for workflow in target/.github/workflows/*.md; do [ -f "$workflow" ] || continue WORKFLOW_ID=$(sed -n 's/^tracker-id:[[:space:]]*//p' "$workflow" | head -n 1 | tr -d '\r' | sed 's/[[:space:]]*$//') @@ -115,7 +120,7 @@ steps: # Skip the AI credit monitoring family in downstream repositories. # In the source repo (githubnext/central-agentic-ops) they remain valid targets; # in any other repo, optimization suggestions for them belong upstream. - if [[ "$GITHUB_REPOSITORY" != "githubnext/central-agentic-ops" && \ + if [[ "$TARGET_REPO" != "githubnext/central-agentic-ops" && \ ("$WORKFLOW_ID" == "optimization-ai-credit-optimizer" || "$WORKFLOW_ID" == "optimization-ai-credit-auditor") ]]; then echo "⏭️ Skipping $WORKFLOW_ID (AI credit monitoring family — optimize in githubnext/central-agentic-ops, not here)" continue @@ -168,7 +173,7 @@ steps: fi BEFORE_COUNT=$(jq '(.runs // []) | length' /tmp/gh-aw/token-audit/all-runs.json) - if [[ "$GITHUB_REPOSITORY" != "githubnext/central-agentic-ops" ]]; then + if [[ "$TARGET_REPO" != "githubnext/central-agentic-ops" ]]; then jq ' (.runs // []) | map(select( @@ -238,7 +243,8 @@ steps: run: | set -euo pipefail - OPT_LOG="/tmp/gh-aw/repo-memory/default/optimization-log.json" + TARGET_PREFIX=$(printf '%s' "$TARGET_REPO" | sed 's|/|__|') + OPT_LOG="/tmp/gh-aw/repo-memory/default/${TARGET_PREFIX}__optimization-log.json" if [ -f "$OPT_LOG" ]; then echo "✅ Previous optimizations:" jq -r '.[] | "\(.date): \(.workflow_name)"' "$OPT_LOG" @@ -385,11 +391,11 @@ Create one issue with: ## Phase 6 — Update Optimization Log -Append one entry to `/tmp/gh-aw/repo-memory/default/optimization-log.json`: +Append one entry to the target-specific optimization log at `/tmp/gh-aw/repo-memory/default/____optimization-log.json`. Derive `__` from `${{ inputs.target_repo }}`; do not write the unprefixed log for a dispatched target. -`{"date":"YYYY-MM-DD","workflow_name":"...","total_ai_credits_analyzed":F,"total_tokens_analyzed":N,"runs_audited":N,"recommendations_count":N,"subagent_candidates":N,"estimated_aic_savings_per_run":F}` +`{"date":"YYYY-MM-DD","target_repo":"${{ inputs.target_repo }}","workflow_name":"...","workflow_path":".github/workflows/....lock.yml","optimizer_run_id":"${{ github.run_id }}","total_ai_credits_analyzed":F,"total_tokens_analyzed":N,"runs_audited":N,"recommendations_count":N,"subagent_candidates":N,"estimated_aic_savings_per_run":F}` -Use `subagent_candidates` for the count of inline sub-agent candidates you actually recommend in the issue body. +Use the selected candidate's exact `workflow_path`; do not substitute its display name. Use `subagent_candidates` for the count of inline sub-agent candidates you actually recommend in the issue body. Load the existing array if present, append, keep only the last 30 entries, and save. diff --git a/.github/workflows/workflow-contracts.yml b/.github/workflows/workflow-contracts.yml index 269d6f8..8c20a63 100644 --- a/.github/workflows/workflow-contracts.yml +++ b/.github/workflows/workflow-contracts.yml @@ -21,10 +21,24 @@ jobs: cache: npm - name: Install dependencies run: npm ci - - name: Install gh-aw - env: - GH_TOKEN: ${{ github.token }} - run: gh extension install github/gh-aw --pin v0.87.2 + - name: Checkout grader-capable gh-aw + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + ref: 5cd744cc263a9d1ec5660fbf5604eaceb6f83430 + path: .cache/gh-aw + persist-credentials: false + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: .cache/gh-aw/go.mod + cache-dependency-path: .cache/gh-aw/go.sum + - name: Install grader-capable gh-aw + working-directory: .cache/gh-aw + run: | + go build -ldflags "-s -w -X main.version=v0.87.6" -o gh-aw ./cmd/gh-aw + gh extension remove aw || true + gh extension install . - name: Run release gate env: CENTRAL_AGENTIC_OPS_PACKAGE_SOURCE: ${{ github.repository }}@${{ github.sha }} diff --git a/.gitignore b/.gitignore index c7426c8..23012ca 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,6 @@ # This repo is the source only, compile happens at the repo where gh aw add is run .github/workflows/*.lock.yml -# Experimental value generation -.github/skills/aw-value - # Astro documentation site .astro/ dist/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..dbd4bd7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "github.copilot.enable": { + "markdown": true + } +} \ No newline at end of file diff --git a/aw.yml b/aw.yml index 578b200..d0d71b3 100644 --- a/aw.yml +++ b/aw.yml @@ -1,6 +1,6 @@ name: Central Agentic Ops description: Central agentic workflow control plane for reusable packages at the organization level -min-version: v0.87.2 +min-version: v0.87.6 includes: - .github/workflows/dependabot.md - .github/workflows/optimization.md \ No newline at end of file diff --git a/dependabot/aw.yml b/dependabot/aw.yml index e1be5b2..7369e39 100644 --- a/dependabot/aw.yml +++ b/dependabot/aw.yml @@ -1,5 +1,5 @@ name: Dependabot description: Immutable workflow for manifest-aware, risk-ranked dependency release-train updates. -min-version: v0.87.2 +min-version: v0.87.6 includes: - .github/workflows/dependabot.md \ No newline at end of file diff --git a/docs/orchestrators-and-workers.md b/docs/orchestrators-and-workers.md index b1e4cbd..42a8f8d 100644 --- a/docs/orchestrators-and-workers.md +++ b/docs/orchestrators-and-workers.md @@ -64,29 +64,33 @@ It does not receive a token, discovery query, or permission to dispatch another Operational value is measured per worker, not per orchestrator or operation. Dispatch counts, generated outputs, and model assessments do not prove that a worker attained its intended repository outcome. -The catalog repository keeps frozen contracts under `.github/ops-values/.sh`. Package manifests remain workflow-only while value evaluation is experimental, so neither these contracts nor the `aw-value` authoring and report-generation skill are installed into consumer repositories. +Each adopted worker registers a frozen schema-version 4 evaluator under `.github/graders/-operational-value.sh`. gh-aw executes that evaluator for the workflow run, records its assigned opportunity and evidence provenance, and publishes the result in the unified `agent` artifact's `grader_results.json`. -A frozen function exposes its contract with `--definition`, scores evidence with `--metric`, and collects batched evidence with `--collect-batch`. Authoring new functions and generating reports remain catalog-maintenance tasks until the skill is ready to ship. +An evaluator exposes its contract with `--definition`, scores evidence with `--metric`, and observes one run with `--grade-run`. Pages reads these actual workflow artifacts; it does not recollect repository history or render committed synthetic timelines. It retains run observations through their maturity horizon, regrades due runs from a trusted checkout with the frozen evaluator, and aggregates only the latest evaluator digest. Repeated runs assigned to one opportunity are collapsed before aggregation. ```bash -VALUE_FUNCTION=".github/ops-values/.sh" +EVALUATOR=".github/graders/-operational-value.sh" -"$VALUE_FUNCTION" --definition -"$VALUE_FUNCTION" --metric < evidence.json -"$VALUE_FUNCTION" --collect-batch < repositories.json +"$EVALUATOR" --definition +"$EVALUATOR" --metric < evidence.json +gh aw graders operational-value RUN_ID --evidence-at TIMESTAMP --json ``` :::tip[Measure repository outcomes] Count an outcome only when accepted evidence satisfies the worker's frozen contract. A successful dispatch or generated suggestion is activity, not attained value. ::: +:::caution[Verify package transport] +A packaged worker is grader-enabled only when a clean `gh aw add` consumer receives both its Markdown workflow and referenced `.github/graders/*.sh` evaluator. The gh-aw operational-value merge commit validates and freezes evaluators but its package installer does not yet transport that directory, so publishing these grader-enabled bundles remains blocked on installer support. Direct checkouts of this repository compile and run the graders. +::: + Apply the process independently to every worker in an operation. Workers may receive different classifications because their outcomes and available history differ: - `baseline-comparable` applies the same outcome measure before and after adoption; - `attainment-only` measures post-adoption attainment when comparable history cannot be reconstructed; - `not measurable` records that no deterministic opportunity, outcome, or accepted-evidence rule can currently be defined. -Do not create placeholder functions or reports while a new worker is unadopted. A frozen function requires its real adoption commit, and evaluation requires matured outcome evidence. The operation creation skill records this as a post-adoption follow-up for each new worker. +Do not create placeholder evaluators while a new worker is unadopted. A frozen evaluator requires its real adoption commit and a stable run-to-opportunity assignment. The operation creation skill records this as a post-adoption follow-up for each new worker. ## Current Worker Eligibility diff --git a/optimization/aw.yml b/optimization/aw.yml index 9414cf6..e649dda 100644 --- a/optimization/aw.yml +++ b/optimization/aw.yml @@ -1,5 +1,5 @@ name: Optimization description: Organization-wide workflow for auditing and optimizing GitHub Agentic Workflow AI credit usage. -min-version: v0.87.2 +min-version: v0.87.6 includes: - .github/workflows/optimization.md \ No newline at end of file diff --git a/pages/pages.yml b/pages/pages.yml index 3887774..70b9f0e 100644 --- a/pages/pages.yml +++ b/pages/pages.yml @@ -49,6 +49,27 @@ jobs: with: version: v0.87.2 + - name: Checkout operational-value replay runtime + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + ref: 5cd744cc263a9d1ec5660fbf5604eaceb6f83430 + path: .cache/gh-aw-operational-value-source + persist-credentials: false + + - name: Setup Go for operational-value replay + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: .cache/gh-aw-operational-value-source/go.mod + cache-dependency-path: .cache/gh-aw-operational-value-source/go.sum + + - name: Build operational-value replay runtime + working-directory: .cache/gh-aw-operational-value-source + run: | + mkdir -p "$RUNNER_TEMP/gh-aw-operational-value" + go build -ldflags "-s -w -X main.version=v0.87.6" \ + -o "$RUNNER_TEMP/gh-aw-operational-value/gh-aw" ./cmd/gh-aw + - name: Restore AI Credit usage cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -57,6 +78,14 @@ jobs: restore-keys: | ${{ runner.os }}-pages-aic-${{ github.repository }}- + - name: Restore operational-value observation cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .cache/pages-operational-values + key: ${{ runner.os }}-pages-operational-values-${{ github.repository }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-pages-operational-values-${{ github.repository }}- + - name: Collect AI Credit usage env: GH_TOKEN: ${{ github.token }} @@ -66,6 +95,16 @@ jobs: REPORT_DEPLOYED_WORKFLOWS: ${{ runner.temp }}/deployed-workflows.json run: node .github/scripts/pages-report/aic-usage.mjs + - name: Collect operational-value observations + env: + GH_TOKEN: ${{ github.token }} + REPORT_DEPLOYED_WORKFLOWS: ${{ runner.temp }}/deployed-workflows.json + REPORT_GH_AW_BIN: ${{ runner.temp }}/gh-aw-operational-value/gh-aw + REPORT_OPERATIONAL_VALUES: ${{ runner.temp }}/operational-values.json + REPORT_VALUE_CACHE: .cache/pages-operational-values/observations.json + REPORT_VALUE_CONCURRENCY: "3" + run: node .github/scripts/pages-report/operational-values.mjs + - name: Save AI Credit usage cache if: always() uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -73,6 +112,13 @@ jobs: path: .cache/pages-aic key: ${{ runner.os }}-pages-aic-${{ github.repository }}-${{ github.run_id }} + - name: Save operational-value observation cache + if: always() + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .cache/pages-operational-values + key: ${{ runner.os }}-pages-operational-values-${{ github.repository }}-${{ github.run_id }} + - name: Build operations report env: GITHUB_TOKEN: ${{ github.token }} @@ -81,8 +127,8 @@ jobs: REPORT_AIC_USAGE: ${{ runner.temp }}/aic-usage.json REPORT_INVENTORY: ${{ runner.temp }}/control-plane-inventory.json REPORT_DEPLOYED_WORKFLOWS: ${{ runner.temp }}/deployed-workflows.json + REPORT_OPERATIONAL_VALUES: ${{ runner.temp }}/operational-values.json REPORT_OUTPUT: _site - REPORT_VALUE_ROOT: .github/value run: node .github/scripts/pages-report/report.mjs - name: Upload Pages artifact diff --git a/tests/README.md b/tests/README.md index ad979cb..b01ef34 100644 --- a/tests/README.md +++ b/tests/README.md @@ -157,7 +157,8 @@ Compilation checks prove the source policy reaches the generated GitHub Actions | 🟢 Pass | All worker workflow safe outputs | staged mode and review/live routing vocabulary checked. | | 🟢 Pass | All five generated workflows | Emitted GitHub Actions settings checked in a clean-room compile. | | 🟢 Pass | Core catalog package | Installs no Pages workflow, renderer, or Pages permission surface. | -| 🟢 Pass | Experimental ops values | Remain catalog-local under `.github/ops-values/` and are excluded from package manifests. | -| 🟢 Pass | Pages add-on | Explicit nested package contains only the conventional publisher and report skill. | +| 🟢 Pass | Operational value | Schema-v4 evaluators are registered by workers and Pages consumes actual `grader_results.json` observations. | +| 🟢 Pass | Pages add-on | Conventional publisher remains outside the reusable Agentic Workflow packages. | +| 🟡 Upstream blocked | Grader package transport | The gh-aw operational-value merge commit does not install referenced `.github/graders/*.sh` files into a clean package consumer. | Exhaustive coverage: 18 scheduled plus 108 manual cases, for 126 unique policy configurations. \ No newline at end of file diff --git a/tests/unit/workflow-contract.test.mjs b/tests/unit/workflow-contract.test.mjs index c36b11d..e504889 100644 --- a/tests/unit/workflow-contract.test.mjs +++ b/tests/unit/workflow-contract.test.mjs @@ -309,41 +309,59 @@ test("deterministic workflows pin third-party actions by commit SHA", () => { } }); -test("package manifests exclude repository-only tests and experimental ops values", () => { +test("package manifests exclude repository-only tests", () => { for (const relativePath of ["aw.yml", join("dependabot", "aw.yml"), join("optimization", "aw.yml")]) { const manifest = readFileSync(join(root, relativePath), "utf8"); - assert.doesNotMatch(manifest, /(?:\.github\/)?ops-values/, relativePath); assert.doesNotMatch(manifest, /(?:staged-smoke|enterprise-canary|enterprise-stress|tests\/e2e|\.github\/aw\/e2e)/, relativePath); } }); -test("ops-value contracts expose deterministic validation examples", () => { - const opsValuesDirectory = join(root, ".github", "ops-values"); - const opsValues = readdirSync(opsValuesDirectory).filter((name) => name.endsWith(".sh")).sort(); - assert.deepEqual(opsValues, [ - "dependabot-release-train-updater.sh", - "optimization-ai-credit-auditor.sh", - "optimization-ai-credit-optimizer.sh", +test("operational-value graders expose deterministic run-scoped contracts", () => { + const gradersDirectory = join(root, ".github", "graders"); + const graders = readdirSync(gradersDirectory).filter((name) => name.endsWith("-operational-value.sh")).sort(); + assert.deepEqual(graders, [ + "dependabot-release-train-updater-operational-value.sh", + "optimization-ai-credit-auditor-operational-value.sh", + "optimization-ai-credit-optimizer-operational-value.sh", ]); - for (const name of opsValues) { - const executable = join(opsValuesDirectory, name); + for (const name of graders) { + const executable = join(gradersDirectory, name); + const workflowName = name.replace(/-operational-value\.sh$/, ".md"); + assert.match( + workflow(workflowName), + new RegExp(`graders:\\s+operational-value:\\s+run: \\.github/graders/${name.replace(".", "\\.")}`), + `${name}: workflow must execute the frozen operational-value evaluator`, + ); const definition = JSON.parse(execFileSync(executable, ["--definition"], { encoding: "utf8" })); - assert.equal(definition.schemaVersion, 3, name); - assert.equal(definition.metrics.filter(({ role }) => role === "primary").length, 1, name); - - for (const metric of definition.metrics) { - const score = (example) => JSON.parse(execFileSync(executable, ["--metric", metric.id], { - encoding: "utf8", - input: JSON.stringify(definition.validationExamples[example]), - })); - assert.ok(score("targetAttained") > score("targetMissed"), `${name}: ${metric.id}`); - if (metric.role === "primary") { - assert.equal(score("missing"), null, `${name}: ${metric.id} missing`); - assert.equal(score("malformed"), null, `${name}: ${metric.id} malformed`); - } - } + assert.equal(definition.schemaVersion, 4, name); + assert.equal(definition.grader, "operational-value", name); + const score = (example) => JSON.parse(execFileSync(executable, ["--metric"], { + encoding: "utf8", + input: JSON.stringify(definition.validationExamples[example]), + })); + assert.ok(score("targetAttained") > score("targetMissed"), name); + assert.equal(score("missing"), null, `${name}: missing`); + assert.equal(score("malformed"), null, `${name}: malformed`); } + + const dependabotWorker = workflow("dependabot-release-train-updater.md"); + const auditorWorker = workflow("optimization-ai-credit-auditor.md"); + const auditorEvaluator = readFileSync(join(gradersDirectory, "optimization-ai-credit-auditor-operational-value.sh"), "utf8"); + const optimizerWorker = workflow("optimization-ai-credit-optimizer.md"); + const optimizerEvaluator = readFileSync(join(gradersDirectory, "optimization-ai-credit-optimizer-operational-value.sh"), "utf8"); + assert.match(dependabotWorker, /checks: read/); + assert.match(dependabotWorker, /statuses: read/); + assert.match(auditorWorker, /window_start: \$windowStart/); + assert.match(auditorWorker, /window_end: \$windowEnd/); + assert.match(auditorEvaluator, /workflow_path \/\/ \.workflow_name/); + assert.match(auditorEvaluator, /evidenceRepo: \.run\.repository/); + assert.match(optimizerWorker, /GH_REPO: \$\{\{ inputs\.target_repo \}\}/); + assert.match(optimizerWorker, /for workflow in target\/\.github\/workflows\/\*\.md/); + assert.match(optimizerWorker, /\$\{TARGET_PREFIX\}__optimization-log\.json/); + assert.match(optimizerWorker, /"optimizer_run_id":"\$\{\{ github\.run_id \}\}"/); + assert.match(optimizerEvaluator, /\.optimizer_run_id \| tostring/); + assert.match(optimizerEvaluator, /target-workflow:\$\{target_repo\}:\$\{workflow\}:\$\{optimizer_run_id\}/); }); test("staged smoke is manual, bounded, and cannot request writes", () => { @@ -644,12 +662,14 @@ test("clean-room compilation emits the expected GitHub Actions settings", { time test("Pages is an explicit least-privilege add-on", () => { const rootManifest = readFileSync(join(root, "aw.yml"), "utf8"); const pagesWorkflow = readFileSync(join(root, "pages", "pages.yml"), "utf8"); - const reportAssets = ["aic-usage.mjs", "deployed-workflows.mjs", "inventory.mjs", "report.mjs"]; + const reportAssets = ["aic-usage.mjs", "deployed-workflows.mjs", "inventory.mjs", "operational-values.mjs", "report.mjs"]; assert.doesNotMatch(rootManifest, /pages\/pages|pages-report/); assert.ok(!existsSync(join(root, "pages", "aw.yml")), "Pages must not masquerade as an Agentic Workflow package"); assert.match(pagesWorkflow, /pages: write/); assert.match(pagesWorkflow, /id-token: write/); + assert.match(pagesWorkflow, /REPORT_VALUE_CACHE: \.cache\/pages-operational-values\/observations\.json/); + assert.match(pagesWorkflow, /Save operational-value observation cache/); for (const assetName of reportAssets) { assert.ok(existsSync(join(root, ".github", "scripts", "pages-report", assetName)), `missing report script ${assetName}`); assert.match(pagesWorkflow, new RegExp(`\\.github/scripts/pages-report/${assetName.replace(".", "\\.")}`));