diff --git a/.github/workflows/ci-eval.yml b/.github/workflows/ci-eval.yml index 6967ea6b05a09b..d2221622627895 100644 --- a/.github/workflows/ci-eval.yml +++ b/.github/workflows/ci-eval.yml @@ -231,7 +231,7 @@ jobs: with: persist-credentials: false - - name: Preserve the trusted eval specs + - name: Preserve the trusted eval inputs if: steps.guard.outputs.run == 'true' env: SPECS: ${{ steps.sel.outputs.specs }} @@ -240,6 +240,18 @@ jobs: for spec in $SPECS; do cp "$spec" "$RUNNER_TEMP/trusted-evals/$(basename "$spec")" done + if test -f .github/workflows/evals/search-kbe-issues.cjs; then + cp .github/workflows/evals/search-kbe-issues.cjs \ + "$RUNNER_TEMP/trusted-evals/search-kbe-issues.cjs" + fi + if test -f .github/workflows/evals/kbe-candidate-reads-grader.mjs; then + cp .github/workflows/evals/kbe-candidate-reads-grader.mjs \ + "$RUNNER_TEMP/trusted-evals/kbe-candidate-reads-grader.mjs" + fi + if test -f .github/workflows/evals/kbe-search.test.mjs; then + cp .github/workflows/evals/kbe-search.test.mjs \ + "$RUNNER_TEMP/trusted-evals/kbe-search.test.mjs" + fi if [ -f .github/workflows/shared/filter-scanner-kbes.sh ]; then cp .github/workflows/shared/filter-scanner-kbes.sh \ "$RUNNER_TEMP/trusted-evals/filter-scanner-kbes.sh" @@ -265,7 +277,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: gh pr checkout ${{ github.event.issue.number }} - - name: Restore the trusted eval specs + - name: Restore the trusted eval inputs if: steps.guard.outputs.run == 'true' env: SPECS: ${{ steps.sel.outputs.specs }} @@ -274,19 +286,45 @@ jobs: rm -f -- "$spec" cp -- "$RUNNER_TEMP/trusted-evals/$(basename "$spec")" "$spec" done + rm -f -- .github/workflows/evals/search-kbe-issues.cjs + if test -f "$RUNNER_TEMP/trusted-evals/search-kbe-issues.cjs"; then + cp -- "$RUNNER_TEMP/trusted-evals/search-kbe-issues.cjs" \ + .github/workflows/evals/search-kbe-issues.cjs + fi + rm -f -- .github/workflows/evals/kbe-candidate-reads-grader.mjs + if test -f "$RUNNER_TEMP/trusted-evals/kbe-candidate-reads-grader.mjs"; then + cp -- "$RUNNER_TEMP/trusted-evals/kbe-candidate-reads-grader.mjs" \ + .github/workflows/evals/kbe-candidate-reads-grader.mjs + fi + rm -f -- .github/workflows/evals/kbe-search.test.mjs + if test -f "$RUNNER_TEMP/trusted-evals/kbe-search.test.mjs"; then + cp -- "$RUNNER_TEMP/trusted-evals/kbe-search.test.mjs" \ + .github/workflows/evals/kbe-search.test.mjs + fi if [ -f "$RUNNER_TEMP/trusted-evals/filter-scanner-kbes.sh" ]; then mkdir -p .github/workflows/shared cp -- "$RUNNER_TEMP/trusted-evals/filter-scanner-kbes.sh" \ .github/workflows/shared/filter-scanner-kbes.sh fi + - name: Test the KBE search support + if: steps.guard.outputs.run == 'true' + run: | + if test -f .github/workflows/evals/kbe-search.test.mjs; then + node --test .github/workflows/evals/kbe-search.test.mjs + fi + - name: Lint the selected spec(s) if: steps.guard.outputs.run == 'true' env: SPECS: ${{ steps.sel.outputs.specs }} run: | + grader_args=() + if test -f .github/workflows/evals/kbe-candidate-reads-grader.mjs; then + grader_args+=(--grader-plugin ./kbe-candidate-reads-grader.mjs) + fi for spec in $SPECS; do - vally lint --eval-spec "$spec" --strict + vally lint --eval-spec "$spec" --strict "${grader_args[@]}" done - name: Run the eval(s) @@ -317,11 +355,16 @@ jobs: report="$workdir/report.md" : > "$report" rc=0 + grader_args=() + if test -f .github/workflows/evals/kbe-candidate-reads-grader.mjs; then + grader_args+=(--grader-plugin ./kbe-candidate-reads-grader.mjs) + fi for spec in $SPECS; do name="$(basename "$spec" .eval.yaml)" echo "## $name" >> "$report" if vally eval --eval-spec "$spec" --skill-dir .github/workflows \ - --workspace "$workdir/ws-$name" --output-dir "$workdir/out-$name" > "$workdir/log-$name.txt" 2>&1; then + --workspace "$workdir/ws-$name" --output-dir "$workdir/out-$name" \ + "${grader_args[@]}" > "$workdir/log-$name.txt" 2>&1; then echo "✅ passed" >> "$report" else rc=1 diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index c7d8e6ef6d0ac3..81ba27826f595d 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9098a4e3e6ccdd1f9f936b64ee6ac4362232803d71959645d9e6023543fec795","body_hash":"8258800d731e8cf437014b12f868c5aec40be23451abe644e2dd1f3b68ef40c5","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"075c9576e8d93729320794053448cb6b1355033a9f89ca0fb13bf2009009fb14","body_hash":"53d288e3fa0f03bf81833afa5d07a53d7ab9421c5e5eb3dc38bf68f0e77805f2","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"}]} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -61,7 +61,6 @@ # - ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627 # - ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f # - ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 -# - ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e name: "CI Outer-Loop Failure Scanner" on: @@ -304,7 +303,7 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" + GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `mcpscripts` — run `mcpscripts --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} with: script: | @@ -435,6 +434,11 @@ jobs: with: persist-credentials: false fetch-depth: 50 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false - name: Create gh-aw temp directory run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" - name: Configure gh CLI for GitHub Enterprise @@ -482,6 +486,7 @@ jobs: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} GH_AW_GITHUB_MIN_INTEGRITY: 'approved' + GH_AW_GITHUB_REPOS: '["dotnet/runtime"]' with: script: | const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); @@ -509,7 +514,7 @@ jobs: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7 ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627 ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7 ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627 ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 - name: Generate Safe Outputs Config run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" @@ -648,15 +653,156 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); await main(); + - name: Write MCP Scripts Config + run: | + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-scripts/logs" + cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/tools.json" << 'GH_AW_MCP_SCRIPTS_TOOLS_a77c230c2abaee0f_EOF' + { + "serverName": "mcpscripts", + "version": "1.0.0", + "logDir": "${RUNNER_TEMP}/gh-aw/mcp-scripts/logs", + "tools": [ + { + "name": "search-kbe-issues", + "description": "Search only dotnet/runtime issues and return inert candidate identifiers with author metadata. Use issue_read to inspect every candidate.", + "inputSchema": { + "properties": { + "query": { + "description": "GitHub issue search query without a repo qualifier. The tool adds repo:dotnet/runtime and is:issue.", + "type": "string" + } + }, + "required": [ + "query" + ], + "type": "object" + }, + "handler": "search-kbe-issues.cjs", + "env": { + "GITHUB_TOKEN": "GITHUB_TOKEN" + }, + "timeout": 60 + } + ] + } + GH_AW_MCP_SCRIPTS_TOOLS_a77c230c2abaee0f_EOF + cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/mcp-server.cjs" << 'GH_AW_MCP_SCRIPTS_SERVER_e0a10bf70e571677_EOF' + const path = require("path"); + const { startHttpServer } = require("./mcp_scripts_mcp_server_http.cjs"); + const configPath = path.join(__dirname, "tools.json"); + const port = parseInt(process.env.GH_AW_MCP_SCRIPTS_PORT || "3000", 10); + const apiKey = process.env.GH_AW_MCP_SCRIPTS_API_KEY || ""; + startHttpServer(configPath, { + port: port, + stateless: true, + logDir: "${RUNNER_TEMP}/gh-aw/mcp-scripts/logs" + }).catch(error => { + console.error("Failed to start mcp-scripts HTTP server:", error); + process.exit(1); + }); + GH_AW_MCP_SCRIPTS_SERVER_e0a10bf70e571677_EOF + chmod +x "${RUNNER_TEMP}/gh-aw/mcp-scripts/mcp-server.cjs" + + - name: Write MCP Scripts Tool Files + run: | + cat > "${RUNNER_TEMP}/gh-aw/mcp-scripts/search-kbe-issues.cjs" << 'GH_AW_MCP_SCRIPTS_JS_SEARCH-KBE-ISSUES_bb368a8ea03af812_EOF' + async function execute(inputs) { + const { query } = inputs || {}; + if (typeof query !== "string" || !query.trim()) { + throw new Error("query must be a non-empty string"); + } + const token = process.env.GITHUB_TOKEN; + if (!token) { + throw new Error("GITHUB_TOKEN must be set"); + } + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const { stdout } = await promisify(execFile)("gh", [ + "api", + "search/issues", + "--method", + "GET", + "--raw-field", + `q=${query.trim()} repo:dotnet/runtime is:issue`, + "--field", + "per_page=10" + ], { + env: { ...process.env, GH_TOKEN: token, GITHUB_TOKEN: token }, + maxBuffer: 1024 * 1024 + }); + let result; + try { + result = JSON.parse(stdout); + } catch { + throw new Error("gh issue search returned invalid JSON"); + } + if (result.incomplete_results !== false || !Array.isArray(result.items)) { + throw new Error("GitHub issue search returned an invalid response"); + } + return result.items.map((item) => { + if (item.repository_url !== "https://api.github.com/repos/dotnet/runtime" || + item.pull_request !== undefined || + !Number.isInteger(item.number) || + typeof item.user?.login !== "string" || + item.user.login.length === 0) { + throw new Error("GitHub issue search returned an invalid candidate"); + } + return { + number: item.number, + user: { login: item.user.login } + }; + }); + } + module.exports = { execute }; + if (require.main === module) { + require("./mcp-scripts-runner.cjs")(execute); + } + GH_AW_MCP_SCRIPTS_JS_SEARCH-KBE-ISSUES_bb368a8ea03af812_EOF + + - name: Generate MCP Scripts Server Config + id: mcp-scripts-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3000 + + # Set outputs for next steps + { + echo "mcp_scripts_api_key=${API_KEY}" + echo "mcp_scripts_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "MCP Scripts server will run on port ${PORT}" + + - name: Start MCP Scripts HTTP Server + id: mcp-scripts-start + env: + DEBUG: '*' + GH_AW_MCP_SCRIPTS_PORT: ${{ steps.mcp-scripts-config.outputs.mcp_scripts_port }} + GH_AW_MCP_SCRIPTS_API_KEY: ${{ steps.mcp-scripts-config.outputs.mcp_scripts_api_key }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_MCP_SCRIPTS_PORT + export GH_AW_MCP_SCRIPTS_API_KEY + + bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_scripts_server.sh" + - name: Start MCP Gateway id: start-mcp-gateway env: + GH_AW_MCP_SCRIPTS_API_KEY: ${{ steps.mcp-scripts-start.outputs.api_key }} + GH_AW_MCP_SCRIPTS_PORT: ${{ steps.mcp-scripts-start.outputs.port }} GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eo pipefail @@ -676,36 +822,62 @@ jobs: export DEBUG="*" export GH_AW_ENGINE="copilot" + export GITHUB_PERSONAL_ACCESS_TOKEN="$GITHUB_MCP_SERVER_TOKEN" MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.9' + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_PERSONAL_ACCESS_TOKEN -e GH_AW_MCP_SCRIPTS_PORT -e GH_AW_MCP_SCRIPTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.9' mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_5421d352a29bcb5b_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_421f01deaa5e6d14_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { - "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.9.0", + "type": "http", + "url": "https://api.githubcopilot.com/mcp/", + "headers": { + "Authorization": "Bearer \${GITHUB_PERSONAL_ACCESS_TOKEN}", + "X-MCP-Features": "fields_param", + "X-MCP-Readonly": "true", + "X-MCP-Toolsets": "pull_requests,issues" + }, + "tools": [ + "issue_read", + "search_pull_requests", + "pull_request_read" + ], "env": { - "GITHUB_FEATURES": "fields_param", "GITHUB_HOST": "${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", - "GITHUB_READ_ONLY": "1", - "GITHUB_TOOLSETS": "pull_requests,repos,issues,search" + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}" }, "guard-policies": { "allow-only": { "approval-labels": ${{ steps.parse-guard-vars.outputs.approval_labels }}, "blocked-users": ${{ steps.parse-guard-vars.outputs.blocked_users }}, "min-integrity": "approved", - "repos": "all", + "repos": [ + "dotnet/runtime" + ], "trusted-users": ${{ steps.parse-guard-vars.outputs.trusted_users }} } } }, + "mcpscripts": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_MCP_SCRIPTS_PORT", + "headers": { + "Authorization": "\${GH_AW_MCP_SCRIPTS_API_KEY}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "private:dotnet/runtime" + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" + } + } + }, "safeoutputs": { "type": "stdio", "container": "ghcr.io/github/gh-aw-node", @@ -733,7 +905,7 @@ jobs: "guard-policies": { "write-sink": { "accept": [ - "*" + "private:dotnet/runtime" ], "sink-visibility": "${GH_AW_SINK_VISIBILITY}" } @@ -748,7 +920,7 @@ jobs: "startupTimeout": 120 } } - GH_AW_MCP_CONFIG_5421d352a29bcb5b_EOF + GH_AW_MCP_CONFIG_421f01deaa5e6d14_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -773,7 +945,10 @@ jobs: - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): - # --allow-tool github + # --allow-tool github(issue_read) + # --allow-tool github(pull_request_read) + # --allow-tool github(search_pull_requests) + # --allow-tool mcpscripts # --allow-tool safeoutputs # --allow-tool shell(awk) # --allow-tool shell(basename) @@ -794,6 +969,7 @@ jobs: # --allow-tool shell(head) # --allow-tool shell(jq) # --allow-tool shell(ls) + # --allow-tool shell(mcpscripts:*) # --allow-tool shell(mkdir) # --allow-tool shell(printf) # --allow-tool shell(pwd) @@ -837,7 +1013,7 @@ jobs: export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-1000}" - printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.44/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.blob.core.windows.net\",\"*.githubusercontent.com\",\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"dev.azure.com\",\"docs.github.com\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.blog\",\"github.com\",\"github.githubassets.com\",\"helix.dot.net\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"patch-diff.githubusercontent.com\",\"patchdiff.githubusercontent.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"telemetry.enterprise.githubcopilot.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.44,squid=sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627,agent=sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4,api-proxy=sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7,cli-proxy=sha256:c064d15974f7c933ec7d3f7b4038f4fd203547b3154bdc821afd379144887eff\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.44/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.blob.core.windows.net\",\"*.githubusercontent.com\",\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"dev.azure.com\",\"docs.github.com\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.blog\",\"github.com\",\"github.githubassets.com\",\"helix.dot.net\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"patch-diff.githubusercontent.com\",\"patchdiff.githubusercontent.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"telemetry.enterprise.githubcopilot.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"blockDomains\":[\"*.githubusercontent.com\",\"api.github.com\",\"codeload.github.com\",\"docs.github.com\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.blog\",\"github.com\",\"github.githubassets.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"patch-diff.githubusercontent.com\",\"patchdiff.githubusercontent.com\",\"raw.githubusercontent.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.44,squid=sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627,agent=sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4,api-proxy=sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7,cli-proxy=sha256:c064d15974f7c933ec7d3f7b4038f4fd203547b3154bdc821afd379144887eff\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" GH_AW_DOCKER_HOST="" @@ -855,8 +1031,8 @@ jobs: fi fi # shellcheck disable=SC1003,SC2016,SC2086 - awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(awk)'\'' --allow-tool '\''shell(basename)'\'' --allow-tool '\''shell(bash)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(chmod)'\'' --allow-tool '\''shell(curl:*)'\'' --allow-tool '\''shell(cut)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dirname)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(env)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(mkdir)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sh)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(tee)'\'' --allow-tool '\''shell(test)'\'' --allow-tool '\''shell(tr)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(xargs)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env GITHUB_TOKEN --exclude-env MCP_GATEWAY_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool '\''github(issue_read)'\'' --allow-tool '\''github(pull_request_read)'\'' --allow-tool '\''github(search_pull_requests)'\'' --allow-tool mcpscripts --allow-tool safeoutputs --allow-tool '\''shell(awk)'\'' --allow-tool '\''shell(basename)'\'' --allow-tool '\''shell(bash)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(chmod)'\'' --allow-tool '\''shell(curl:*)'\'' --allow-tool '\''shell(cut)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dirname)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(env)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(mcpscripts:*)'\'' --allow-tool '\''shell(mkdir)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sh)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(tee)'\'' --allow-tool '\''shell(test)'\'' --allow-tool '\''shell(tr)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(xargs)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -875,10 +1051,11 @@ jobs: GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_WORKSPACE: ${{ github.workspace }} GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com GIT_AUTHOR_NAME: github-actions[bot] @@ -971,6 +1148,15 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); await main(); + - name: Parse MCP Scripts logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_scripts_logs.cjs'); + await main(); - name: Parse MCP Gateway logs for step summary if: always() id: parse-mcp-gateway @@ -1026,6 +1212,7 @@ jobs: /tmp/gh-aw/mcp-logs/ /tmp/gh-aw/proxy-logs/ !/tmp/gh-aw/proxy-logs/proxy-tls/ + /tmp/gh-aw/mcp-scripts/logs/ /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent-stdio.log /tmp/gh-aw/pre-agent-audit.txt diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 9ebecc66e39f72..228d8a5dfbe52f 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -41,11 +41,71 @@ concurrency: tools: github: - toolsets: [pull_requests, repos, issues, search] + type: remote + github-token: ${{ secrets.GITHUB_TOKEN }} + toolsets: [pull_requests, issues] + allowed: [issue_read, search_pull_requests, pull_request_read] + allowed-repos: [dotnet/runtime] min-integrity: approved edit: bash: ["dotnet", "git", "find", "ls", "cat", "grep", "head", "tail", "wc", "curl", "jq", "tee", "sed", "awk", "tr", "cut", "sort", "uniq", "xargs", "echo", "date", "mkdir", "test", "env", "basename", "dirname", "bash", "sh", "chmod"] +mcp-scripts: + search-kbe-issues: + description: "Search only dotnet/runtime issues and return inert candidate identifiers with author metadata. Use issue_read to inspect every candidate." + inputs: + query: + type: string + required: true + description: "GitHub issue search query without a repo qualifier. The tool adds repo:dotnet/runtime and is:issue." + script: | + if (typeof query !== "string" || !query.trim()) { + throw new Error("query must be a non-empty string"); + } + const token = process.env.GITHUB_TOKEN; + if (!token) { + throw new Error("GITHUB_TOKEN must be set"); + } + const { execFile } = await import("node:child_process"); + const { promisify } = await import("node:util"); + const { stdout } = await promisify(execFile)("gh", [ + "api", + "search/issues", + "--method", + "GET", + "--raw-field", + `q=${query.trim()} repo:dotnet/runtime is:issue`, + "--field", + "per_page=10" + ], { + env: { ...process.env, GH_TOKEN: token, GITHUB_TOKEN: token }, + maxBuffer: 1024 * 1024 + }); + let result; + try { + result = JSON.parse(stdout); + } catch { + throw new Error("gh issue search returned invalid JSON"); + } + if (result.incomplete_results !== false || !Array.isArray(result.items)) { + throw new Error("GitHub issue search returned an invalid response"); + } + return result.items.map((item) => { + if (item.repository_url !== "https://api.github.com/repos/dotnet/runtime" || + item.pull_request !== undefined || + !Number.isInteger(item.number) || + typeof item.user?.login !== "string" || + item.user.login.length === 0) { + throw new Error("GitHub issue search returned an invalid candidate"); + } + return { + number: item.number, + user: { login: item.user.login } + }; + }); + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + checkout: fetch-depth: 50 @@ -64,6 +124,9 @@ network: - dev.azure.com - helix.dot.net - "*.blob.core.windows.net" + blocked: + - github + - api.github.com --- # CI Outer-Loop Failure Scanner @@ -90,6 +153,7 @@ The agent runs read-only. All writes go through `safe-outputs`. 10. **All intermediate state under `/tmp/gh-aw/agent/`.** Each bash invocation is a fresh subshell; persist anything you want to keep. 11. **AzDO API: anonymous only.** Stay on `_apis/build/...`. Never call `_apis/test/...` or `vstmr.dev.azure.com` (both redirect to sign-in). 12. **Don't add `area-*` references to issue titles.** Multi-area titles produce multi-label assignments from the labeler bot. +13. **Issue search transport is fixed.** Use only `search-kbe-issues` for GitHub issue searches, then inspect every returned candidate with `github` `issue_read` method `get`. Never use `github` `search_issues` or shell `gh` issue search. ## What this run must accomplish @@ -292,6 +356,16 @@ Follow exactly these sections from `.github/workflows/shared/create-kbe.instruct 4. `` / `## Search for existing PRs already handling the failure` 5. `` / `## Verify every embedded issue number exists` +All issue-search query variants in those sections MUST go through +`search-kbe-issues`. The wrapper adds `repo:dotnet/runtime is:issue` and returns +at most 10 inert candidate records containing only the issue number and +`user.login`. For every returned number, call `github` `issue_read` with +`owner: dotnet`, `repo: runtime`, and `method: get` before making any semantic +duplicate or tracker decision. A wrapper failure or integrity-filtered/failed +candidate read must fail closed as specified in the shared instructions. +Use the built-in `search_pull_requests` and `pull_request_read` tools only for +the PR searches in that flow. + When searching, account for the fact that the same signature can be filed in different `ErrorMessage` representations. A KBE recorded in `` multi-line array form will not surface from a single-substring search, and vice diff --git a/.github/workflows/evals/README.md b/.github/workflows/evals/README.md index 272a504328e0af..f1e3714599914d 100644 --- a/.github/workflows/evals/README.md +++ b/.github/workflows/evals/README.md @@ -26,10 +26,16 @@ for conformance, behavior, and constructiveness. Every grader must pass. The workflow preserves the eval specs and installs Vally from the trusted base branch before it checks out the PR head. This lets it evaluate PR changes to the workflow prompts without allowing the PR to weaken its graders or toolchain. -Each eval attaches a read-only GitHub MCP server with the `pull_requests`, -`repos`, `issues`, and `search` toolsets. The `GITHUB_TOKEN` that the eval job -supplies to that server has only the job's read permissions, allowing the -scanner to use the `github` MCP server's `search_issues` tool. +Each eval attaches a read-only GitHub MCP server with the toolsets its scenario +needs. The `GITHUB_TOKEN` that the eval job supplies to that server has only the +job's read permissions. The scanner eval omits the built-in `search` toolset and +invokes a CLI harness for the workflow's `search-kbe-issues` MCP-script tool +through Node because the eval runner does not launch workflow frontmatter MCP +servers. It uses the `github` MCP server's `issue_read` tool for candidate +inspection. A trusted static grader correlates every candidate returned by the +harness with a successful, unfiltered `issue_read`. Focused Node tests keep the +workflow-frontmatter and CLI wrapper behavior in sync and exercise the grader's +candidate correlation. These are format and behavior gates, not full ground-truth measurements. The second stage, a collector that scrapes the real failures and KBEs that actually @@ -42,8 +48,9 @@ exist and scores workflow output against them, is deferred. title, exactly `Known Build Error` plus one blocking label, the three sections, collapsed authoring guidance, a single json signature, the collapsed workflow-owned positive match-count metadata, and no test-muting. They also check - `tool-calls` evidence that it actually fetched a real build and searched existing - KBEs. + `tool-calls` evidence that it actually fetched a real build, searched existing + KBEs through the wrapper, and inspected returned candidates through + `issue_read`. - **`ci-failure-fix`** runs the workflow's deterministic scanner-author filter in trusted eval setup before the agent starts. The agent then reads a diff --git a/.github/workflows/evals/ci-failure-scan.eval.yaml b/.github/workflows/evals/ci-failure-scan.eval.yaml index 8c10f9cbada284..18adb1985dd689 100644 --- a/.github/workflows/evals/ci-failure-scan.eval.yaml +++ b/.github/workflows/evals/ci-failure-scan.eval.yaml @@ -24,7 +24,7 @@ environment: - --env - GITHUB_READ_ONLY=1 - --env - - GITHUB_TOOLSETS=pull_requests,repos,issues,search + - GITHUB_TOOLSETS=pull_requests,repos,issues - ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 stimuli: @@ -35,6 +35,8 @@ stimuli: dest: .github/workflows/ci-failure-scan.md - src: ../shared/create-kbe.instructions.md dest: .github/workflows/shared/create-kbe.instructions.md + - src: search-kbe-issues.cjs + dest: .github/workflows/evals/search-kbe-issues.cjs constraints: max_tokens: 500000 max_turns: 100 @@ -48,7 +50,7 @@ stimuli: 1. Find a recent failed build on `main` of an outer-loop pipeline via the dnceng-public AzDO REST API. It is anonymous and needs no auth, so stay on `_apis/build/...`. Start with `runtime-extra-platforms`, which is definition 154, and fall back to other outer-loop defs if needed. `curl -fsSL "https://dev.azure.com/dnceng-public/public/_apis/build/builds?definitions=154&branchName=refs/heads/main&statusFilter=completed&resultFilter=failed&\$top=5&api-version=7.1"` 2. Open that build's timeline at `/_apis/build/builds/{id}/timeline?api-version=7.1`, pick a failed leg, and read its log at `/_apis/build/builds/{id}/logs/{logId}?api-version=7.1` to extract a concrete error or assertion signature. Save the leg log you extract from as `failure.log` under /tmp and grep it for your signature to get the hit count. - 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. Run `gh search issues 'in:title [ci-scan]' --repo dotnet/runtime --state open` and also search by your signature text. Assume you would file only if none matches. + 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. The eval runner does not launch workflow-frontmatter MCP servers, so invoke its CLI harness for the workflow's `search-kbe-issues` wrapper with `node .github/workflows/evals/search-kbe-issues.cjs ''` for `in:title "[ci-scan]"` and for your signature text. For every candidate returned by either query, call the GitHub `issue_read` tool with `owner: dotnet`, `repo: runtime`, and `method: get` before deciding whether it matches. Do not use any other shell issue search or the built-in `search_issues` tool. Assume you would file only if every readable candidate is unrelated; fail closed if a candidate cannot be read. Then **emit the create-issue safe-output you would file, as a single markdown document at `./out/kbe.md`**, in exactly this shape. @@ -62,6 +64,8 @@ stimuli: rubric: - | The emitted `./out/kbe.md` conforms to the static Known Build Error format. It has a `[ci-scan] ` title naming exactly one failure shape with no run-specific IDs, the `Known Build Error` label plus exactly one of `blocking-clean-ci` or `blocking-clean-ci-optional`, the three body sections, a collapsed `KBE authoring guidance (ci-failure-scan)` block, and exactly one fenced json block with all four keys where exactly one of ErrorMessage or ErrorPattern is a non-empty specific substring rather than a bare identifier and the other is `""`. The Build Information section cites a concrete real build id. After the JSON, it has a collapsed `Agentic workflow metadata (ci-failure-scan)` block identifying the `kbe-verification` artifact and a positive verified match count. + - | + The trajectory invokes only the `search-kbe-issues` CLI harness for issue searches, parses every candidate returned by both required queries, and successfully calls `issue_read` for every returned issue number before deciding that no candidate matches. If any candidate cannot be read, the run fails closed instead of emitting `./out/kbe.md`. graders: - type: file-matches @@ -111,7 +115,15 @@ stimuli: config: required: - name: '^(bash|powershell)$' - command: 'gh (search|issue)' + command: '\.github/workflows/evals/search-kbe-issues\.cjs' + min_count: 2 + disallowed: + - name: '^(bash|powershell)$' + command: '(gh (search|issue)|gh\s+api\s+search/issues|api\.github\.com/search/issues)' + - name: 'search_issues' + + - type: kbe-candidate-reads + name: read-every-search-candidate - type: prompt name: kbe-format-conformant diff --git a/.github/workflows/evals/kbe-candidate-reads-grader.mjs b/.github/workflows/evals/kbe-candidate-reads-grader.mjs new file mode 100644 index 00000000000000..f1173ce4719244 --- /dev/null +++ b/.github/workflows/evals/kbe-candidate-reads-grader.mjs @@ -0,0 +1,169 @@ +function callKey(event) { + return JSON.stringify([event.agentId ?? null, event.data.toolCallId]); +} + +function resultText(result) { + if (typeof result === "string") { + return result; + } + + if (typeof result?.content === "string") { + return result.content; + } + + if (result === null || result === undefined) { + return ""; + } + + try { + return JSON.stringify(result); + } catch { + return String(result); + } +} + +function isSearchHarnessCall(event) { + if (event.type !== "tool_call" || !/^(bash|powershell)$/.test(event.data.toolName)) { + return false; + } + + const command = event.data.arguments?.command; + return typeof command === "string" && + /(?:^|[\\/])search-kbe-issues\.cjs(?:\s|$)/.test(command); +} + +function isIssueReadCall(event) { + const toolName = event.data?.toolName; + return event.type === "tool_call" && + typeof toolName === "string" && + (toolName === "issue_read" || /^(?:mcp__)?github[-_.]+issue_read$/.test(toolName)); +} + +function parseCandidates(result) { + const text = resultText(result).trim(); + const payloads = [text, ...text.split(/\r?\n/).reverse()]; + let parsed; + for (const payload of payloads) { + try { + const value = JSON.parse(payload); + if (Array.isArray(value)) { + parsed = value; + break; + } + } catch { + // Shell tools may surround stdout with execution metadata. + } + } + + if (!parsed) { + throw new Error("search-kbe-issues returned output that is not JSON"); + } + + for (const candidate of parsed) { + if (!Number.isInteger(candidate?.number) || + typeof candidate?.user?.login !== "string" || + candidate.user.login.length === 0) { + throw new Error("search-kbe-issues returned a malformed candidate"); + } + } + + return parsed.map((candidate) => candidate.number); +} + +class KbeCandidateReadsGrader { + metadata = { + name: "kbe-candidate-reads", + description: "Verifies every issue-search candidate receives a successful, unfiltered issue_read", + behavior: {}, + determinism: "complex-static", + reference: "reference-free", + temporalScope: "trajectory-level", + costProfile: "free", + }; + + async grade(input) { + if (!input.trajectory) { + throw new Error("Missing trajectory"); + } + + const calls = new Map(); + const candidates = new Map(); + const reads = new Map(); + const errors = []; + let harnessCallCount = 0; + + for (const [index, event] of input.trajectory.events.entries()) { + if (event.type === "tool_call") { + if (isSearchHarnessCall(event)) { + harnessCallCount++; + calls.set(callKey(event), { kind: "search" }); + } else if (isIssueReadCall(event)) { + const args = event.data.arguments ?? {}; + calls.set(callKey(event), { + kind: "read", + number: Number(args.issue_number), + validScope: args.owner === "dotnet" && + args.repo === "runtime" && + args.method === "get", + }); + } + } else if (event.type === "tool_result") { + const call = calls.get(callKey(event)); + if (!call) { + continue; + } + + if (call.kind === "search") { + if (!event.data.success) { + errors.push("search-kbe-issues call failed"); + continue; + } + + try { + for (const number of parseCandidates(event.data.result)) { + if (!candidates.has(number)) { + candidates.set(number, index); + } + } + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } else if (call.validScope && Number.isInteger(call.number) && + event.data.success && !resultText(event.data.result).includes("[Filtered]")) { + reads.set(call.number, index); + } + } + } + + if (harnessCallCount === 0) { + errors.push("search-kbe-issues was not called"); + } + + const missing = [...candidates] + .filter(([number, searchIndex]) => !reads.has(number) || reads.get(number) < searchIndex) + .map(([number]) => number) + .sort((left, right) => left - right); + const passed = errors.length === 0 && missing.length === 0; + const evidence = passed + ? `Read all ${candidates.size} candidate issue(s) returned by ${harnessCallCount} search call(s).` + : [...errors, missing.length > 0 ? `Missing successful issue_read for: ${missing.join(", ")}` : ""] + .filter(Boolean) + .join("\n"); + + return { + name: this.metadata.name, + kind: "code", + passed, + score: passed ? 1 : 0, + evidence, + metadata: { + candidates: [...candidates.keys()].sort((left, right) => left - right), + missing, + }, + }; + } +} + +export function registerGraders(registry) { + registry.register(new KbeCandidateReadsGrader()); +} diff --git a/.github/workflows/evals/kbe-search.test.mjs b/.github/workflows/evals/kbe-search.test.mjs new file mode 100644 index 00000000000000..f1dfe14db33b3d --- /dev/null +++ b/.github/workflows/evals/kbe-search.test.mjs @@ -0,0 +1,290 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { registerGraders } from "./kbe-candidate-reads-grader.mjs"; + +const require = createRequire(import.meta.url); +const { runGhApi, searchKbeIssues } = require("./search-kbe-issues.cjs"); +const testToken = "test-token"; +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + +async function productionScript() { + const workflow = await readFile(new URL("../ci-failure-scan.md", import.meta.url), "utf8"); + const scriptMatch = workflow.match( + /^ search-kbe-issues:\r?\n[\s\S]*?^ script: \|\r?\n(?