From daa7fc466b9a1d792393cb18b8af407a1ca6c55c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 29 Sep 2026 05:27:46 -0600 Subject: [PATCH] chore(release): remove temporary model probe --- .github/workflows/release-model-probe.yml | 86 ---------- scripts/release/model-probe.mjs | 182 ---------------------- 2 files changed, 268 deletions(-) delete mode 100644 .github/workflows/release-model-probe.yml delete mode 100644 scripts/release/model-probe.mjs diff --git a/.github/workflows/release-model-probe.yml b/.github/workflows/release-model-probe.yml deleted file mode 100644 index 1d85c946..00000000 --- a/.github/workflows/release-model-probe.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Release Model Probe - -on: - workflow_dispatch: - inputs: - commit: - description: Exact main commit of the candidate - required: true - type: string - candidate_run_id: - description: Successful candidate Release workflow run - required: true - type: string - mode: - description: Raw Router diagnostic or installed Braid task - required: true - default: braid - type: choice - options: - - raw - - braid - -concurrency: - group: braid-protected-account-proof - cancel-in-progress: false - -jobs: - probe: - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: release-live - permissions: - actions: read - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check out the exact candidate source - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.commit }} - path: candidate-source - fetch-depth: 0 - - name: Verify candidate source - env: - REQUESTED_COMMIT: ${{ inputs.commit }} - run: | - set -euo pipefail - test "$(git -C candidate-source rev-parse HEAD)" = "$REQUESTED_COMMIT" - test "$(printf '%s' "$REQUESTED_COMMIT" | wc -c)" = 40 - git -C candidate-source fetch origin main - git -C candidate-source merge-base --is-ancestor "$REQUESTED_COMMIT" origin/main - - name: Verify the candidate workflow run - env: - GH_TOKEN: ${{ github.token }} - BRAID_RELEASE_RUN_ID: ${{ inputs.candidate_run_id }} - BRAID_RELEASE_EXPECT_WORKFLOW: .github/workflows/release.yml - BRAID_RELEASE_EXPECT_COMMIT: ${{ inputs.commit }} - BRAID_RELEASE_EXPECT_BRANCH: main - run: node scripts/release/verify-workflow-run.mjs - - name: Download exact candidate - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: braid-release-candidate-${{ inputs.commit }} - path: ${{ runner.temp }}/braid-model-probe - github-token: ${{ github.token }} - run-id: ${{ inputs.candidate_run_id }} - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22.19.0 - cache: pnpm - - name: Install probe source dependencies - run: pnpm install --frozen-lockfile - - name: Install and build exact candidate source - run: | - cd candidate-source - pnpm install --frozen-lockfile - pnpm run build - - name: Run one bounded Gemini probe - env: - BRAID_LIVE_TANGLE_ENV_JSON: ${{ secrets.BRAID_LIVE_TANGLE_ENV_JSON }} - BRAID_PROBE_SOURCE: ${{ github.workspace }}/candidate-source - BRAID_PROBE_ARTIFACT_ROOT: ${{ runner.temp }}/braid-model-probe - BRAID_PROBE_COMMIT: ${{ inputs.commit }} - BRAID_PROBE_MODE: ${{ inputs.mode }} - run: node scripts/release/model-probe.mjs diff --git a/scripts/release/model-probe.mjs b/scripts/release/model-probe.mjs deleted file mode 100644 index a6edef81..00000000 --- a/scripts/release/model-probe.mjs +++ /dev/null @@ -1,182 +0,0 @@ -import { randomUUID } from 'node:crypto' -import { readFile, writeFile } from 'node:fs/promises' -import { join, resolve } from 'node:path' - -import { installPackedBraid } from '../packed-binary.mjs' -import { readCandidateIdentity } from './build-identity.mjs' -import { connectionConfiguration } from '../live-required/configuration.mjs' -import { safeMessage } from '../live-required/contracts.mjs' -import { - closeSession, - prepareProductionWorkspace, - runHeadlessTurn, -} from '../live-required/headless.mjs' - -const model = 'gemini-2.5-flash-lite' -const marker = 'BRAID_GEMINI_PROBE_OK' -const mode = process.env.BRAID_PROBE_MODE ?? 'braid' -if (mode !== 'raw' && mode !== 'braid') throw new Error('Unknown probe mode') -const source = resolve(process.env.BRAID_PROBE_SOURCE ?? '') -const artifactRoot = resolve(process.env.BRAID_PROBE_ARTIFACT_ROOT ?? '') -const commit = process.env.BRAID_PROBE_COMMIT -const rawSecret = process.env.BRAID_LIVE_TANGLE_ENV_JSON -if (!/^[a-f0-9]{40}$/u.test(commit ?? '') || !rawSecret) { - throw new Error('Exact candidate commit and protected environment are required') -} -const protectedEnvironment = JSON.parse(rawSecret) -if ( - protectedEnvironment === null || - typeof protectedEnvironment !== 'object' || - Array.isArray(protectedEnvironment) -) { - throw new Error('Protected environment is invalid') -} -const environment = { ...process.env, ...protectedEnvironment } -delete environment.BRAID_LIVE_TANGLE_ENV_JSON -const values = connectionConfiguration(environment, { - prefix: 'BRAID_TANGLE', - kind: 'tangle-inference', - endpointNames: ['BRAID_TANGLE_INFERENCE_ENDPOINT'], - modelNames: ['BRAID_TANGLE_INFERENCE_MODEL'], - runnerNames: ['BRAID_TANGLE_INFERENCE_RUNNER'], - providerNames: ['BRAID_TANGLE_INFERENCE_PROVIDER'], - fallbackModelProvider: 'tangle-router', - fallbackRunner: 'cli-base', -}) -if (values.credentialValue === undefined) { - throw new Error('Probe requires the existing protected credential value') -} -if (values.modelProvider !== 'tangle-router') { - throw new Error('Protected credential is not configured for Tangle Router') -} -const catalogUrl = new URL('/v1/models', values.endpoint) -const auth = values.credentialValue.replace(/^Bearer\s+/iu, '') -const catalogResponse = await fetch(catalogUrl, { - headers: { Authorization: `Bearer ${auth}` }, - signal: AbortSignal.timeout(10_000), -}) -if (!catalogResponse.ok) { - throw new Error(`Model catalog returned HTTP ${catalogResponse.status}`) -} -const catalog = await catalogResponse.json() -if (!Array.isArray(catalog.data) || !catalog.data.some((item) => item?.id === model)) { - throw new Error('Model catalog does not list the selected model') -} -const { identity } = await readCandidateIdentity({ - repository: source, - artifactRoot, - expectedCommit: commit, - expectedVersion: '0.3.4', -}) -if (mode === 'raw') { - const response = await fetch(new URL('/v1/chat/completions', values.endpoint), { - method: 'POST', - headers: { - Authorization: `Bearer ${auth}`, - 'Content-Type': 'application/json', - 'Idempotency-Key': `braid-model-probe-${randomUUID()}`, - 'X-Tangle-Client': 'braid-release-probe/0.3.4', - }, - body: JSON.stringify({ - model, - messages: [ - { - role: 'system', - content: 'Follow the release prompt exactly and return the requested marker.', - }, - { role: 'user', content: `Reply with exactly ${marker}.` }, - ], - max_tokens: 32, - max_completion_tokens: 64, - reasoning_effort: 'none', - }), - signal: AbortSignal.timeout(30_000), - }) - const body = await response.json().catch(() => null) - const costHeader = Number(response.headers.get('x-tangle-cost-usd')) - const costUsd = - Number.isFinite(costHeader) && costHeader >= 0 && response.headers.has('x-tangle-cost-usd') - ? costHeader - : null - const diagnostic = { - schema: 'braid.release-model-raw-probe.v1', - commit, - tarballSha256: identity.tarballSha256, - requestedModel: model, - httpStatus: response.status, - generationId: response.headers.get('x-generation-id'), - servedModel: response.headers.get('x-tangle-served-model') ?? body?.model ?? null, - costUsd, - promptTokens: body?.usage?.prompt_tokens ?? null, - completionTokens: body?.usage?.completion_tokens ?? null, - errorCode: body?.error?.code ?? null, - errorMessage: response.ok - ? null - : safeMessage(body?.error?.message ?? 'unknown Router error', environment), - maxVisibleOutputTokens: 32, - maxTotalOutputTokens: 64, - } - process.stdout.write(`${JSON.stringify(diagnostic)}\n`) - if (!response.ok || (costUsd !== null && costUsd > 0.01)) process.exitCode = 1 -} else { - const packed = await installPackedBraid(source, { - tarballPath: join(artifactRoot, identity.tarballPath), - }) - let config - let session - try { - if (packed.tarballSha256 !== identity.tarballSha256) { - throw new Error('Installed archive differs from the candidate') - } - config = await prepareProductionWorkspace({ - repository: source, - environment, - ...values, - model, - }) - const profilePath = join( - config.workspace, - '.braid', - 'profiles', - 'profile-tangle-inference.json', - ) - const profile = JSON.parse(await readFile(profilePath, 'utf8')) - profile.model.maxVisibleOutputTokens = 32 - profile.model.maxTotalOutputTokens = 64 - await writeFile(profilePath, `${JSON.stringify(profile, null, 2)}\n`, { mode: 0o600 }) - process.env.BRAID_LIVE_REQUIRED_TIMEOUT_MS = '60000' - const turn = await runHeadlessTurn({ - binary: packed.binary, - config, - marker, - prompt: `Reply with exactly ${marker}.`, - }) - session = turn.session - const receipt = { - schema: 'braid.release-model-probe.v1', - commit, - version: identity.braidVersion, - tarballSha256: packed.tarballSha256, - model: turn.run.model ?? model, - status: turn.run.status, - runId: turn.run.id, - materializationDigest: turn.run.materializationDigest, - costStatus: turn.run.costStatus ?? 'unknown', - costUsd: turn.run.costUsd ?? null, - maxVisibleOutputTokens: 32, - maxTotalOutputTokens: 64, - markerMatched: true, - } - process.stdout.write(`${JSON.stringify(receipt)}\n`) - if (typeof receipt.costUsd === 'number' && receipt.costUsd > 0.01) { - throw new Error('One-turn cost exceeded the $0.01 probe ceiling') - } - } catch (error) { - process.stderr.write(`Protected model probe failed: ${safeMessage(error, environment)}\n`) - process.exitCode = 1 - } finally { - if (session) await closeSession(session) - if (config) await config.cleanup() - await packed.cleanup() - } -}