diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b3e882..8392d3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: cache: npm - name: Install dependencies - run: npm ci + run: npm ci --no-audit --no-fund - name: Verify Node compatibility run: npm run check:node-compatibility @@ -50,6 +50,9 @@ jobs: with: node-version: "22.x" cache: npm + cache-dependency-path: | + package-lock.json + examples/ai-failure-lab/package-lock.json - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 @@ -62,10 +65,18 @@ jobs: deno-version: "2.2.7" - name: Install dependencies - run: npm ci + run: npm ci --no-audit --no-fund + + - name: Install AI Failure Lab dependencies + run: npm ci --no-audit --no-fund + working-directory: examples/ai-failure-lab - name: Verify package run: npm run verify + - name: Verify AI Failure Lab decision parity + run: npm test + working-directory: examples/ai-failure-lab + - name: Verify coverage run: npm run test:coverage diff --git a/.github/workflows/deploy-use-cases-site.yml b/.github/workflows/deploy-use-cases-site.yml index 33dc2f8..644f2cf 100644 --- a/.github/workflows/deploy-use-cases-site.yml +++ b/.github/workflows/deploy-use-cases-site.yml @@ -10,6 +10,7 @@ on: paths: - ".github/workflows/deploy-use-cases-site.yml" - "apps/use-cases-site/**" + - "examples/ai-failure-lab/**" - "package.json" - "package-lock.json" - "packages/core/**" @@ -19,6 +20,7 @@ on: paths: - ".github/workflows/deploy-use-cases-site.yml" - "apps/use-cases-site/**" + - "examples/ai-failure-lab/**" - "package.json" - "package-lock.json" - "packages/core/**" @@ -46,14 +48,27 @@ jobs: cache-dependency-path: | package-lock.json apps/use-cases-site/package-lock.json + examples/ai-failure-lab/package-lock.json - name: Install root dependencies - run: npm ci + run: npm ci --no-audit --no-fund - name: Install site dependencies - run: npm ci + run: npm ci --no-audit --no-fund working-directory: apps/use-cases-site + - name: Install AI Failure Lab dependencies + run: npm ci --no-audit --no-fund + working-directory: examples/ai-failure-lab + + - name: Test site contracts + run: npm test + working-directory: apps/use-cases-site + + - name: Test AI Failure Lab decision parity + run: npm test + working-directory: examples/ai-failure-lab + - name: Build site run: npm run build working-directory: apps/use-cases-site diff --git a/apps/use-cases-site/package.json b/apps/use-cases-site/package.json index 1bc94f0..101a8c6 100644 --- a/apps/use-cases-site/package.json +++ b/apps/use-cases-site/package.json @@ -11,7 +11,7 @@ "runtime": "node server/runtime-server.mjs", "generate:evidence": "node scripts/generate-evidence.mjs", "smoke:runtime": "node scripts/smoke-runtime.mjs", - "test:data": "npm run generate:evidence && node scripts/test-data-contract.mjs", + "test:data": "npm run generate:evidence && node scripts/test-data-contract.mjs && node --test scripts/test-public-data-adapters.mjs scripts/test-use-case-routing.mjs", "test": "npm run test:data && npm run smoke:runtime", "typecheck": "tsc --noEmit -p tsconfig.json", "build": "npm run generate:evidence && npm run typecheck && vite build", diff --git a/apps/use-cases-site/scripts/generate-evidence.mjs b/apps/use-cases-site/scripts/generate-evidence.mjs index 2e21334..5ba7981 100644 --- a/apps/use-cases-site/scripts/generate-evidence.mjs +++ b/apps/use-cases-site/scripts/generate-evidence.mjs @@ -19,6 +19,7 @@ const samples = [ { id: "agent-tree-cancel", path: "packages/core/samples/agent-tree-cancel.sample.js" }, { id: "conversation-agent", path: "packages/core/samples/conversation-agent.sample.js" }, { id: "race-providers", path: "packages/core/samples/race-providers.sample.js" }, + { id: "incident-decision-gate", path: "packages/core/samples/incident-decision-gate.sample.js" }, { id: "budget-rag", path: "packages/core/samples/budget-rag.sample.js" }, ]; diff --git a/apps/use-cases-site/scripts/smoke-runtime.mjs b/apps/use-cases-site/scripts/smoke-runtime.mjs index cb152d8..c3b7ae1 100644 --- a/apps/use-cases-site/scripts/smoke-runtime.mjs +++ b/apps/use-cases-site/scripts/smoke-runtime.mjs @@ -37,6 +37,7 @@ try { await waitForHealth(); await assertVibeCodingRun(); await assertConversationRun(); + await assertIncidentDecisionGateRun(); await assertRagRun(); await assertUnknownExample(); process.stdout.write("site-runtime-smoke: passed\n"); @@ -128,6 +129,20 @@ async function assertRagRun() { assertLine(result.receipt, "audit.sources: 2"); } +async function assertIncidentDecisionGateRun() { + const result = await getJson("/api/examples/incident-decision-gate/run"); + + assert.equal(result.source, "live-node"); + assert.equal(result.sample, "incident-decision-gate"); + assertLine(result.events, "quality_rejected -> retry_same_candidate -> accepted"); + assertLine(result.events, "approval: requires_user_input"); + assertLine(result.receipt, "selectedCandidate: grounded-reasoner"); + assertLine(result.receipt, "retryBudget: 1/1"); + assertLine(result.receipt, "productionChangesExecuted: 0"); + assertLine(result.receipt, "credentialsRedacted: true"); + assertNoLines(result.receipt, ["secret-for-"]); +} + async function assertUnknownExample() { const response = await fetch(`${origin}/api/examples/missing/run`); const body = await response.json(); diff --git a/apps/use-cases-site/scripts/test-data-contract.mjs b/apps/use-cases-site/scripts/test-data-contract.mjs index ca402a0..b20e1d2 100644 --- a/apps/use-cases-site/scripts/test-data-contract.mjs +++ b/apps/use-cases-site/scripts/test-data-contract.mjs @@ -60,6 +60,25 @@ const exampleContracts = [ ], liveEvents: ["task:cancelled", "race_lost"], }, + { + id: "incident-decision-gate", + sampleId: "incident-decision-gate", + samplePath: "packages/core/samples/incident-decision-gate.sample.js", + liveReceipt: [ + "runtime: @workit/core", + "sample: incident-decision-gate", + "selectedCandidate: grounded-reasoner", + "retryBudget: 1/1", + "approval.reasonCode: production_change_requires_approval", + "productionChangesExecuted: 0", + "credentialsRedacted: true", + ], + liveEvents: [ + "quality_rejected -> retry_same_candidate -> accepted", + "approval: requires_user_input", + "productionChangesExecuted: 0", + ], + }, { id: "rag-pipeline", sampleId: "budget-rag", @@ -88,11 +107,6 @@ const deniedDisplayedStrings = [ "killer", ]; -await assertUseCasesMatchExecutableSamples(); -await assertLiveRunnersMatchUseCaseContracts(); -await assertRuntimeApiUsesStaticFallbackOnPublicPages(); -process.stdout.write("site-data-contract: passed\n"); - async function assertUseCasesMatchExecutableSamples() { const snapshots = readJson(snapshotsPath); const { useCases } = await importUseCases(); @@ -187,29 +201,37 @@ function assertUseCaseLinesMatchSnapshot(useCase, result) { assertLine(rendered, `sample: ${result.sample}`, `${useCase.id} rendered lines`); - switch (result.sample) { - case "agent-tree-cancel": - assertLine(rendered, `reason.tag: ${result.reason.tag}`, `${useCase.id} rendered lines`); - assertLine(rendered, `cleanups.count: ${result.cleanups.length}`, `${useCase.id} rendered lines`); - break; - case "conversation-agent": - assertLine(rendered, `tokens: ${result.tokens.length}`, `${useCase.id} rendered lines`); - assertLine(rendered, `toolResults: ${result.toolResults.join(", ")}`, `${useCase.id} rendered lines`); - assertLine(rendered, `memoryWrites: ${result.memoryWrites}`, `${useCase.id} rendered lines`); - break; - case "race-providers": - assertLine(rendered, `winner: ${result.winner}`, `${useCase.id} rendered lines`); - assertLine(rendered, `cancelledProviders.count: ${result.cancelledProviders.length}`, `${useCase.id} rendered lines`); - break; - case "budget-rag": - assertLine(rendered, `spent: ${result.spent}`, `${useCase.id} rendered lines`); - assertLine(rendered, `audit.sources: ${result.audits[0].sources}`, `${useCase.id} rendered lines`); - break; - default: - assert.fail(`Unhandled sample result ${result.sample}.`); - } + const assertions = SAMPLE_ASSERTIONS[result.sample]; + assert.equal(typeof assertions, "function", `Unhandled sample result ${result.sample}.`); + assertions(rendered, result, useCase.id); } +const SAMPLE_ASSERTIONS = Object.freeze({ + "agent-tree-cancel": (rendered, result, id) => { + assertLine(rendered, `reason.tag: ${result.reason.tag}`, `${id} rendered lines`); + assertLine(rendered, `cleanups.count: ${result.cleanups.length}`, `${id} rendered lines`); + }, + "conversation-agent": (rendered, result, id) => { + assertLine(rendered, `tokens: ${result.tokens.length}`, `${id} rendered lines`); + assertLine(rendered, `toolResults: ${result.toolResults.join(", ")}`, `${id} rendered lines`); + assertLine(rendered, `memoryWrites: ${result.memoryWrites}`, `${id} rendered lines`); + }, + "race-providers": (rendered, result, id) => { + assertLine(rendered, `winner: ${result.winner}`, `${id} rendered lines`); + assertLine(rendered, `cancelledProviders.count: ${result.cancelledProviders.length}`, `${id} rendered lines`); + }, + "incident-decision-gate": (rendered, result, id) => { + assertLine(rendered, `selectedCandidate: ${result.selection.selectedCandidate}`, `${id} rendered lines`); + assertLine(rendered, `retryBudget: ${result.selection.retryBudget.spent}/${result.selection.retryBudget.limit}`, `${id} rendered lines`); + assertLine(rendered, `approval.reasonCode: ${result.approval.reasonCode}`, `${id} rendered lines`); + assertLine(rendered, `productionChangesExecuted: ${result.approval.productionChangesExecuted}`, `${id} rendered lines`); + }, + "budget-rag": (rendered, result, id) => { + assertLine(rendered, `spent: ${result.spent}`, `${id} rendered lines`); + assertLine(rendered, `audit.sources: ${result.audits[0].sources}`, `${id} rendered lines`); + }, +}); + function assertEvidencePathsExist(useCase) { for (const item of useCase.evidence) { assert.equal(item.status, "tracked"); @@ -281,3 +303,8 @@ async function importBundledTypeScript(relativePath, outputName) { function readJson(path) { return JSON.parse(readFileSync(path, "utf8")); } + +await assertUseCasesMatchExecutableSamples(); +await assertLiveRunnersMatchUseCaseContracts(); +await assertRuntimeApiUsesStaticFallbackOnPublicPages(); +process.stdout.write("site-data-contract: passed\n"); diff --git a/apps/use-cases-site/scripts/test-public-data-adapters.mjs b/apps/use-cases-site/scripts/test-public-data-adapters.mjs new file mode 100644 index 0000000..e4aafcb --- /dev/null +++ b/apps/use-cases-site/scripts/test-public-data-adapters.mjs @@ -0,0 +1,89 @@ +/** + * Security and contract tests for allowlisted public-data adapters. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; +import { build } from "esbuild"; + +const siteRoot = resolve(import.meta.dirname, ".."); +const modules = await importAdapters(); + +test("GitHub importer uses only the curated API origin and no authorization header", async () => { + let request; + const scenario = await modules.importGitHubIssuesScenario("vercel/ai", { + fetcher: async (url, init) => { + request = { url: String(url), init }; + return jsonResponse([{ number: 42, title: "Tool kept running", html_url: "https://github.com/vercel/ai/issues/42", state: "open", labels: [{ name: "bug" }] }]); + }, + }); + assert.match(request.url, /^https:\/\/api\.github\.com\/repos\/vercel\/ai\/issues\?/); + assert.deepEqual(request.init.headers, { accept: "application/json" }); + assert.equal(scenario.source.kind, "github_issues"); + assert.equal(scenario.candidates[0].id, "issue-42"); +}); + +test("GitHub importer rejects repositories outside the allowlist before fetch", async () => { + let called = false; + await assert.rejects( + modules.importGitHubIssuesScenario("attacker/repository", { fetcher: async () => { called = true; } }), + /allowlist/, + ); + assert.equal(called, false); +}); + +test("Open-Meteo importer validates coordinates and maps bounded evidence", async () => { + let requestUrl; + const scenario = await modules.importOpenMeteoScenario(-25.97, 32.59, { + fetcher: async (url) => { + requestUrl = String(url); + return jsonResponse({ current: { time: "2026-09-04T10:00", temperature_2m: 27.1, wind_speed_10m: 12.4, precipitation: 0 } }); + }, + }); + assert.match(requestUrl, /^https:\/\/api\.open-meteo\.com\/v1\/forecast\?/); + assert.equal(scenario.source.kind, "open_meteo"); + assert.equal(scenario.candidates[0].outcomes[0].evidence.length, 4); + await assert.rejects(modules.importOpenMeteoScenario(91, 0), /latitude/); +}); + +test("bounded fetch rejects oversized responses and exposes rate limits without bodies", async () => { + const oversized = "x".repeat(262_145); + await assert.rejects( + modules.fetchBoundedJson(new URL("https://example.invalid"), { fetcher: async () => jsonResponse(oversized) }), + (error) => error.code === "response_too_large" && !error.message.includes(oversized), + ); + await assert.rejects( + modules.fetchBoundedJson(new URL("https://example.invalid"), { fetcher: async () => new Response("secret body", { status: 429, headers: { "x-ratelimit-reset": "100" } }) }), + (error) => error.code === "rate_limited" && error.retryAt === 100_000 && !error.message.includes("secret body"), + ); +}); + +function jsonResponse(value) { + return new Response(JSON.stringify(value), { status: 200, headers: { "content-type": "application/json" } }); +} + +async function importAdapters() { + const temp = mkdtempSync(join(tmpdir(), "workit-public-data-")); + const output = join(temp, "adapters.mjs"); + const entry = join(temp, "entry.ts"); + const source = ` + export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/boundedFetch.ts"))}; + export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/githubIssues.ts"))}; + export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/openMeteo.ts"))}; + `; + const { writeFileSync } = await import("node:fs"); + writeFileSync(entry, source); + try { + await build({ entryPoints: [entry], bundle: true, platform: "node", format: "esm", outfile: output, logLevel: "silent" }); + return await import(pathToFileURL(output).href); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +} diff --git a/apps/use-cases-site/scripts/test-use-case-routing.mjs b/apps/use-cases-site/scripts/test-use-case-routing.mjs new file mode 100644 index 0000000..d335c8c --- /dev/null +++ b/apps/use-cases-site/scripts/test-use-case-routing.mjs @@ -0,0 +1,46 @@ +/** + * Regression tests for shareable WorkIt use-case links. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + buildUseCaseRoute, + resolveUseCaseId, + USE_CASE_SECTION_HASH, +} from "../src/navigation/useCaseRoute.mjs"; + +const USE_CASE_IDS = Object.freeze([ + "vibe-coding-agent", + "conversation-agent", + "provider-fallback", + "incident-decision-gate", + "rag-pipeline", +]); +const FALLBACK_ID = USE_CASE_IDS[0]; + +for (const useCaseId of USE_CASE_IDS) { + test(`resolves the ${useCaseId} deep link`, () => { + assert.equal(resolveUseCaseId(`?example=${useCaseId}`, USE_CASE_IDS, FALLBACK_ID), useCaseId); + }); +} + +test("rejects unknown and encoded hostile use-case ids", () => { + assert.equal(resolveUseCaseId("?example=unknown", USE_CASE_IDS, FALLBACK_ID), FALLBACK_ID); + assert.equal(resolveUseCaseId("?example=%2F%2Fevil.example", USE_CASE_IDS, FALLBACK_ID), FALLBACK_ID); +}); + +test("builds a relative same-origin route and preserves unrelated query state", () => { + const route = buildUseCaseRoute( + "https://workruntime.github.io/workit/?source=article#failure-lab", + "incident-decision-gate", + ); + + assert.equal( + route, + `/workit/?source=article&example=incident-decision-gate${USE_CASE_SECTION_HASH}`, + ); +}); diff --git a/apps/use-cases-site/server/runners.mjs b/apps/use-cases-site/server/runners.mjs index 57c503c..f94a8a3 100644 --- a/apps/use-cases-site/server/runners.mjs +++ b/apps/use-cases-site/server/runners.mjs @@ -5,17 +5,22 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { execFile } from "node:child_process"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; import { CancellationError, ContextBagImpl, CostBudget, group, run } from "@workit/core"; const repoRoot = resolve(fileURLToPath(new URL("../../../", import.meta.url))); +const executeFile = promisify(execFile); +const INCIDENT_GATE_SAMPLE_PATH = "packages/core/samples/incident-decision-gate.sample.js"; export const runners = { "vibe-coding-agent": runAgentTree, "conversation-agent": runConversationAgent, "provider-fallback": runProviderFallback, + "incident-decision-gate": runIncidentDecisionGate, "rag-pipeline": runRagPipeline, }; @@ -91,6 +96,30 @@ async function runProviderFallback() { }; } +async function runIncidentDecisionGate() { + const result = await runJsonSample(INCIDENT_GATE_SAMPLE_PATH); + const decisions = result.selection.decisions.join(" -> "); + + return { + sample: result.sample, + events: [ + `selection: ${decisions}`, + `approval: ${result.approval.status}`, + `productionChangesExecuted: ${result.approval.productionChangesExecuted}`, + ], + receipt: [ + "runtime: @workit/core", + `sample: ${result.sample}`, + `selectedCandidate: ${result.selection.selectedCandidate}`, + `retryBudget: ${result.selection.retryBudget.spent}/${result.selection.retryBudget.limit}`, + `approval.reasonCode: ${result.approval.reasonCode}`, + `productionChangesExecuted: ${result.approval.productionChangesExecuted}`, + `credentialsRedacted: ${result.selection.credentialsRedacted}`, + ], + code: await readSample(INCIDENT_GATE_SAMPLE_PATH), + }; +} + async function runRagPipeline() { const events = []; const budget = { spent: 0, limit: 10, unit: "USD" }; @@ -257,6 +286,16 @@ function readSample(path) { return readFile(resolve(repoRoot, path), "utf8"); } +async function runJsonSample(path) { + const { stdout } = await executeFile(process.execPath, [path], { + cwd: repoRoot, + env: process.env, + maxBuffer: 1024 * 1024, + windowsHide: true, + }); + return JSON.parse(stdout.trim()); +} + function sleep(ms, signal) { return new Promise((resolve, reject) => { const timer = setTimeout(resolve, ms); diff --git a/apps/use-cases-site/src/App.tsx b/apps/use-cases-site/src/App.tsx index c872d9f..e077363 100644 --- a/apps/use-cases-site/src/App.tsx +++ b/apps/use-cases-site/src/App.tsx @@ -10,12 +10,20 @@ import { SiteHeader } from "./components/SiteHeader"; import { UseCaseRail } from "./components/UseCaseRail"; import { UseCaseWorkbench } from "./components/UseCaseWorkbench"; import { defaultUseCase, useCases } from "./data/useCases"; +import { ScenarioStudio } from "./labs/ScenarioStudio"; +import { buildUseCaseRoute, resolveUseCaseId } from "./navigation/useCaseRoute.mjs"; import { runLiveExample } from "./runtimeApi"; import type { ExampleRunResult, RunPhase, UseCase } from "./types"; +const useCaseIds = Object.freeze(useCases.map(({ id }) => id)); + +function selectedUseCaseId() { + return resolveUseCaseId(window.location.search, useCaseIds, defaultUseCase.id); +} + /** Render the responsive WorkIt examples workbench. */ export default function App() { - const [selectedId, setSelectedId] = useState(defaultUseCase.id); + const [selectedId, setSelectedId] = useState(selectedUseCaseId); const [phase, setPhase] = useState("idle"); const [runStep, setRunStep] = useState(0); const [runResult, setRunResult] = useState(null); @@ -34,15 +42,21 @@ export default function App() { runTimersRef.current = []; } - function selectUseCase(useCase: UseCase) { + function resetRunState(useCaseId: string) { clearRunTimers(); runGenerationRef.current++; - setSelectedId(useCase.id); + setSelectedId(useCaseId); setPhase("idle"); setRunStep(0); setRunResult(null); } + function selectUseCase(useCase: UseCase) { + if (useCase.id === selectedId) return; + resetRunState(useCase.id); + window.history.pushState(null, "", buildUseCaseRoute(window.location.href, useCase.id)); + } + function runScenario() { clearRunTimers(); const generation = ++runGenerationRef.current; @@ -82,7 +96,17 @@ export default function App() { setRunResult(null); } - useEffect(() => clearRunTimers, []); + useEffect(() => { + function restoreUseCaseFromHistory() { + resetRunState(selectedUseCaseId()); + } + + window.addEventListener("popstate", restoreUseCaseFromHistory); + return () => { + window.removeEventListener("popstate", restoreUseCaseFromHistory); + clearRunTimers(); + }; + }, []); async function resolveLiveRun(useCase: UseCase, fallbackPhase: RunPhase, generation: number) { const live = await runLiveExample(useCase.id); @@ -97,6 +121,7 @@ export default function App() { return (
+
diff --git a/apps/use-cases-site/src/components/SiteHeader.tsx b/apps/use-cases-site/src/components/SiteHeader.tsx index 96085d2..1c7bf46 100644 --- a/apps/use-cases-site/src/components/SiteHeader.tsx +++ b/apps/use-cases-site/src/components/SiteHeader.tsx @@ -22,6 +22,7 @@ export function SiteHeader() {