Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
cache: npm

- name: Install dependencies
run: npm ci
run: npm ci --no-audit --no-fund

- name: Verify Node compatibility
run: npm run check:node-compatibility
Expand All @@ -50,6 +50,9 @@ jobs:
with:
node-version: "22.x"
cache: npm
cache-dependency-path: |
package-lock.json
examples/ai-failure-lab/package-lock.json

- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
Expand All @@ -62,10 +65,18 @@ jobs:
deno-version: "2.2.7"

- name: Install dependencies
run: npm ci
run: npm ci --no-audit --no-fund

- name: Install AI Failure Lab dependencies
run: npm ci --no-audit --no-fund
working-directory: examples/ai-failure-lab

- name: Verify package
run: npm run verify

- name: Verify AI Failure Lab decision parity
run: npm test
working-directory: examples/ai-failure-lab

- name: Verify coverage
run: npm run test:coverage
19 changes: 17 additions & 2 deletions .github/workflows/deploy-use-cases-site.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ on:
paths:
- ".github/workflows/deploy-use-cases-site.yml"
- "apps/use-cases-site/**"
- "examples/ai-failure-lab/**"
- "package.json"
- "package-lock.json"
- "packages/core/**"
Expand All @@ -19,6 +20,7 @@ on:
paths:
- ".github/workflows/deploy-use-cases-site.yml"
- "apps/use-cases-site/**"
- "examples/ai-failure-lab/**"
- "package.json"
- "package-lock.json"
- "packages/core/**"
Expand Down Expand Up @@ -46,14 +48,27 @@ jobs:
cache-dependency-path: |
package-lock.json
apps/use-cases-site/package-lock.json
examples/ai-failure-lab/package-lock.json

- name: Install root dependencies
run: npm ci
run: npm ci --no-audit --no-fund

- name: Install site dependencies
run: npm ci
run: npm ci --no-audit --no-fund
working-directory: apps/use-cases-site

- name: Install AI Failure Lab dependencies
run: npm ci --no-audit --no-fund
working-directory: examples/ai-failure-lab

- name: Test site contracts
run: npm test
working-directory: apps/use-cases-site

- name: Test AI Failure Lab decision parity
run: npm test
working-directory: examples/ai-failure-lab

- name: Build site
run: npm run build
working-directory: apps/use-cases-site
Expand Down
2 changes: 1 addition & 1 deletion apps/use-cases-site/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"runtime": "node server/runtime-server.mjs",
"generate:evidence": "node scripts/generate-evidence.mjs",
"smoke:runtime": "node scripts/smoke-runtime.mjs",
"test:data": "npm run generate:evidence && node scripts/test-data-contract.mjs",
"test:data": "npm run generate:evidence && node scripts/test-data-contract.mjs && node --test scripts/test-public-data-adapters.mjs scripts/test-use-case-routing.mjs",
"test": "npm run test:data && npm run smoke:runtime",
"typecheck": "tsc --noEmit -p tsconfig.json",
"build": "npm run generate:evidence && npm run typecheck && vite build",
Expand Down
1 change: 1 addition & 0 deletions apps/use-cases-site/scripts/generate-evidence.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const samples = [
{ id: "agent-tree-cancel", path: "packages/core/samples/agent-tree-cancel.sample.js" },
{ id: "conversation-agent", path: "packages/core/samples/conversation-agent.sample.js" },
{ id: "race-providers", path: "packages/core/samples/race-providers.sample.js" },
{ id: "incident-decision-gate", path: "packages/core/samples/incident-decision-gate.sample.js" },
{ id: "budget-rag", path: "packages/core/samples/budget-rag.sample.js" },
];

Expand Down
15 changes: 15 additions & 0 deletions apps/use-cases-site/scripts/smoke-runtime.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ try {
await waitForHealth();
await assertVibeCodingRun();
await assertConversationRun();
await assertIncidentDecisionGateRun();
await assertRagRun();
await assertUnknownExample();
process.stdout.write("site-runtime-smoke: passed\n");
Expand Down Expand Up @@ -128,6 +129,20 @@ async function assertRagRun() {
assertLine(result.receipt, "audit.sources: 2");
}

async function assertIncidentDecisionGateRun() {
const result = await getJson("/api/examples/incident-decision-gate/run");

assert.equal(result.source, "live-node");
assert.equal(result.sample, "incident-decision-gate");
assertLine(result.events, "quality_rejected -> retry_same_candidate -> accepted");
assertLine(result.events, "approval: requires_user_input");
assertLine(result.receipt, "selectedCandidate: grounded-reasoner");
assertLine(result.receipt, "retryBudget: 1/1");
assertLine(result.receipt, "productionChangesExecuted: 0");
assertLine(result.receipt, "credentialsRedacted: true");
assertNoLines(result.receipt, ["secret-for-"]);
}

async function assertUnknownExample() {
const response = await fetch(`${origin}/api/examples/missing/run`);
const body = await response.json();
Expand Down
79 changes: 53 additions & 26 deletions apps/use-cases-site/scripts/test-data-contract.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,25 @@ const exampleContracts = [
],
liveEvents: ["task:cancelled", "race_lost"],
},
{
id: "incident-decision-gate",
sampleId: "incident-decision-gate",
samplePath: "packages/core/samples/incident-decision-gate.sample.js",
liveReceipt: [
"runtime: @workit/core",
"sample: incident-decision-gate",
"selectedCandidate: grounded-reasoner",
"retryBudget: 1/1",
"approval.reasonCode: production_change_requires_approval",
"productionChangesExecuted: 0",
"credentialsRedacted: true",
],
liveEvents: [
"quality_rejected -> retry_same_candidate -> accepted",
"approval: requires_user_input",
"productionChangesExecuted: 0",
],
},
{
id: "rag-pipeline",
sampleId: "budget-rag",
Expand Down Expand Up @@ -88,11 +107,6 @@ const deniedDisplayedStrings = [
"killer",
];

await assertUseCasesMatchExecutableSamples();
await assertLiveRunnersMatchUseCaseContracts();
await assertRuntimeApiUsesStaticFallbackOnPublicPages();
process.stdout.write("site-data-contract: passed\n");

async function assertUseCasesMatchExecutableSamples() {
const snapshots = readJson(snapshotsPath);
const { useCases } = await importUseCases();
Expand Down Expand Up @@ -187,29 +201,37 @@ function assertUseCaseLinesMatchSnapshot(useCase, result) {

assertLine(rendered, `sample: ${result.sample}`, `${useCase.id} rendered lines`);

switch (result.sample) {
case "agent-tree-cancel":
assertLine(rendered, `reason.tag: ${result.reason.tag}`, `${useCase.id} rendered lines`);
assertLine(rendered, `cleanups.count: ${result.cleanups.length}`, `${useCase.id} rendered lines`);
break;
case "conversation-agent":
assertLine(rendered, `tokens: ${result.tokens.length}`, `${useCase.id} rendered lines`);
assertLine(rendered, `toolResults: ${result.toolResults.join(", ")}`, `${useCase.id} rendered lines`);
assertLine(rendered, `memoryWrites: ${result.memoryWrites}`, `${useCase.id} rendered lines`);
break;
case "race-providers":
assertLine(rendered, `winner: ${result.winner}`, `${useCase.id} rendered lines`);
assertLine(rendered, `cancelledProviders.count: ${result.cancelledProviders.length}`, `${useCase.id} rendered lines`);
break;
case "budget-rag":
assertLine(rendered, `spent: ${result.spent}`, `${useCase.id} rendered lines`);
assertLine(rendered, `audit.sources: ${result.audits[0].sources}`, `${useCase.id} rendered lines`);
break;
default:
assert.fail(`Unhandled sample result ${result.sample}.`);
}
const assertions = SAMPLE_ASSERTIONS[result.sample];
assert.equal(typeof assertions, "function", `Unhandled sample result ${result.sample}.`);
assertions(rendered, result, useCase.id);
}

const SAMPLE_ASSERTIONS = Object.freeze({
"agent-tree-cancel": (rendered, result, id) => {
assertLine(rendered, `reason.tag: ${result.reason.tag}`, `${id} rendered lines`);
assertLine(rendered, `cleanups.count: ${result.cleanups.length}`, `${id} rendered lines`);
},
"conversation-agent": (rendered, result, id) => {
assertLine(rendered, `tokens: ${result.tokens.length}`, `${id} rendered lines`);
assertLine(rendered, `toolResults: ${result.toolResults.join(", ")}`, `${id} rendered lines`);
assertLine(rendered, `memoryWrites: ${result.memoryWrites}`, `${id} rendered lines`);
},
"race-providers": (rendered, result, id) => {
assertLine(rendered, `winner: ${result.winner}`, `${id} rendered lines`);
assertLine(rendered, `cancelledProviders.count: ${result.cancelledProviders.length}`, `${id} rendered lines`);
},
"incident-decision-gate": (rendered, result, id) => {
assertLine(rendered, `selectedCandidate: ${result.selection.selectedCandidate}`, `${id} rendered lines`);
assertLine(rendered, `retryBudget: ${result.selection.retryBudget.spent}/${result.selection.retryBudget.limit}`, `${id} rendered lines`);
assertLine(rendered, `approval.reasonCode: ${result.approval.reasonCode}`, `${id} rendered lines`);
assertLine(rendered, `productionChangesExecuted: ${result.approval.productionChangesExecuted}`, `${id} rendered lines`);
},
"budget-rag": (rendered, result, id) => {
assertLine(rendered, `spent: ${result.spent}`, `${id} rendered lines`);
assertLine(rendered, `audit.sources: ${result.audits[0].sources}`, `${id} rendered lines`);
},
});

function assertEvidencePathsExist(useCase) {
for (const item of useCase.evidence) {
assert.equal(item.status, "tracked");
Expand Down Expand Up @@ -281,3 +303,8 @@ async function importBundledTypeScript(relativePath, outputName) {
function readJson(path) {
return JSON.parse(readFileSync(path, "utf8"));
}

await assertUseCasesMatchExecutableSamples();
await assertLiveRunnersMatchUseCaseContracts();
await assertRuntimeApiUsesStaticFallbackOnPublicPages();
process.stdout.write("site-data-contract: passed\n");
89 changes: 89 additions & 0 deletions apps/use-cases-site/scripts/test-public-data-adapters.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/**
* Security and contract tests for allowlisted public-data adapters.
*
* @author Admilson B. F. Cossa
* SPDX-License-Identifier: Apache-2.0
*/

import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import test from "node:test";
import { pathToFileURL } from "node:url";
import { build } from "esbuild";

const siteRoot = resolve(import.meta.dirname, "..");
const modules = await importAdapters();

test("GitHub importer uses only the curated API origin and no authorization header", async () => {
let request;
const scenario = await modules.importGitHubIssuesScenario("vercel/ai", {
fetcher: async (url, init) => {
request = { url: String(url), init };
return jsonResponse([{ number: 42, title: "Tool kept running", html_url: "https://github.com/vercel/ai/issues/42", state: "open", labels: [{ name: "bug" }] }]);
},
});
assert.match(request.url, /^https:\/\/api\.github\.com\/repos\/vercel\/ai\/issues\?/);
assert.deepEqual(request.init.headers, { accept: "application/json" });
assert.equal(scenario.source.kind, "github_issues");
assert.equal(scenario.candidates[0].id, "issue-42");
});

test("GitHub importer rejects repositories outside the allowlist before fetch", async () => {
let called = false;
await assert.rejects(
modules.importGitHubIssuesScenario("attacker/repository", { fetcher: async () => { called = true; } }),
/allowlist/,
);
assert.equal(called, false);
});

test("Open-Meteo importer validates coordinates and maps bounded evidence", async () => {
let requestUrl;
const scenario = await modules.importOpenMeteoScenario(-25.97, 32.59, {
fetcher: async (url) => {
requestUrl = String(url);
return jsonResponse({ current: { time: "2026-09-04T10:00", temperature_2m: 27.1, wind_speed_10m: 12.4, precipitation: 0 } });
},
});
assert.match(requestUrl, /^https:\/\/api\.open-meteo\.com\/v1\/forecast\?/);
assert.equal(scenario.source.kind, "open_meteo");
assert.equal(scenario.candidates[0].outcomes[0].evidence.length, 4);
await assert.rejects(modules.importOpenMeteoScenario(91, 0), /latitude/);
});

test("bounded fetch rejects oversized responses and exposes rate limits without bodies", async () => {
const oversized = "x".repeat(262_145);
await assert.rejects(
modules.fetchBoundedJson(new URL("https://example.invalid"), { fetcher: async () => jsonResponse(oversized) }),
(error) => error.code === "response_too_large" && !error.message.includes(oversized),
);
await assert.rejects(
modules.fetchBoundedJson(new URL("https://example.invalid"), { fetcher: async () => new Response("secret body", { status: 429, headers: { "x-ratelimit-reset": "100" } }) }),
(error) => error.code === "rate_limited" && error.retryAt === 100_000 && !error.message.includes("secret body"),
);
});

function jsonResponse(value) {
return new Response(JSON.stringify(value), { status: 200, headers: { "content-type": "application/json" } });
}

async function importAdapters() {
const temp = mkdtempSync(join(tmpdir(), "workit-public-data-"));
const output = join(temp, "adapters.mjs");
const entry = join(temp, "entry.ts");
const source = `
export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/boundedFetch.ts"))};
export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/githubIssues.ts"))};
export * from ${JSON.stringify(resolve(siteRoot, "src/labs/public-data/openMeteo.ts"))};
`;
const { writeFileSync } = await import("node:fs");
writeFileSync(entry, source);
try {
await build({ entryPoints: [entry], bundle: true, platform: "node", format: "esm", outfile: output, logLevel: "silent" });
return await import(pathToFileURL(output).href);
} finally {
rmSync(temp, { recursive: true, force: true });
}
}
46 changes: 46 additions & 0 deletions apps/use-cases-site/scripts/test-use-case-routing.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/**
* Regression tests for shareable WorkIt use-case links.
*
* @author Admilson B. F. Cossa
* SPDX-License-Identifier: Apache-2.0
*/

import assert from "node:assert/strict";
import test from "node:test";
import {
buildUseCaseRoute,
resolveUseCaseId,
USE_CASE_SECTION_HASH,
} from "../src/navigation/useCaseRoute.mjs";

const USE_CASE_IDS = Object.freeze([
"vibe-coding-agent",
"conversation-agent",
"provider-fallback",
"incident-decision-gate",
"rag-pipeline",
]);
const FALLBACK_ID = USE_CASE_IDS[0];

for (const useCaseId of USE_CASE_IDS) {
test(`resolves the ${useCaseId} deep link`, () => {
assert.equal(resolveUseCaseId(`?example=${useCaseId}`, USE_CASE_IDS, FALLBACK_ID), useCaseId);
});
}

test("rejects unknown and encoded hostile use-case ids", () => {
assert.equal(resolveUseCaseId("?example=unknown", USE_CASE_IDS, FALLBACK_ID), FALLBACK_ID);
assert.equal(resolveUseCaseId("?example=%2F%2Fevil.example", USE_CASE_IDS, FALLBACK_ID), FALLBACK_ID);
});

test("builds a relative same-origin route and preserves unrelated query state", () => {
const route = buildUseCaseRoute(
"https://workruntime.github.io/workit/?source=article#failure-lab",
"incident-decision-gate",
);

assert.equal(
route,
`/workit/?source=article&example=incident-decision-gate${USE_CASE_SECTION_HASH}`,
);
});
Loading