import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";
configureAgent(copilotEngine());
const DEADLINE = Date.now() + 25 * 60_000;
function msLeft() { return Math.max(0, DEADLINE - Date.now()); }
// Agent role: pick a concrete, multi-faceted task suitable for multi-agent decomposition.
const taskPicker = agent({
model: "small",
maxTurns: 3,
addons: [repair()],
instructions: `Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models. Choose from: multi-source research and synthesis, a multi-file coding task with distinct design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. The task must be self-contained (solvable from its description alone, with no external file or live web access) and concrete enough to grade. Prefer variety across domains from run to run. Return the task as structured JSON.`,
output: s.object({
title: s.string,
domain: s.string,
description: s.string,
successCriteria: s.array(s.string),
}),
});
// Agent role: solve the given task in a single pass with no delegation.
const singleSolver = agent({
model: "medium",
maxTurns: 1,
instructions: `You will be given a task description and success criteria. Solve the task completely and correctly, addressing every success criterion. Provide your answer as a detailed solution in the 'solution' field.`,
input: s.object({ title: s.string, description: s.string, successCriteria: s.array(s.string) }),
output: s.object({ solution: s.string }),
});
// Agent role: write a self-contained rig TypeScript program that decomposes the task across at least two agents.
const programWriter = agent({
model: "medium",
maxTurns: 2,
instructions: `You will be given a task description and success criteria. Write a self-contained rig TypeScript program that decomposes the task across at least two agents, each with a // Agent role: ... comment, each choosing small for simple sub-steps and medium or large for harder ones, coordinated so the final answer combines their outputs rather than coming from one agent solving everything.\n\nCRITICAL requirements for the generated program:\n- Import only from "rig"\n- Call configureAgent(copilotEngine()) at the top after imports\n- The root export must be: export default workflow({ meta: { name: "...", description: "..." }, body: async ({ call }) => { ... return { solution: combinedAnswer }; } })\n- The body must return { solution: string } combining all sub-agent outputs\n- Use at least 2 agent() calls inside the workflow body\n- Each agent must have a // Agent role: ... comment\n- The workflow must have a // Workflow role: ... comment\n- Do NOT declare output on workflow() - it has no output field\n- Do NOT invoke the root - just export default workflow(...)\n- No console.log anywhere\n\nHere is the exact skeleton to follow (expand it to at least 2 agents):\n\nimport { agent, workflow, s, configureAgent, copilotEngine } from "rig";\n\nconfigureAgent(copilotEngine());\n\n// Agent role: <role>\nconst analyze = agent({ model: "small", instructions: "...", output: s.string });\n\n// Workflow role: combines the agents' outputs into the final solution\nexport default workflow({\n meta: { name: "decomposed-solution", description: "..." },\n body: async ({ call }) => {\n const analysis = await call(analyze, "...");\n return { solution: analysis ?? "" };\n },\n});\n\nReturn only the TypeScript source code, nothing else.`,
input: s.object({ title: s.string, description: s.string, successCriteria: s.array(s.string) }),
output: s.string,
});
// Agent role: write a fixed version of the rig program given the previous source and error.
const programFixer = agent({
model: "medium",
maxTurns: 2,
instructions: `You will be given: a rig TypeScript program that failed typecheck or execution, and the exact error message. Fix precisely what the error names, preserving everything that already worked. Do not invent unrelated API edits.\n\nCRITICAL requirements:\n- Import only from "rig"\n- Call configureAgent(copilotEngine()) at the top after imports\n- export default workflow({ meta: ..., body: async ({ call }) => { ... return { solution: string }; } })\n- At least 2 agents with // Agent role: ... comments\n- workflow has // Workflow role: ... comment\n- workflow() has no output field\n- Do NOT invoke the root\n- No console.log\n\nReturn only the TypeScript source code, nothing else.`,
input: s.object({ previousSource: s.string, error: s.string, title: s.string, description: s.string, successCriteria: s.array(s.string) }),
output: s.string,
});
// Agent role: grade both solutions on correctness and completeness against success criteria.
const grader = agent({
model: "large",
maxTurns: 1,
instructions: `You will be given a task with success criteria, a single-call solution, and a decomposed solution. Grade each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge on correctness and completeness of content only — not on length, not on which approach produced the solution. Pick a winner: "single-call", "decomposed", or "tie". Explain your reasoning.`,
input: s.object({
title: s.string,
successCriteria: s.array(s.string),
singleCallSolution: s.string,
decomposedSolution: s.string,
}),
output: s.object({
singleCallScore: s.int,
decomposedScore: s.int,
winner: s.enum("single-call", "decomposed", "tie"),
rationale: s.string,
}),
});
async function runInSkillDir(
args: string[],
stdinData: string,
timeoutMs: number
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const spawnFn = (globalThis as unknown as { __rigSpawn?: unknown }).__rigSpawn ??
await (async () => {
const mod = await import(["child","process"].join("_"));
return (mod as { spawn: unknown }).spawn;
})();
return new Promise((resolve) => {
const spawn = spawnFn as (
cmd: string,
args: string[],
opts: { cwd: string; stdio: string[] }
) => {
stdout: { on: (e: string, cb: (d: Uint8Array) => void) => void };
stderr: { on: (e: string, cb: (d: Uint8Array) => void) => void };
stdin: { write: (d: string) => void; end: () => void };
kill: (sig: string) => void;
on: (e: string, cb: (code: number) => void) => void;
};
const child = spawn("node", ["rig.ts", ...args], {
cwd: "/home/runner/work/rig/rig/.github/skills/rig",
stdio: ["pipe", "pipe", "pipe"],
});
const stdoutChunks: Uint8Array[] = [];
const stderrChunks: Uint8Array[] = [];
child.stdout.on("data", (d: Uint8Array) => stdoutChunks.push(d));
child.stderr.on("data", (d: Uint8Array) => stderrChunks.push(d));
child.stdin.write(stdinData);
child.stdin.end();
const timer = setTimeout(() => { child.kill("SIGTERM"); }, timeoutMs);
child.on("close", (code: number) => {
clearTimeout(timer);
const dec = new TextDecoder();
resolve({
stdout: dec.decode(concatBytes(stdoutChunks)),
stderr: dec.decode(concatBytes(stderrChunks)),
exitCode: code ?? 1,
});
});
});
}
function concatBytes(arrays: Uint8Array[]): Uint8Array {
const total = arrays.reduce((n, a) => n + a.length, 0);
const out = new Uint8Array(total);
let offset = 0;
for (const a of arrays) { out.set(a, offset); offset += a.length; }
return out;
}
// Workflow role: orchestrate the benchmark — pick task, solve single/decomposed, grade, return report.
export default workflow({
meta: { name: "rig-decomposition-benchmark", description: "Benchmark single-call vs decomposed rig solutions" },
body: async ({ call }) => {
const task = await call(taskPicker, "Pick a task");
if (!task) throw new Error("Task picker returned null");
const singleStart = Date.now();
const singleResult = await call(singleSolver, {
title: task.title,
description: task.description,
successCriteria: task.successCriteria,
});
const singleDuration = Date.now() - singleStart;
const singleSolution = singleResult?.solution ?? "";
const decompStart = Date.now();
const attempts: Array<{
attempt: number; source: string;
typecheckPassed: boolean; executePassed: boolean;
typecheckOutput: string; executeOutput: string;
}> = [];
let decomposedSolution = "";
let finalSource = "";
let decompFinalPass = false;
let currentSource: string | null = null;
let lastError = "";
for (let attempt = 1; attempt <= 2; attempt++) {
const typecheckBudget = Math.min(2 * 60_000, msLeft());
const executeBudget = Math.min(5 * 60_000, msLeft());
if (typecheckBudget <= 5000) break;
let source: string | null = null;
if (attempt === 1) {
source = await call(programWriter, { title: task.title, description: task.description, successCriteria: task.successCriteria });
} else if (currentSource !== null) {
source = await call(programFixer, { previousSource: currentSource, error: lastError, title: task.title, description: task.description, successCriteria: task.successCriteria });
}
if (!source) {
attempts.push({ attempt, source: "", typecheckPassed: false, executePassed: false, typecheckOutput: "writer returned null", executeOutput: "" });
break;
}
currentSource = source;
const tcResult = await runInSkillDir(["--typecheck"], source, typecheckBudget);
const tcPassed = tcResult.exitCode === 0;
const tcOutput = (tcResult.stdout + tcResult.stderr).trim();
if (!tcPassed) {
const progErrors = tcOutput.split("\n").filter((l: string) => l.includes("program.ts")).join("\n");
const errText = progErrors || tcOutput.slice(0, 2000);
attempts.push({ attempt, source, typecheckPassed: false, executePassed: false, typecheckOutput: errText, executeOutput: "" });
lastError = errText;
continue;
}
if (executeBudget <= 5000) {
attempts.push({ attempt, source, typecheckPassed: true, executePassed: false, typecheckOutput: tcOutput, executeOutput: "budget exhausted" });
break;
}
const execResult = await runInSkillDir(["--server"], source, executeBudget);
const execPassed = execResult.exitCode === 0;
const execOutput = execResult.stdout.trim();
const execErr = execResult.stderr.trim();
attempts.push({ attempt, source, typecheckPassed: true, executePassed: execPassed, typecheckOutput: tcOutput, executeOutput: execPassed ? execOutput.slice(0, 500) : (execErr || execOutput).slice(0, 2000) });
if (execPassed) {
finalSource = source;
decompFinalPass = true;
try {
const parsed = JSON.parse(execOutput);
decomposedSolution = (parsed as { solution?: string })?.solution ?? execOutput;
} catch { decomposedSolution = execOutput; }
break;
} else {
lastError = (execErr || execOutput).slice(0, 2000);
}
}
const decompDuration = Date.now() - decompStart;
const gradeResult = await call(grader, {
title: task.title,
successCriteria: task.successCriteria,
singleCallSolution: singleSolution || "(no solution produced)",
decomposedSolution: decomposedSolution || "(decomposition produced no solution)",
});
return { task, singleDuration, decompDuration, singleSolution, decomposedSolution, decompFinalPass, attempts, finalDecomposedSource: finalSource || (attempts[attempts.length - 1]?.source ?? ""), grading: gradeResult };
},
});
Benchmark Run: 2026-08-25
Status: FAILED — The benchmark program ran exactly once as required, but the workflow threw an unrecoverable error at the first step.
Task
No task was picked — the workflow failed at the first step (task picker agent returned
null).Timing Comparison
Decomposed program final pass/fail: Not reached
Decomposition Attempts
No decomposition attempts were made — the workflow failed before this phase.
Grading
No grading was performed — the workflow failed before any solutions were produced.
Single-Call Solution
Single-call solution
No solution produced — workflow failed before this phase.
Decomposed Rig Program Source
No decomposed program was generated — workflow failed before this phase.
Decomposed Solution
Decomposed solution
No solution produced — workflow failed before this phase.
Captured Stderr (bench_stderr.txt)
Stdout (bench_output.json) was empty. The
taskPickeragent (model:small, withrepair()addon,maxTurns: 3) returnednull— the rig behavior when all agent call attempts fail. A parallel direct test ofnode rig.ts --serverpiping a minimal agent returnedAUTHENTICATION_FAILED, suggesting the Copilot SDK subprocess sessions were not available inside the running workflow context.Benchmark Program (Step 1)
bench.ts — the full benchmark program written in Step 1
Verdict
No winner determined — the benchmark failed at the first step when the task-picker agent (model
smallwithrepair()) returnednull, indicating all Copilot model calls within the rig workflow subprocess context failed to complete.