diff --git a/packages/core/src/autoresearch/autoresearch.test.ts b/packages/core/src/autoresearch/autoresearch.test.ts index ac791a6983..d071303ea9 100644 --- a/packages/core/src/autoresearch/autoresearch.test.ts +++ b/packages/core/src/autoresearch/autoresearch.test.ts @@ -142,6 +142,14 @@ function reportText(value: number, summary = "tweak"): string { return `Done.\n\`\`\`autoresearch\nmetric: ${value}\nsummary: ${summary}\n\`\`\``; } +function researchText( + summary = "Mapped the execution path", + finding = "The metric is computed in the workspace server.", + nextStep = "Trace the benchmark command", +): string { + return `Investigated.\n\`\`\`autoresearch\ntype: research\nsummary: ${summary}\nfinding: ${finding}\nnext: ${nextStep}\n\`\`\``; +} + /** A session `model` config option (and optional `thought_level`). */ function modelConfig( currentValue: string, @@ -202,6 +210,14 @@ function completeTurn(text: string, taskRunId = TASK_RUN_ID): void { }); } +function streamTurnText(text: string, taskRunId = TASK_RUN_ID): void { + const events = sessionStore.getState().sessions[taskRunId]?.events ?? []; + sessionStoreSetters.updateSession(taskRunId, { + isPromptPending: true, + events: [...events, agentChunkEvent(text)], + }); +} + function runTurn(text: string, taskRunId = TASK_RUN_ID): void { beginTurn(taskRunId); completeTurn(text, taskRunId); @@ -251,6 +267,7 @@ function makeRun( phase: null, originalModel: null, originalEffort: null, + researchFindings: [], iterations: [], startedAt: 1_000, endedAt: null, @@ -350,7 +367,7 @@ describe("AutoresearchService", () => { }); describe("registerRun", () => { - it("registers without sending — the kickoff rode the task's initial prompt", () => { + it("registers without sending because the kickoff rode the task initial prompt", () => { const run = service.registerRun(baseConfig); expect(run.status).toBe("running"); @@ -367,6 +384,47 @@ describe("AutoresearchService", () => { expect(sentPrompts[0].prompt).toContain("iteration 2"); }); + it("tracks reports from an initial prompt already in progress", () => { + beginTurn(); + service.registerRun(baseConfig); + + streamTurnText(`${reportText(10, "baseline")}\nIteration 2 starts now.`); + streamTurnText( + `${reportText(12, "iteration 2")}\nIteration 3 starts now.`, + ); + streamTurnText( + `${reportText(14, "iteration 3")}\nIteration 4 starts now.`, + ); + + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ index: 1, value: 10 }), + expect.objectContaining({ index: 2, value: 12 }), + expect.objectContaining({ index: 3, value: 14 }), + ]); + expect(sentPrompts).toHaveLength(0); + + completeTurn("Continuing after the reported iterations."); + + expect(activeRun().iterations).toHaveLength(3); + expect(sentPrompts).toHaveLength(1); + expect(sentPrompts[0].prompt).toContain("iteration 4"); + }); + + it("recovers when an active prompt was incorrectly marked handled", () => { + beginTurn(); + const run = service.registerRun(baseConfig); + const internals = service as unknown as { + promptCursor: Map; + }; + internals.promptCursor.set(run.id, 1); + + streamTurnText(`${reportText(10, "baseline")}\nIteration 2 starts now.`); + + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ index: 1, value: 10 }), + ]); + }); + it("shares the one-live-run-per-task guard with startRun", () => { service.registerRun(baseConfig); expect(() => service.startRun(baseConfig)).toThrow(/already running/); @@ -374,6 +432,97 @@ describe("AutoresearchService", () => { }); describe("iteration loop", () => { + it("records a metric block after the next iteration starts", () => { + service.startRun(baseConfig); + beginTurn(); + streamTurnText(`${reportText(10, "baseline")}\nIteration 2 starts now.`); + + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ index: 1, value: 10, summary: "baseline" }), + ]); + expect(sentPrompts).toHaveLength(1); + + streamTurnText("Continuing to inspect the next change."); + expect(activeRun().iterations).toHaveLength(1); + + completeTurn("Finished the turn."); + expect(activeRun().iterations).toHaveLength(1); + expect(sentPrompts).toHaveLength(2); + expect(sentPrompts.at(-1)?.prompt).toContain( + "Then make the next focused change", + ); + }); + + it("keeps the tail report provisional until the turn ends", () => { + service.startRun({ ...baseConfig, maxIterations: 1 }); + beginTurn(); + streamTurnText(reportText(10, "draft baseline")); + + expect(activeRun().iterations).toHaveLength(0); + expect(activeRun().status).toBe("running"); + + streamTurnText(reportText(12, "corrected baseline")); + expect(activeRun().iterations).toHaveLength(0); + + completeTurn("Finalized the corrected report."); + + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ + index: 1, + value: 12, + summary: "corrected baseline", + }), + ]); + expect(activeRun().status).toBe("completed"); + expect(activeRun().endReason).toBe("max-iterations"); + }); + + it("records prebaseline research without consuming an iteration", () => { + service.startRun(baseConfig); + runTurn(researchText()); + + const researchingRun = activeRun(); + expect(researchingRun.researchFindings).toEqual([ + expect.objectContaining({ + index: 1, + summary: "Mapped the execution path", + finding: "The metric is computed in the workspace server.", + nextStep: "Trace the benchmark command", + }), + ]); + expect(researchingRun.iterations).toHaveLength(0); + expect(sentPrompts.at(-1)?.prompt).toContain( + "Continue investigating the codebase or establish the baseline measurement", + ); + + runTurn(reportText(10, "baseline")); + + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ index: 1, value: 10, summary: "baseline" }), + ]); + }); + + it("does not accept research checkpoints after the baseline", async () => { + service.startRun(baseConfig); + runTurn(reportText(10, "baseline")); + runTurn(researchText()); + await passReminderGrace(); + + expect(activeRun().researchFindings).toHaveLength(0); + expect(activeRun().iterations).toHaveLength(1); + expect(sentPrompts.at(-1)?.prompt).toContain("did not include"); + }); + + it("prefers a metric when a prebaseline reply contains both report types", () => { + service.startRun(baseConfig); + runTurn(`${researchText()}\n${reportText(10, "baseline")}`); + + expect(activeRun().researchFindings).toHaveLength(0); + expect(activeRun().iterations).toEqual([ + expect.objectContaining({ index: 1, value: 10, summary: "baseline" }), + ]); + }); + it("records an iteration and sends a continuation prompt", () => { service.startRun(baseConfig); runTurn(reportText(10, "baseline")); @@ -534,7 +683,7 @@ describe("AutoresearchService", () => { ); expect(activeRun().metricUnit).toBe("kB"); - // First unit wins, like the name — a stable unit keeps values readable. + // First unit wins, like the name. A stable unit keeps values readable. runTurn("```autoresearch\nmetric: 400000\nunit: bytes\n```"); expect(activeRun().metricUnit).toBe("kB"); }); @@ -572,7 +721,7 @@ describe("AutoresearchService", () => { expect(modelSwitches).toEqual([ { taskId: TASK_ID, model: "claude-opus-4-8" }, ]); - expect(sentPrompts.at(-1)?.prompt).toContain("implementation phase"); + expect(sentPrompts.at(-1)?.prompt).toContain("Implementation phase"); expect(sentPrompts.at(-1)?.prompt).toContain( "Do NOT run the measurement", ); @@ -610,7 +759,7 @@ describe("AutoresearchService", () => { expect(activeRun().iterations).toHaveLength(2); expect(activeRun().phase).toBe("implement"); - expect(sentPrompts.at(-1)?.prompt).toContain("implementation phase"); + expect(sentPrompts.at(-1)?.prompt).toContain("Implementation phase"); }); it("still fails a measure turn that never reports", async () => { @@ -988,6 +1137,9 @@ describe("AutoresearchService", () => { bestValue: 10, delta: null, summary: "baseline", + hypothesis: null, + plan: null, + approach: null, at: 1_000, }, ], @@ -1024,7 +1176,7 @@ describe("AutoresearchService", () => { ]); await service.rehydrate(); - // No session exists for task-9 yet — recovery asks the host to + // No session exists for task 9 yet. Recovery asks the host to // reconnect it. await passRecoveryDelay(); expect(reconnectCalls).toEqual(["task-9"]); @@ -1074,7 +1226,7 @@ describe("AutoresearchService", () => { // The in-memory run (with its recorded iteration) wins over the row. expect(state.runs[live.id]?.iterations).toHaveLength(1); expect(state.runs["ar-past"]?.status).toBe("completed"); - // The live run stays the active one — it started later. + // The live run stays active because it started later. expect(state.activeRunIdByTask[TASK_ID]).toBe(live.id); }); @@ -1203,7 +1355,9 @@ describe("AutoresearchService", () => { // No phase alternation: the next prompt is a plain continuation. expect(activeRun().phase).toBeNull(); - expect(sentPrompts.at(-1)?.prompt).toContain("Continue:"); + expect(sentPrompts.at(-1)?.prompt).toContain( + "Then make the next focused change", + ); }); }); diff --git a/packages/core/src/autoresearch/autoresearch.ts b/packages/core/src/autoresearch/autoresearch.ts index f837a2c8af..6b0f64bbab 100644 --- a/packages/core/src/autoresearch/autoresearch.ts +++ b/packages/core/src/autoresearch/autoresearch.ts @@ -28,10 +28,13 @@ import { buildMeasurePrompt, buildPhasePrompt, buildReportReminderPrompt, + buildResearchContinuationPrompt, buildResumePrompt, countPromptRequests, extractLastAgentTurnText, - parseMetricReport, + parseMetricReports, + parseResearchReports, + parseStreamedMetricReports, } from "./prompts"; import { type AutoresearchConfigInput, @@ -68,7 +71,7 @@ export const MAX_RECOVERY_ATTEMPTS = 20; * * Infrastructure obstacles (session drop, idle-kill, usage limit, app * restart) never end a run: they mark it `interrupted` and the service keeps - * trying to bring it back — reconnecting the session when it can and + * trying to bring it back by reconnecting the session when it can and * resuming the loop once the session is usable again. Only the user * (pause/stop), the iteration budget, the target, or a protocol breakdown * (the agent repeatedly not reporting) ends a run. Every mutation is @@ -107,10 +110,14 @@ export class AutoresearchService { private pendingReactions = new Map>(); /** * Count of session/prompt requests already handled per run. A turn - * completion is only processed when the count moved past this cursor — + * completion is only processed when the count moved past this cursor. * `isPromptPending` flips without a new prompt when a send fails. */ private promptCursor = new Map(); + private streamedReportCursor = new Map< + string, + { promptCount: number; metricCount: number; researchCount: number } + >(); private recoveryTimers = new Map>(); private recoveryAttempts = new Map(); /** Per-run write-through chain so persisted saves land in call order. */ @@ -125,7 +132,7 @@ export class AutoresearchService { private rehydrated = false; /** - * Register a run whose kickoff prompt the host has already delivered — + * Register a run whose kickoff prompt the host has already delivered. * the create-task flow sends it as the new task's initial prompt. The * engine takes over from the agent's first reply. */ @@ -152,6 +159,7 @@ export class AutoresearchService { phase: null, originalModel: session ? currentSessionModel(session) : null, originalEffort: session ? currentSessionEffort(session) : null, + researchFindings: [], iterations: [], startedAt: Date.now(), endedAt: null, @@ -162,9 +170,10 @@ export class AutoresearchService { autoresearchStoreActions.upsertRun(run); this.liveRunIds.add(run.id); + const promptCount = session ? countPromptRequests(session.events) : 0; this.promptCursor.set( run.id, - session ? countPromptRequests(session.events) : 0, + Math.max(0, promptCount - (session?.isPromptPending ? 1 : 0)), ); this.persist(run.id); this.ensureSubscribed(); @@ -180,7 +189,7 @@ export class AutoresearchService { * Register a run and send its kickoff into the task's existing session. * Used to start a fresh run on a task that already ran autoresearch. * (For composer-created tasks the kickoff rides the initial prompt and the - * baseline turn runs on the task's creation model — stage models take over + * baseline turn runs on the task's creation model. Stage models take over * from iteration 2.) */ startRun(input: AutoresearchConfigInput): AutoresearchRun { @@ -233,7 +242,7 @@ export class AutoresearchService { (session.isPromptPending || session.isCompacting) ) { // A turn is already in flight; the loop re-engages when it completes. - // Don't move the cursor — that in-flight prompt is the next turn. + // Do not move the cursor. That prompt in progress is the next turn. autoresearchStoreActions.setRunStatus(runId, "running"); this.persist(runId); this.log.info("Autoresearch run resumed", { runId }); @@ -241,7 +250,7 @@ export class AutoresearchService { } if (!session || !isSessionUsable(session)) { // The session is down; let the recovery machinery bring it back and - // resume the loop. Attempt immediately — the user asked for it now. + // resume the loop. Attempt immediately because the user asked for it now. // A manual resume is a fresh start, so it does not spend the automatic // recovery budget. this.recoveryAttempts.delete(runId); @@ -285,7 +294,7 @@ export class AutoresearchService { if (this.rehydrated) return; this.rehydrated = true; - // Feature-flagged: for ungated users the whole feature stays dormant — + // Feature flagged: for ungated users the whole feature stays dormant. // no restored runs, no auto-resume, no session subscription. if (!(await this.gate.isEnabled())) { this.log.info("Autoresearch disabled by feature flag; skipping restore"); @@ -409,7 +418,7 @@ export class AutoresearchService { } if (run.status === "interrupted") { - // Resume as soon as the session comes back — the reconnect may have + // Resume as soon as the session comes back. The reconnect may have // been ours (recovery) or the app's own reconciliation. if (isSessionUsable(session) && !isSessionUsable(prevSession)) { this.resumeFromInterruption(run.id); @@ -420,31 +429,127 @@ export class AutoresearchService { const turnCompleted = prevSession?.isPromptPending === true && session.isPromptPending === false; - if (!turnCompleted) continue; - const promptCount = countPromptRequests(session.events); + + if (session.isPromptPending) { + if (promptCount === 0) continue; + this.ingestCompletedReports(run, session, false); + continue; + } if (promptCount <= (this.promptCursor.get(run.id) ?? 0)) continue; + if (!turnCompleted) continue; + + const reports = this.ingestCompletedReports(run, session, true); this.promptCursor.set(run.id, promptCount); - this.onAgentTurnComplete(run, session); + this.onAgentTurnComplete(run.id, reports); } } - private onAgentTurnComplete( + private ingestCompletedReports( run: AutoresearchRun, session: AgentSession, - ): void { + turnComplete: boolean, + ): { hasMetric: boolean; hasResearch: boolean } { const text = extractLastAgentTurnText(session.events); - const report = parseMetricReport(text); - - if (!report) { - // Deferred: a reportless turn is either a split-run implement turn - // (advance to measure) or a genuine missing report (remind). Both wait - // out the grace window so a cancel/rate-limit resolving in the - // meantime pauses/interrupts the run first. - this.scheduleReportlessReaction(run.id); + const streamedMetricReports = parseStreamedMetricReports(text); + const allMetricReports = turnComplete ? parseMetricReports(text) : []; + const finalMetricReport = + allMetricReports.length > streamedMetricReports.length + ? allMetricReports.at(-1) + : null; + const metricReports = finalMetricReport + ? [...streamedMetricReports, finalMetricReport] + : streamedMetricReports; + const researchReports = parseResearchReports(text); + const promptCount = countPromptRequests(session.events); + const existing = this.streamedReportCursor.get(run.id); + const cursor = + existing?.promptCount === promptCount + ? existing + : { promptCount, metricCount: 0, researchCount: 0 }; + + if (metricReports.length === 0) { + for (const report of researchReports.slice(cursor.researchCount)) { + const current = autoresearchStore.getState().runs[run.id]; + if (!current || isTerminal(current)) break; + if (current.iterations.length > 0) break; + this.recordResearchFinding(current, report); + } + } + for (const report of metricReports.slice(cursor.metricCount)) { + const current = autoresearchStore.getState().runs[run.id]; + if (!current || isTerminal(current)) break; + this.recordMetricReport(current, report); + } + this.streamedReportCursor.set(run.id, { + promptCount, + metricCount: metricReports.length, + researchCount: researchReports.length, + }); + return { + hasMetric: metricReports.length > 0, + hasResearch: metricReports.length === 0 && researchReports.length > 0, + }; + } + + private recordResearchFinding( + run: AutoresearchRun, + researchReport: ReturnType[number], + ): void { + this.clearPendingReaction(run.id); + this.remindersSent.delete(run.id); + this.recoveryAttempts.delete(run.id); + autoresearchStoreActions.appendResearchFinding(run.id, { + index: run.researchFindings.length + 1, + summary: researchReport.summary, + finding: researchReport.finding, + nextStep: researchReport.nextStep, + area: researchReport.area, + at: Date.now(), + }); + this.persist(run.id); + } + + private onAgentTurnComplete( + runId: string, + reports: { hasMetric: boolean; hasResearch: boolean }, + ): void { + const run = autoresearchStore.getState().runs[runId]; + if (!run || run.status !== "running") return; + if (reports.hasMetric) { + const decision = evaluateContinuation(run); + if (decision.done) { + this.endRun(run.id, "completed", { endReason: decision.reason }); + this.log.info("Autoresearch run completed", { + runId: run.id, + reason: decision.reason, + iterations: run.iterations.length, + }); + } else { + this.continueLoop(run); + } + return; + } + if (reports.hasResearch && run.iterations.length === 0) { + void this.send( + run.id, + run.config.taskId, + buildResearchContinuationPrompt(run), + ); return; } + // Deferred: a reportless turn is either a split-run implement turn + // (advance to measure) or a genuine missing report (remind). Both wait + // out the grace window so a cancel/rate-limit resolving in the + // meantime pauses/interrupts the run first. + this.scheduleReportlessReaction(run.id); + } + + private recordMetricReport( + run: AutoresearchRun, + report: AutoresearchReport, + ): void { this.clearPendingReaction(run.id); this.remindersSent.delete(run.id); // The loop is demonstrably turning again; future interruptions restart @@ -464,22 +569,6 @@ export class AutoresearchService { } } this.persist(run.id); - - const updated = autoresearchStore.getState().runs[run.id]; - if (!updated || updated.status !== "running") return; - - const decision = evaluateContinuation(updated); - if (decision.done) { - this.endRun(run.id, "completed", { endReason: decision.reason }); - this.log.info("Autoresearch run completed", { - runId: run.id, - reason: decision.reason, - iterations: updated.iterations.length, - }); - return; - } - - this.continueLoop(updated); } /** Kick off the next iteration after a recorded report. */ @@ -517,7 +606,7 @@ export class AutoresearchService { } /** - * Switch the session's stage (model and/or effort), then send — in that + * Switch the session stage, including model or effort, then send in that * order, so the turn runs on the intended configuration rather than racing * the switch. A stage with nothing to set skips straight to the send with * no await, so ordering is unchanged for it. When there is a switch, a run @@ -541,7 +630,7 @@ export class AutoresearchService { } /** - * Apply a stage's model/effort to the session. Failures only warn — the + * Apply a stage model or effort to the session. Failures only warn. The * turn falls back to whatever the session currently has (effort options * can also legitimately differ per model, so a stale effort may be * rejected by the session). @@ -588,7 +677,7 @@ export class AutoresearchService { * React to a reportless turn once the grace window has passed and no * cancel/rate-limit/interruption intervened. In a split run's implement * phase that means advancing to the measure phase; otherwise it is a - * missing report — remind once, then fail. + * missing report. Remind once, then fail. */ private scheduleReportlessReaction(runId: string): void { const run = autoresearchStore.getState().runs[runId]; @@ -646,6 +735,9 @@ export class AutoresearchService { bestValue, delta: previous ? report.value - previous.value : null, summary: report.summary, + hypothesis: report.hypothesis, + plan: report.plan, + approach: report.approach, at: Date.now(), }); } @@ -667,7 +759,7 @@ export class AutoresearchService { "Usage limit reached. The loop retries automatically.", ); } else if (stopReason === "cancelled") { - // The user stopped the turn — hand them the wheel instead of + // The user stopped the turn. Hand them control instead of // immediately re-prompting the agent they just silenced. const run = autoresearchStore.getState().runs[runId]; if (run?.status === "running") { @@ -682,7 +774,7 @@ export class AutoresearchService { } } else if (stopReason === "queued") { // The session was busy (a turn/compaction in flight), so our prompt - // sits in the session queue and drains when it frees up — producing + // sits in the session queue and drains when it frees up, producing // a turn the subscription processes normally. Nothing to do but note // it; if the session never frees, its error path drives recovery. this.log.warn("Autoresearch prompt queued behind a busy session", { @@ -831,6 +923,7 @@ export class AutoresearchService { this.remindersSent.delete(runId); this.recoveryAttempts.delete(runId); this.promptCursor.delete(runId); + this.streamedReportCursor.delete(runId); this.liveRunIds.delete(runId); if (run) this.restoreOriginalStage(run); } @@ -849,7 +942,7 @@ export class AutoresearchService { /** * Write-through persistence. Saves for a run are chained so they land in - * call order — a later transition can never be overwritten by an in-flight + * call order. A later transition can never be overwritten by a pending * earlier one. The in-memory store stays authoritative. */ private persist(runId: string): void { @@ -897,7 +990,7 @@ function measureStage(run: AutoresearchRun): AutoresearchStage { /** * Split runs alternate an implement turn and a measure turn per iteration. - * Any difference between the stages — model or effort — makes the run split; + * Any difference between the stages, model or effort, makes the run split; * identical stages run as single turns with no mid-loop switching. */ function isSplitRun(run: AutoresearchRun): boolean { diff --git a/packages/core/src/autoresearch/autoresearchStore.ts b/packages/core/src/autoresearch/autoresearchStore.ts index 0e6123ac2c..dcf82fc0d3 100644 --- a/packages/core/src/autoresearch/autoresearchStore.ts +++ b/packages/core/src/autoresearch/autoresearchStore.ts @@ -4,6 +4,7 @@ import { type AutoresearchInterruptionReason, type AutoresearchIteration, type AutoresearchPhase, + type AutoresearchResearchFinding, type AutoresearchRun, type AutoresearchRunStatus, isTerminalRunStatus, @@ -50,6 +51,16 @@ export const autoresearchStoreActions = { })); }, + appendResearchFinding( + runId: string, + finding: AutoresearchResearchFinding, + ): void { + updateRun(runId, (run) => ({ + ...run, + researchFindings: [...run.researchFindings, finding], + })); + }, + /** Record the metric label the agent chose in its reports. */ setMetricName(runId: string, metricName: string): void { updateRun(runId, (run) => ({ ...run, metricName })); diff --git a/packages/core/src/autoresearch/identifiers.ts b/packages/core/src/autoresearch/identifiers.ts index 3a6edc0c5f..f2b39eaf2f 100644 --- a/packages/core/src/autoresearch/identifiers.ts +++ b/packages/core/src/autoresearch/identifiers.ts @@ -40,7 +40,7 @@ export interface StoredAutoresearchRun { */ export interface AutoresearchStorageClient { save(run: StoredAutoresearchRun): Promise; - /** Runs not yet terminal — the ones worth resuming after a restart. */ + /** Runs not yet terminal. These are worth resuming after a restart. */ listOpen(): Promise; listByTask(taskId: string): Promise; } diff --git a/packages/core/src/autoresearch/prompts.test.ts b/packages/core/src/autoresearch/prompts.test.ts index c664ed2612..9b2ee670f2 100644 --- a/packages/core/src/autoresearch/prompts.test.ts +++ b/packages/core/src/autoresearch/prompts.test.ts @@ -2,13 +2,18 @@ import type { AcpMessage } from "@posthog/shared"; import { describe, expect, it } from "vitest"; import { buildContinuationPrompt, + buildImplementPrompt, buildKickoffPreamble, buildKickoffPrompt, + buildMeasurePrompt, buildReportReminderPrompt, buildResumePrompt, countPromptRequests, extractLastAgentTurnText, parseMetricReport, + parsePlanReport, + parseResearchReport, + parseStreamedMetricReports, } from "./prompts"; import type { AutoresearchConfig, @@ -38,7 +43,17 @@ function makeIteration( value: number, summary: string | null = null, ): AutoresearchIteration { - return { index, value, bestValue: value, delta: null, summary, at: index }; + return { + index, + value, + bestValue: value, + delta: null, + summary, + hypothesis: null, + plan: null, + approach: null, + at: index, + }; } function makeRun( @@ -54,6 +69,7 @@ function makeRun( phase: null, originalModel: null, originalEffort: null, + researchFindings: [], iterations, startedAt: 0, endedAt: null, @@ -77,6 +93,9 @@ describe("parseMetricReport", () => { name: null, unit: null, summary: "swapped JSON parser", + hypothesis: null, + plan: null, + approach: null, }); }); @@ -86,6 +105,9 @@ describe("parseMetricReport", () => { name: null, unit: null, summary: null, + hypothesis: null, + plan: null, + approach: null, }); }); @@ -104,6 +126,9 @@ describe("parseMetricReport", () => { name: null, unit: null, summary: "final", + hypothesis: null, + plan: null, + approach: null, }); }); @@ -114,6 +139,9 @@ describe("parseMetricReport", () => { name: null, unit: null, summary: "good", + hypothesis: null, + plan: null, + approach: null, }); }); @@ -132,10 +160,120 @@ describe("parseMetricReport", () => { name: null, unit: null, summary: "tidy", + hypothesis: null, + plan: null, + approach: null, + }); + }); + + it("parses experiment context from a metric report", () => { + expect( + parseMetricReport( + "```autoresearch\nmetric: 90\nsummary: memoized selectors\nhypothesis: repeated selectors dominate render time\nplan: memoize selectors and rerun the benchmark\napproach: rendering\n```", + ), + ).toEqual({ + value: 90, + name: null, + unit: null, + summary: "memoized selectors", + hypothesis: "repeated selectors dominate render time", + plan: "memoize selectors and rerun the benchmark", + approach: "rendering", + }); + }); + + it("ignores research only blocks", () => { + expect( + parseMetricReport( + "```autoresearch\ntype: research\nsummary: traced routing\nfinding: routes are contributed by UI modules\n```", + ), + ).toBeNull(); + }); +}); + +describe("parseStreamedMetricReports", () => { + it("ignores a report at the streaming tail", () => { + expect(parseStreamedMetricReports(report("10", "draft"))).toEqual([]); + }); + + it("accepts a report once the next iteration starts", () => { + expect( + parseStreamedMetricReports( + `${report("10", "baseline")}\nIteration 2 starts now.`, + ), + ).toEqual([expect.objectContaining({ value: 10, summary: "baseline" })]); + }); +}); + +describe("parseResearchReport", () => { + it("parses a valid research checkpoint", () => { + expect( + parseResearchReport( + "```autoresearch\ntype: research\nsummary: traced routing\nfinding: routes are contributed by UI modules\nnext: inspect the contribution\n```", + ), + ).toEqual({ + summary: "traced routing", + finding: "routes are contributed by UI modules", + nextStep: "inspect the contribution", + area: null, + }); + }); + + it.each([ + ["missing type", "summary: traced routing\nfinding: found it"], + ["missing summary", "type: research\nfinding: found it"], + ["missing finding", "type: research\nsummary: traced routing"], + ])("rejects a checkpoint with %s", (_name, body) => { + expect( + parseResearchReport(`\`\`\`autoresearch\n${body}\n\`\`\``), + ).toBeNull(); + }); + + it("uses the last valid research checkpoint", () => { + const text = [ + "```autoresearch", + "type: research", + "summary: first", + "finding: initial finding", + "```", + "```autoresearch", + "type: research", + "summary: second", + "finding: final finding", + "```", + ].join("\n"); + + expect(parseResearchReport(text)).toEqual({ + summary: "second", + finding: "final finding", + nextStep: null, + area: null, }); }); }); +describe("parsePlanReport", () => { + it("parses a complete experiment plan", () => { + expect( + parsePlanReport( + "```autoresearch\ntype: plan\nhypothesis: repeated queries dominate latency\nplan: cache the query and rerun the benchmark\napproach: caching\n```", + ), + ).toEqual({ + hypothesis: "repeated queries dominate latency", + plan: "cache the query and rerun the benchmark", + approach: "caching", + }); + }); + + it("requires every plan field", () => { + expect( + parsePlanReport( + "```autoresearch\ntype: plan\nhypothesis: repeated queries dominate latency\n```", + ), + ).toBeNull(); + }); +}); + function promptEvent(ts: number): AcpMessage { return { type: "acp_message", @@ -260,6 +398,12 @@ describe("buildKickoffPreamble", () => { "Optimize the HTTP handler throughput benchmark.", ); }); + + it("requires autoresearch pull request attribution", () => { + const preamble = buildKickoffPreamble(makeConfig()); + expect(preamble).toContain("feat(autoresearch): "); + expect(preamble).toContain('"Created with Autoresearch."'); + }); }); describe("buildContinuationPrompt", () => { @@ -276,14 +420,34 @@ describe("buildContinuationPrompt", () => { expect(prompt).toContain("```autoresearch"); }); + it("preserves the pull request convention in later iterations", () => { + const run = makeRun([makeIteration(1, 100, "baseline")]); + expect(buildContinuationPrompt(run)).toContain( + "feat(autoresearch): ", + ); + expect(buildImplementPrompt(run)).toContain('"Created with Autoresearch."'); + expect(buildMeasurePrompt(run)).toContain( + "feat(autoresearch): ", + ); + }); + + it("requests an experiment plan in every implementation path", () => { + const run = makeRun([makeIteration(1, 100, "baseline")]); + expect(buildContinuationPrompt(run)).toContain("type: plan"); + expect(buildImplementPrompt(run)).toContain("type: plan"); + expect(buildMeasurePrompt(run)).toContain( + "Repeat the hypothesis, plan, and approach", + ); + }); + it("only lists the five most recent iterations", () => { const run = makeRun( Array.from({ length: 7 }, (_, i) => makeIteration(i + 1, i + 1)), ); const prompt = buildContinuationPrompt(run); - expect(prompt).not.toContain("- Iteration 2:"); - expect(prompt).toContain("- Iteration 3:"); - expect(prompt).toContain("- Iteration 7:"); + expect(prompt).not.toContain("Iteration 2:"); + expect(prompt).toContain("Iteration 3:"); + expect(prompt).toContain("Iteration 7:"); }); }); @@ -308,7 +472,7 @@ describe("buildResumePrompt", () => { it("warns about half-applied changes from the aborted iteration", () => { const prompt = buildResumePrompt(makeRun([]), "app-restart"); expect(prompt).toContain("the app restarted"); - expect(prompt).toContain("half-applied change"); + expect(prompt).toContain("partially applied change"); }); }); @@ -321,6 +485,9 @@ describe("unit parsing and rendering", () => { name: "bundle size", unit: "kB", summary: "trimmed deps", + hypothesis: null, + plan: null, + approach: null, }); }); @@ -336,7 +503,7 @@ describe("unit parsing and rendering", () => { metricUnit: "ms", }; const prompt = buildContinuationPrompt(run); - expect(prompt).toContain("- Iteration 1: 100 ms — baseline"); + expect(prompt).toContain("Iteration 1: 100 ms. baseline"); expect(prompt).toContain("Best so far: 100 ms (iteration 1)"); expect(prompt).toContain("Last: 100 ms"); }); diff --git a/packages/core/src/autoresearch/prompts.ts b/packages/core/src/autoresearch/prompts.ts index f4c11f8d1c..aded244303 100644 --- a/packages/core/src/autoresearch/prompts.ts +++ b/packages/core/src/autoresearch/prompts.ts @@ -2,14 +2,16 @@ * The autoresearch protocol: how we brief the agent, how the agent reports * metric measurements back, and how we read those reports out of the session * transcript. Prompt builders and the report parser are two sides of the same - * contract — keep them in sync. + * contract. Keep them in sync. */ import type { AcpMessage } from "@posthog/shared"; import { isJsonRpcNotification, isJsonRpcRequest } from "@posthog/shared"; import type { AutoresearchDraftConfig, AutoresearchInterruptionReason, + AutoresearchPlanReport, AutoresearchReport, + AutoresearchResearchReport, AutoresearchRun, } from "./schemas"; import { computeBest } from "./stats"; @@ -17,17 +19,43 @@ import { computeBest } from "./stats"; const REPORT_BLOCK_EXAMPLE = [ "```autoresearch", "metric: ", - "name: ", - "unit: ", + "name: ", + "unit: ", "summary: ", + "hypothesis: ", + "plan: ", + "approach: ", "```", ].join("\n"); +const RESEARCH_BLOCK_EXAMPLE = [ + "```autoresearch", + "type: research", + "summary: ", + "area: ", + "finding: ", + "next: ", + "```", +].join("\n"); + +const PLAN_BLOCK_EXAMPLE = [ + "```autoresearch", + "type: plan", + "hypothesis: ", + "plan: ", + "approach: ", + "```", +].join("\n"); + +const PULL_REQUEST_CONVENTION = `Pull requests created during this run must follow both rules: +Title: \`feat(autoresearch): \` +Description: include the sentence "Created with Autoresearch."`; + function directionPhrase(config: AutoresearchDraftConfig): string { return config.direction === "maximize" ? "maximize" : "minimize"; } -/** The metric as we can name it so far — reports define it, the brief implies it. */ +/** Reports define the metric name. Until then, the brief only implies it. */ function metricPhrase(run: AutoresearchRun): string { return run.metricName ? `"${run.metricName}"` : "the metric"; } @@ -42,20 +70,30 @@ function targetLine(config: AutoresearchDraftConfig): string { * Everything the kickoff says before the optimization brief. Hosts that * deliver the kickoff as a new task's initial prompt prepend this to the * user's own prompt content, so file/folder chips survive untouched. - * The metric is not named here — the brief defines it and the agent labels + * The metric is not named here. The brief defines it and the agent labels * it in every report's `name:` line. */ export function buildKickoffPreamble(config: AutoresearchDraftConfig): string { return `You are now in autoresearch mode: an iterative optimization loop to ${directionPhrase(config)} the metric defined by the brief below. Protocol for every iteration: -1. Make ONE focused change aimed at improving the metric. Keep changes small and attributable. -2. Measure the metric after your change. -3. End your reply with exactly one report block in this format (plain number, no units or thousands separators): +1. Before editing, emit one plan block describing the hypothesis, plan, and approach: + +${PLAN_BLOCK_EXAMPLE} + +2. Make ONE focused change aimed at improving the metric. Keep changes small and attributable. +3. Measure the metric after your change. +4. End your reply with exactly one report block in this format (plain number, no units or thousands separators): ${REPORT_BLOCK_EXAMPLE} -The report block is parsed by a machine — without it the iteration does not count. Budget: up to ${config.maxIterations} iterations.${targetLine(config)} +Before the baseline is available, codebase research can take multiple turns. When a research turn produces useful information but no metric yet, end the reply with this checkpoint instead: + +${RESEARCH_BLOCK_EXAMPLE} + +The report block is parsed by a machine. Without it, the iteration does not count. Budget: up to ${config.maxIterations} iterations.${targetLine(config)} + +${PULL_REQUEST_CONVENTION} Iteration 1 starts now. First establish and report the baseline measurement (your change for this iteration is the measurement setup itself if nothing exists yet), then keep improving in later iterations. If a change regresses the metric, revert it in the next iteration and try a different approach. @@ -78,7 +116,7 @@ function historyBlock(run: AutoresearchRun): string { .slice(-5) .map( (iteration) => - `- Iteration ${iteration.index}: ${iteration.value}${unit}${iteration.summary ? ` — ${iteration.summary}` : ""}`, + `Iteration ${iteration.index}: ${iteration.value}${unit}${iteration.summary ? `. ${iteration.summary}` : ""}`, ) .join("\n"); @@ -96,32 +134,65 @@ export function buildContinuationPrompt(run: AutoresearchRun): string { ${historyBlock(run)} -Continue: make the next focused change, measure ${metricPhrase(run)}, and end your reply with the report block: +${PULL_REQUEST_CONVENTION} + +Before editing, emit the experiment plan: + +${PLAN_BLOCK_EXAMPLE} + +Then make the next focused change, measure ${metricPhrase(run)}, and end your reply with the report block: ${REPORT_BLOCK_EXAMPLE}`; } +export function buildResearchContinuationPrompt(run: AutoresearchRun): string { + const latest = run.researchFindings[run.researchFindings.length - 1]; + return `Autoresearch research checkpoint ${run.researchFindings.length} recorded.${latest?.nextStep ? ` Next step: ${latest.nextStep}.` : ""} + +Continue investigating the codebase or establish the baseline measurement. Before editing or measuring, emit the experiment plan: + +${PLAN_BLOCK_EXAMPLE} + +When you have a metric, end with the metric report block: + +${REPORT_BLOCK_EXAMPLE} + +If another research turn produces a useful finding but no metric, end with a research checkpoint: + +${RESEARCH_BLOCK_EXAMPLE}`; +} + /** * First half of a split iteration: think and change, don't measure. Runs on - * the implement-stage model. + * the implementation stage model. */ export function buildImplementPrompt(run: AutoresearchRun): string { const nextIndex = run.iterations.length + 1; - return `Autoresearch iteration ${nextIndex} of ${run.config.maxIterations} for ${metricPhrase(run)} (${directionPhrase(run.config)}) — implementation phase. + return `Autoresearch iteration ${nextIndex} of ${run.config.maxIterations} for ${metricPhrase(run)} (${directionPhrase(run.config)}). Implementation phase. ${historyBlock(run)} -Plan and implement ONE focused change aimed at improving the metric. Do NOT run the measurement in this turn — a separate measurement turn follows. Reply with a one-line summary of what you changed and why.`; +${PULL_REQUEST_CONVENTION} + +Start by emitting the experiment plan: + +${PLAN_BLOCK_EXAMPLE} + +Then implement ONE focused change aimed at improving the metric. Do NOT run the measurement in this turn. A separate measurement turn follows. Reply with a single line summary of what you changed and why.`; } /** * Second half of a split iteration: run the measurement and report. Runs on - * the measure-stage model, which can be a cheap one — this turn is tool + * the measurement stage model, which can be a cheap one. This turn is tool * calls, not thinking. */ export function buildMeasurePrompt(run: AutoresearchRun): string { - return `Measurement phase: run the measurement for ${metricPhrase(run)} exactly as the brief describes, without changing any code. End your reply with the report block: + return `Measurement phase: run the measurement for ${metricPhrase(run)} exactly as the brief describes, without changing any code. + +${PULL_REQUEST_CONVENTION} + +End your reply with the report block. Repeat the hypothesis, plan, and approach from the implementation phase so the experiment is recorded: ${REPORT_BLOCK_EXAMPLE}`; } @@ -134,7 +205,7 @@ const INTERRUPTION_PHRASE: Record = { }; /** - * The prompt that (re-)enters the loop at the run's current phase — the + * The prompt that enters the loop again at the run's current phase. This is the * single source of truth for phase → prompt, shared by resume and the manual * resume path so the two can't drift. */ @@ -147,14 +218,14 @@ export function buildPhasePrompt(run: AutoresearchRun): string { /** * Continuation sent when the loop re-engages after an interruption. States * why the loop went quiet so the agent can re-check the workspace state - * (a half-applied change from the aborted iteration must be measured or + * (a partially applied change from the aborted iteration must be measured or * reverted, not assumed), then re-enters at the phase the run was in. */ export function buildResumePrompt( run: AutoresearchRun, reason: AutoresearchInterruptionReason, ): string { - return `The autoresearch loop was interrupted (${INTERRUPTION_PHRASE[reason]}) and is resuming now. Check the working tree for any half-applied change from the aborted iteration before continuing. + return `The autoresearch loop was interrupted (${INTERRUPTION_PHRASE[reason]}) and is resuming now. Check the working tree for any partially applied change from the aborted iteration before continuing. ${buildPhasePrompt(run)}`; } @@ -168,14 +239,65 @@ ${REPORT_BLOCK_EXAMPLE}`; const REPORT_BLOCK_REGEX = /```autoresearch\s*\n([\s\S]*?)```/g; /** - * Parse the agent's metric report from a reply. The last well-formed + * Parse the agent's metric report from a reply. The last valid * ```autoresearch fenced block wins, so an agent quoting the protocol and * then reporting still parses correctly. */ export function parseMetricReport(text: string): AutoresearchReport | null { - let report: AutoresearchReport | null = null; + return parseMetricReports(text).at(-1) ?? null; +} + +export function parseMetricReports(text: string): AutoresearchReport[] { + return parseMetricReportBlocks(text).map(({ report }) => report); +} + +export function parseStreamedMetricReports(text: string): AutoresearchReport[] { + return parseMetricReportBlocks(text) + .filter(({ end }) => startsAnotherIteration(text.slice(end))) + .map(({ report }) => report); +} + +function parseMetricReportBlocks( + text: string, +): Array<{ report: AutoresearchReport; end: number }> { + const reports: Array<{ report: AutoresearchReport; end: number }> = []; for (const match of text.matchAll(REPORT_BLOCK_REGEX)) { const parsed = parseReportBody(match[1]); + if (parsed) { + reports.push({ + report: parsed, + end: (match.index ?? 0) + match[0].length, + }); + } + } + return reports; +} + +function startsAnotherIteration(text: string): boolean { + return /\b(?:for\s+)?iteration\s+\d+\b/i.test(text); +} + +export function parseResearchReport( + text: string, +): AutoresearchResearchReport | null { + return parseResearchReports(text).at(-1) ?? null; +} + +export function parseResearchReports( + text: string, +): AutoresearchResearchReport[] { + const reports: AutoresearchResearchReport[] = []; + for (const match of text.matchAll(REPORT_BLOCK_REGEX)) { + const parsed = parseResearchBody(match[1]); + if (parsed) reports.push(parsed); + } + return reports; +} + +export function parsePlanReport(text: string): AutoresearchPlanReport | null { + let report: AutoresearchPlanReport | null = null; + for (const match of text.matchAll(REPORT_BLOCK_REGEX)) { + const parsed = parsePlanBody(match[1]); if (parsed) report = parsed; } return report; @@ -189,6 +311,9 @@ function parseReportBody(body: string): AutoresearchReport | null { let name: string | null = null; let unit: string | null = null; let summary: string | null = null; + let hypothesis: string | null = null; + let plan: string | null = null; + let approach: string | null = null; for (const line of body.split("\n")) { const separator = line.indexOf(":"); if (separator === -1) continue; @@ -207,9 +332,57 @@ function parseReportBody(body: string): AutoresearchReport | null { unit = raw; } else if (key === "summary" && raw.length > 0) { summary = raw; + } else if (key === "hypothesis" && raw.length > 0) { + hypothesis = raw; + } else if (key === "plan" && raw.length > 0) { + plan = raw; + } else if (key === "approach" && raw.length > 0) { + approach = raw; } } - return value === null ? null : { value, name, unit, summary }; + return value === null + ? null + : { value, name, unit, summary, hypothesis, plan, approach }; +} + +function parseResearchBody(body: string): AutoresearchResearchReport | null { + let type: string | null = null; + let summary: string | null = null; + let finding: string | null = null; + let nextStep: string | null = null; + let area: string | null = null; + for (const line of body.split("\n")) { + const separator = line.indexOf(":"); + if (separator === -1) continue; + const key = line.slice(0, separator).trim().toLowerCase(); + const raw = line.slice(separator + 1).trim(); + if (key === "type") type = raw.toLowerCase(); + else if (key === "summary" && raw.length > 0) summary = raw; + else if (key === "finding" && raw.length > 0) finding = raw; + else if (key === "next" && raw.length > 0) nextStep = raw; + else if (key === "area" && raw.length > 0) area = raw; + } + if (type !== "research" || !summary || !finding) return null; + return { summary, finding, nextStep, area }; +} + +function parsePlanBody(body: string): AutoresearchPlanReport | null { + let type: string | null = null; + let hypothesis: string | null = null; + let plan: string | null = null; + let approach: string | null = null; + for (const line of body.split("\n")) { + const separator = line.indexOf(":"); + if (separator === -1) continue; + const key = line.slice(0, separator).trim().toLowerCase(); + const raw = line.slice(separator + 1).trim(); + if (key === "type") type = raw.toLowerCase(); + else if (key === "hypothesis" && raw.length > 0) hypothesis = raw; + else if (key === "plan" && raw.length > 0) plan = raw; + else if (key === "approach" && raw.length > 0) approach = raw; + } + if (type !== "plan" || !hypothesis || !plan || !approach) return null; + return { hypothesis, plan, approach }; } interface AgentMessageChunkUpdate { @@ -222,8 +395,8 @@ interface AgentMessageChunkUpdate { /** * Number of session/prompt requests in the transcript. Used as a turn * cursor: `isPromptPending` can flip false without a turn having run (a - * rate-limited or failed send resets it), so a completion only counts when - * the prompt-request count moved past the last one handled. The count is + * a rate limit or failed send resets it), so a completion only counts when + * the prompt request count moved past the last one handled. The count is * stable across transcript replays, unlike event indexes. */ export function countPromptRequests(events: AcpMessage[]): number { diff --git a/packages/core/src/autoresearch/schemas.test.ts b/packages/core/src/autoresearch/schemas.test.ts index 57d6e6a2ce..6f10d4f1b7 100644 --- a/packages/core/src/autoresearch/schemas.test.ts +++ b/packages/core/src/autoresearch/schemas.test.ts @@ -100,6 +100,11 @@ describe("parseStoredAutoresearchRun", () => { ); }); + it("defaults research findings for legacy persisted runs", () => { + const run = parseStoredAutoresearchRun(storedRun("paused")); + expect(run?.researchFindings).toEqual([]); + }); + it.each([ ["corrupt JSON", "{nope"], ["schema mismatch", JSON.stringify({ id: "ar-1" })], diff --git a/packages/core/src/autoresearch/schemas.ts b/packages/core/src/autoresearch/schemas.ts index 575832724f..16d775ab7a 100644 --- a/packages/core/src/autoresearch/schemas.ts +++ b/packages/core/src/autoresearch/schemas.ts @@ -51,7 +51,7 @@ export const autoresearchConfigSchema = z.object({ * Stage configuration. When the implement and measure stages differ (in * model or effort), each iteration runs as two turns: an * ideation/implementation turn on the implement stage and a measurement - * turn on the measure stage (typically a cheaper model/effort — measuring + * turn on the measure stage, typically with a cheaper model or effort. Measuring * is tool calls, not thinking). When the stages are identical, iterations * are single turns. Null fields mean "leave the session's current value * alone". @@ -63,7 +63,7 @@ export const autoresearchConfigSchema = z.object({ /** * Free-form instructions for the agent: what to optimize, how to measure * the metric, and any constraints to respect. The metric itself is not - * configured anywhere — the agent names it in its reports based on this + * configured anywhere. The agent names it in its reports based on this * brief. */ instructions: z.string().trim().min(1), @@ -94,10 +94,25 @@ export const autoresearchIterationSchema = z.object({ delta: z.number().finite().nullable(), /** Agent's one-line description of what it changed. */ summary: z.string().nullable(), + hypothesis: z.string().nullable().default(null), + plan: z.string().nullable().default(null), + approach: z.string().nullable().default(null), at: z.number(), }); export type AutoresearchIteration = z.infer; +export const autoresearchResearchFindingSchema = z.object({ + index: z.number().int().min(1), + summary: z.string().min(1), + finding: z.string().min(1), + nextStep: z.string().nullable(), + area: z.string().nullable().default(null), + at: z.number(), +}); +export type AutoresearchResearchFinding = z.infer< + typeof autoresearchResearchFindingSchema +>; + export function isTerminalRunStatus(status: AutoresearchRunStatus): boolean { return status === "completed" || status === "stopped" || status === "failed"; } @@ -132,6 +147,7 @@ export const autoresearchRunSchema = z.object({ */ originalModel: z.string().nullable().default(null), originalEffort: z.string().nullable().default(null), + researchFindings: z.array(autoresearchResearchFindingSchema).default([]), iterations: z.array(autoresearchIterationSchema), startedAt: z.number(), endedAt: z.number().nullable(), @@ -177,4 +193,20 @@ export interface AutoresearchReport { /** The metric's unit, e.g. "kB", "ms", "%"; null for unitless counts. */ unit: string | null; summary: string | null; + hypothesis: string | null; + plan: string | null; + approach: string | null; +} + +export interface AutoresearchResearchReport { + summary: string; + finding: string; + nextStep: string | null; + area: string | null; +} + +export interface AutoresearchPlanReport { + hypothesis: string; + plan: string; + approach: string; } diff --git a/packages/core/src/autoresearch/stats.test.ts b/packages/core/src/autoresearch/stats.test.ts index bfed104d05..47489a6a54 100644 --- a/packages/core/src/autoresearch/stats.test.ts +++ b/packages/core/src/autoresearch/stats.test.ts @@ -19,6 +19,9 @@ function iteration( bestValue: value, delta: null, summary: null, + hypothesis: null, + plan: null, + approach: null, at: 1000 + index, ...overrides, }; @@ -49,6 +52,7 @@ function makeRun(overrides: { phase: null, originalModel: null, originalEffort: null, + researchFindings: [], iterations: overrides.iterations ?? [], startedAt: 0, endedAt: null, diff --git a/packages/ui/src/features/autoresearch/AutoresearchComposerControls.test.tsx b/packages/ui/src/features/autoresearch/AutoresearchComposerControls.test.tsx new file mode 100644 index 0000000000..031d0fcd88 --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchComposerControls.test.tsx @@ -0,0 +1,128 @@ +import type { AutoresearchDraftConfig } from "@posthog/core/autoresearch/schemas"; +import { Theme } from "@radix-ui/themes"; +import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { AutoresearchComposerControls } from "./AutoresearchComposerControls"; + +const draft: AutoresearchDraftConfig = { + direction: "maximize", + targetValue: null, + maxIterations: 10, + implementModel: "model-a", + measureModel: "model-a", + implementEffort: "medium", + measureEffort: "medium", +}; + +function renderControls( + overrides: Partial = {}, + onChange = vi.fn(), +) { + render( + + + , + ); + return onChange; +} + +describe("AutoresearchComposerControls", () => { + it("explains the workflow without hiding the prompt requirements", () => { + renderControls(); + + expect(screen.getByText("Autoresearch")).toBeVisible(); + expect( + screen.getByText( + /modifies the codebase and evaluates a user defined metric/i, + ), + ).toBeVisible(); + expect(screen.getByText("Include in your prompt")).toBeVisible(); + expect(screen.getByText("The metric to optimize")).toBeVisible(); + expect( + screen.getByText("The command or steps to measure it"), + ).toBeVisible(); + expect( + screen.getByText("Constraints the agent must preserve"), + ).toBeVisible(); + }); + + it("updates the attempt limit from the primary setup", () => { + const onChange = renderControls(); + + fireEvent.change(screen.getByLabelText("Maximum attempts"), { + target: { value: "6" }, + }); + + expect(onChange).toHaveBeenCalledWith({ maxIterations: 6 }); + }); + + it("shows both metric directions and updates the selected direction", async () => { + const user = userEvent.setup(); + const onChange = renderControls(); + + expect(screen.getByRole("radio", { name: "Increase" })).toBeVisible(); + expect(screen.getByRole("radio", { name: "Decrease" })).toBeVisible(); + + await user.click(screen.getByRole("radio", { name: "Decrease" })); + + expect(onChange).toHaveBeenCalledWith({ direction: "minimize" }); + }); + + it("keeps target configuration behind advanced settings", async () => { + const user = userEvent.setup(); + const onChange = renderControls(); + + expect( + screen.queryByLabelText("Target metric value to stop at"), + ).not.toBeInTheDocument(); + + await user.click( + screen.getByRole("button", { name: "Advanced autoresearch settings" }), + ); + fireEvent.change(screen.getByLabelText("Target metric value to stop at"), { + target: { value: "95" }, + }); + + expect(onChange).toHaveBeenCalledWith({ targetValue: 95 }); + }); + + it("explains the autoresearch loop in a help dialog", async () => { + const user = userEvent.setup(); + renderControls(); + + await user.click( + screen.getByRole("button", { name: "What is autoresearch?" }), + ); + + expect( + screen.getByRole("heading", { name: "What is autoresearch?" }), + ).toBeVisible(); + expect(screen.getByText("Measure a baseline")).toBeVisible(); + expect(screen.getByText("Try an improvement")).toBeVisible(); + expect(screen.getByText("Repeat until it stops")).toBeVisible(); + expect( + screen.getByRole("img", { + name: /Example autoresearch metric improving over five attempts/, + }), + ).toBeVisible(); + expect(screen.getByText("55 ms")).toBeVisible(); + expect(screen.getByText("Example prompt")).toBeVisible(); + expect(screen.getByRole("radio", { name: "Performance" })).toBeVisible(); + expect(screen.getByRole("radio", { name: "Bundle size" })).toBeVisible(); + expect( + screen.getByRole("radio", { name: "Test reliability" }), + ).toBeVisible(); + expect(screen.getByText(/pnpm bench:search/)).toBeVisible(); + expect( + screen.getByRole("button", { name: "Send feedback or report a bug" }), + ).toBeVisible(); + expect(screen.getByText("autoresearch@posthog.com")).toBeVisible(); + }); +}); diff --git a/packages/ui/src/features/autoresearch/AutoresearchComposerControls.tsx b/packages/ui/src/features/autoresearch/AutoresearchComposerControls.tsx index a563797cc0..a2b74c2ad0 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchComposerControls.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchComposerControls.tsx @@ -1,10 +1,33 @@ -import { ChartLineUp, SlidersHorizontal, X } from "@phosphor-icons/react"; +import { + ArrowDown, + ArrowUp, + ChartLineUp, + Command, + EnvelopeSimple, + Gauge, + LockKey, + Package, + Question, + SlidersHorizontal, + Speedometer, + TestTube, + X, +} from "@phosphor-icons/react"; import type { AutoresearchDirection, AutoresearchDraftConfig, } from "@posthog/core/autoresearch/schemas"; -import { Button, Popover, Select, Text, TextField } from "@radix-ui/themes"; -import { Tooltip } from "../../primitives/Tooltip"; +import { + Button, + Dialog, + Popover, + SegmentedControl, + Text, + TextField, +} from "@radix-ui/themes"; +import { domAnimation, LazyMotion, m, useReducedMotion } from "framer-motion"; +import { useState } from "react"; +import { openExternalUrl } from "../../shell/openExternal"; import { type AutoresearchModelOption, clampMaxIterations, @@ -21,17 +44,13 @@ interface AutoresearchComposerControlsProps { onExit: () => void; } +const AUTORESEARCH_FEEDBACK_MAILTO = + "mailto:autoresearch@posthog.com?subject=PostHog%20Code%20Autoresearch%20feedback"; + /** - * Autoresearch settings shown as a bar above the composer while the mode is - * armed. It reads as one sentence — "Optimize to maximize until it reaches N - * or after K iterations using " — so each control explains itself in - * place; tooltips on the labels add the detail. - * - * There is deliberately no metric or instructions field: the prompt IS the - * optimization brief, and the agent names the metric in its reports. - * - * While armed, the composer's own model/effort pickers are hidden and the - * stage popover here is the single place models and efforts are chosen. + * Compact autoresearch setup inside the new task composer. The prompt remains + * the primary input; this row only exposes the two choices most people need. + * Targets and per-stage model tuning stay behind advanced settings. */ export function AutoresearchComposerControls({ draft, @@ -42,109 +61,477 @@ export function AutoresearchComposerControls({ onExit, }: AutoresearchComposerControlsProps) { return ( -
- - - - Autoresearch - - - - {/* The goal: which way to push the metric the brief describes. */} - - - Optimize to - - - onChange({ direction: value as AutoresearchDirection }) +
+
+
+ +
+
+ + Autoresearch + + + Iteratively modifies the codebase and evaluates a user defined + metric. + +
+ +
+ +
+
+ + Goal + + + onChange({ direction: value as AutoresearchDirection }) + } + disabled={disabled} + aria-label="Metric goal" + > + + + + + + + +
+ +
+ + Maximum attempts + +
+ + onChange({ + maxIterations: clampMaxIterations( + Number.parseInt(event.target.value, 10), + ), + }) + } + inputMode="numeric" + aria-label="Maximum attempts" + disabled={disabled} + /> +
+
+ +
+ +
+
+ +
+
+ + Include in your prompt + +
    + + + +
+
+ +
+
+ ); +} + +function PromptRequirement({ + icon: Icon, + label, +}: { + icon: typeof Gauge; + label: string; +}) { + return ( +
  • + + {label} +
  • + ); +} + +function DirectionOption({ + direction, + selected, +}: { + direction: AutoresearchDirection; + selected: boolean; +}) { + const reducedMotion = useReducedMotion(); + const Icon = direction === "maximize" ? ArrowUp : ArrowDown; + const label = direction === "maximize" ? "Increase" : "Decrease"; + + return ( + + + - - - maximize - minimize - -
    -
    - - {/* Two stop conditions, whichever comes first: the metric hits the - target value, or the run exhausts its iteration budget. Only the - label text is wrapped in a tooltip — never the input — so focusing - the field to type doesn't pop the tooltip open. */} - - - until it reaches - - + + + {label} + + ); +} + +function AutoresearchInfoDialog() { + return ( + + + + + + What is autoresearch? + + Autoresearch runs a bounded experiment loop inside this task. + + +
    + +
    + + + +
    +
    + +
    + + Prompt requirements + + + + +
    + + + + + Autoresearch does not invent or independently verify the metric. It + follows the measurement instructions in your prompt. + + +
    +
    + + + autoresearch@posthog.com + +
    + + + +
    +
    +
    + ); +} + +const PROMPT_EXAMPLES = { + performance: { + icon: Speedometer, + title: "Performance", + prompt: + "Reduce the p95 response time of the search endpoint. Measure with `pnpm bench:search` and minimize the reported p95 milliseconds. Preserve response behavior and API compatibility.", + }, + bundle: { + icon: Package, + title: "Bundle size", + prompt: + "Reduce the gzipped JavaScript bundle size reported by `pnpm build:analyze`. Minimize total kB without removing features or changing browser support.", + }, + reliability: { + icon: TestTube, + title: "Test reliability", + prompt: + "Reduce failures in the checkout E2E suite. Measure by running `pnpm test:e2e checkout --repeat-each=20` and minimize failed runs without increasing test timeouts.", + }, +} as const; + +type PromptExampleKey = keyof typeof PROMPT_EXAMPLES; + +function PromptExamples() { + const [selected, setSelected] = useState("performance"); + const example = PROMPT_EXAMPLES[selected]; + const Icon = example.icon; + + return ( +
    +
    + + Example prompt + + - onChange({ - maxIterations: clampMaxIterations( - Number.parseInt(event.target.value, 10), - ), - }) - } - inputMode="numeric" - aria-label="Maximum iterations" - disabled={disabled} - /> - - iterations - - - - {/* The engine: model + effort per stage. */} - - - using - - - - - - - - - + value={selected} + onValueChange={(value) => { + if (value in PROMPT_EXAMPLES) + setSelected(value as PromptExampleKey); + }} + aria-label="Example prompt" + > + {Object.entries(PROMPT_EXAMPLES).map(([key, item]) => ( + + {item.title} + + ))} + +
    +
    +
    + + + {example.title} + +
    + + {example.prompt} + +
    +
    + ); +} + +const EXPERIMENT_POINTS = [ + { x: 38, y: 91, value: 72, label: "Baseline", improved: true }, + { x: 106, y: 70, value: 66, label: "Attempt 1", improved: true }, + { x: 174, y: 79, value: 69, label: "Attempt 2", improved: false }, + { x: 242, y: 45, value: 60, label: "Attempt 3", improved: true }, + { x: 310, y: 27, value: 55, label: "Best", improved: true }, +] as const; + +function ExperimentLoopVisual() { + const reducedMotion = useReducedMotion(); + const path = EXPERIMENT_POINTS.map( + (point, index) => `${index === 0 ? "M" : "L"}${point.x} ${point.y}`, + ).join(" "); + + return ( +
    +
    +
    + + Example: reduce page load time + + + Lower is better · 5 attempts + +
    +
    + + Best result + + + 55 ms + +
    +
    + + + + Example autoresearch metric improving over five attempts + + + Page load time starts at 72 milliseconds, briefly regresses, and + reaches a best result of 55 milliseconds. + + + {[30, 60, 90].map((y) => ( + + ))} + + + + + {EXPERIMENT_POINTS.map((point, index) => ( + + + + {point.value} + + + {index === 0 ? "Baseline" : index} + + + ))} + + + +
    + + improved + + + regressed + +
    +
    + ); +} + +function InfoRow({ + number, + title, + description, +}: { + number: string; + title: string; + description: string; +}) { + return ( +
    +
    + {number} +
    +
    + + {title} + + + {description} + +
    ); } @@ -160,14 +547,7 @@ function stageSummary( return effortLabel ? `${modelLabel} · ${effortLabel}` : modelLabel; } -/** - * Per-stage model and effort. While autoresearch is armed this popover is - * the composer's only model/effort control, so the trigger always shows a - * summary of what the run will use. Identical stages mean single-turn - * iterations; any difference splits each iteration into a build turn and a - * measure turn, switching between the stages. - */ -function StagesPopover({ +function AdvancedSettings({ draft, modelOptions, effortOptions, @@ -183,18 +563,7 @@ function StagesPopover({ const split = draft.implementModel !== draft.measureModel || draft.implementEffort !== draft.measureEffort; - const buildSummary = stageSummary( - draft.implementModel, - draft.implementEffort, - modelOptions, - effortOptions, - ); - const measureSummary = stageSummary( - draft.measureModel, - draft.measureEffort, - modelOptions, - effortOptions, - ); + const hasTarget = draft.targetValue !== null; return ( @@ -202,18 +571,60 @@ function StagesPopover({ - -
    + +
    +
    + + Advanced settings + + + Optional stopping and model controls. + +
    + +
    + + Stop early at this metric value + + { + const raw = event.target.value.trim(); + const numeric = Number(raw); + onChange({ + targetValue: + raw === "" || !Number.isFinite(numeric) ? null : numeric, + }); + }} + placeholder="Optional" + inputMode="decimal" + aria-label="Target metric value to stop at" + disabled={disabled} + /> +
    + onChange({ implementEffort: value })} /> onChange({ measureModel: value })} onEffortChange={(value) => onChange({ measureEffort: value })} /> - - Identical stages run each iteration as one turn. Different stages - split every iteration: build on the first, measure on the second — - pick a cheap model or low effort for measuring. - + + {split && ( + + Build:{" "} + {stageSummary( + draft.implementModel, + draft.implementEffort, + modelOptions, + effortOptions, + )} + . Measure:{" "} + {stageSummary( + draft.measureModel, + draft.measureEffort, + modelOptions, + effortOptions, + )} + . + + )}
    @@ -243,6 +670,7 @@ function StagesPopover({ function StageFields({ legend, + description, model, effort, modelOptions, @@ -251,6 +679,7 @@ function StageFields({ onEffortChange, }: { legend: string; + description: string; model: string | null; effort: string | null; modelOptions: AutoresearchModelOption[]; @@ -260,9 +689,12 @@ function StageFields({ }) { return (
    - + {legend} + + {description} +
    )} diff --git a/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx new file mode 100644 index 0000000000..8931d1cde0 --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchFullDashboard.stories.tsx @@ -0,0 +1,334 @@ +import type { + AutoresearchDirection, + AutoresearchIteration, + AutoresearchRun, + AutoresearchRunStatus, +} from "@posthog/core/autoresearch/schemas"; +import type { AcpMessage } from "@posthog/shared"; +import { Badge, Button, Flex, Text } from "@radix-ui/themes"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { AutoresearchObservability } from "./AutoresearchObservability"; +import { RunStats, RunSummary } from "./AutoresearchPanel"; +import { AutoresearchRuntimeStats } from "./AutoresearchRuntimeStats"; +import { IterationsTable } from "./IterationsTable"; +import { MetricChart } from "./MetricChart"; +import { PreBaselineState } from "./PreBaselineState"; + +interface FullDashboardStoryProps { + status: AutoresearchRunStatus; + direction: AutoresearchDirection; + iterationCount: number; + maxIterations: number; + baselineValue: number; + changePerIteration: number; + targetValue: number; + showTarget: boolean; + metricName: string; + metricUnit: string; + showResearch: boolean; + showActivity: boolean; + showContextUsage: boolean; + contextUsed: number; + contextSize: number; + hypothesis: string; + iterationPlan: string; + approach: string; +} + +const STORY_NOW = Date.now(); +const STARTED_AT = STORY_NOW - 26 * 60_000; + +function FullDashboardStory(props: FullDashboardStoryProps) { + const run = buildRun(props); + const events = buildEvents(props); + const usage = props.showContextUsage + ? { + used: props.contextUsed, + size: props.contextSize, + percentage: + props.contextSize > 0 + ? Math.round((props.contextUsed / props.contextSize) * 100) + : 0, + cost: null, + breakdown: null, + } + : null; + + return ( +
    + + + {run.iterations.length === 0 ? ( + + ) : ( + <> + + + + {run.endedAt !== null && } + + )} + + + +
    + ); +} + +function DashboardHeader({ run }: { run: AutoresearchRun }) { + return ( +
    +
    +
    + + Autoresearch + + {run.config.direction} + + {run.status} + +
    + + Agent is testing focused changes against{" "} + {run.metricName ?? "the metric"}. + +
    +
    + + +
    +
    + ); +} + +function buildRun(props: FullDashboardStoryProps): AutoresearchRun { + const ended = + props.status === "completed" || + props.status === "stopped" || + props.status === "failed"; + return { + id: "run-configurable-dashboard", + config: { + taskId: "task-1", + direction: props.direction, + targetValue: props.showTarget ? props.targetValue : null, + maxIterations: props.maxIterations, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: `Optimize ${props.metricName}.`, + }, + status: props.status, + metricName: props.metricName, + metricUnit: props.metricUnit || null, + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: props.showResearch + ? [ + { + index: 1, + area: "build", + summary: "Located the measurement path", + finding: + "The metric can be reproduced from production build output.", + nextStep: "Measure the baseline", + at: STARTED_AT + 60_000, + }, + { + index: 2, + area: "frontend", + summary: "Mapped the eager dependency graph", + finding: "Several modal modules load before the user opens them.", + nextStep: "Test a lazy loading boundary", + at: STARTED_AT + 3 * 60_000, + }, + ] + : [], + iterations: buildIterations(props), + startedAt: STARTED_AT, + endedAt: ended ? STORY_NOW : null, + endReason: + props.status === "completed" + ? "max-iterations" + : props.status === "stopped" + ? "stopped-by-user" + : null, + interruptedReason: props.status === "interrupted" ? "session-error" : null, + lastError: + props.status === "failed" + ? "The agent stopped reporting the metric." + : null, + }; +} + +function buildIterations( + props: FullDashboardStoryProps, +): AutoresearchIteration[] { + let best: number | null = null; + let previous: number | null = null; + return Array.from({ length: props.iterationCount }, (_, offset) => { + const index = offset + 1; + const directionMultiplier = props.direction === "minimize" ? -1 : 1; + const noise = + offset > 0 && offset % 3 === 0 ? props.changePerIteration * -0.35 : 0; + const value = + props.baselineValue + + directionMultiplier * props.changePerIteration * offset + + noise; + const improved = + best === null || + (props.direction === "minimize" ? value < best : value > best); + if (improved) best = value; + const iteration: AutoresearchIteration = { + index, + value, + bestValue: best ?? value, + delta: previous === null ? null : value - previous, + summary: + index === 1 + ? "Established the baseline" + : index % 3 === 1 + ? "Measured a regression and changed direction" + : `Completed focused experiment ${index}`, + hypothesis: + index === 1 + ? "The baseline captures the current production behavior" + : props.hypothesis, + plan: + index === 1 + ? "Run the reproducible baseline measurement" + : props.iterationPlan, + approach: index === 1 ? "baseline" : props.approach, + at: STARTED_AT + index * 3 * 60_000, + }; + previous = value; + return iteration; + }); +} + +function buildEvents(props: FullDashboardStoryProps): AcpMessage[] { + if (!props.showActivity) return []; + return [ + sessionEvent(STARTED_AT + 12 * 60_000, { + sessionUpdate: "agent_message_chunk", + content: { + type: "text", + text: `\`\`\`autoresearch\ntype: plan\nhypothesis: ${props.hypothesis}\nplan: ${props.iterationPlan}\napproach: ${props.approach}\n\`\`\``, + }, + }), + sessionEvent(STARTED_AT + 13 * 60_000, { + sessionUpdate: "tool_call", + title: "Search relevant modules", + kind: "search", + status: "completed", + }), + sessionEvent(STARTED_AT + 17 * 60_000, { + sessionUpdate: "tool_call", + title: "Edit the selected implementation", + kind: "edit", + status: "completed", + }), + sessionEvent(STARTED_AT + 22 * 60_000, { + sessionUpdate: "tool_call", + title: "Run the metric measurement", + kind: "execute", + status: props.status === "running" ? "in_progress" : "completed", + }), + ]; +} + +function sessionEvent(ts: number, update: Record): AcpMessage { + return { + type: "acp_message", + ts, + message: { + jsonrpc: "2.0", + method: "session/update", + params: { update }, + }, + } as AcpMessage; +} + +const meta = { + title: "Autoresearch/Configurable Full Dashboard", + component: FullDashboardStory, + parameters: { layout: "fullscreen" }, + argTypes: { + status: { + control: "select", + options: [ + "running", + "paused", + "interrupted", + "completed", + "stopped", + "failed", + ], + }, + direction: { control: "inline-radio", options: ["minimize", "maximize"] }, + iterationCount: { control: { type: "range", min: 0, max: 12, step: 1 } }, + maxIterations: { control: { type: "range", min: 1, max: 20, step: 1 } }, + baselineValue: { control: { type: "number", step: 10 } }, + changePerIteration: { control: { type: "number", step: 1 } }, + targetValue: { control: { type: "number", step: 10 } }, + contextUsed: { control: { type: "number", step: 1_000 } }, + contextSize: { control: { type: "number", step: 10_000 } }, + }, + args: { + status: "running", + direction: "minimize", + iterationCount: 4, + maxIterations: 10, + baselineValue: 3850.7, + changePerIteration: 145, + targetValue: 3000, + showTarget: true, + metricName: "dashboard bundle", + metricUnit: "KiB", + showResearch: true, + showActivity: true, + showContextUsage: true, + contextUsed: 175_000, + contextSize: 1_000_000, + hypothesis: "Eager modal imports dominate the dashboard entry bundle", + iterationPlan: + "Lazy load dashboard modals and rerun the production bundle measurement", + approach: "code splitting", + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Playground: Story = {}; + +export const BeforeBaseline: Story = { + args: { iterationCount: 0, showResearch: true, status: "running" }, +}; + +export const CompletedRun: Story = { + args: { status: "completed", iterationCount: 8 }, +}; diff --git a/packages/ui/src/features/autoresearch/AutoresearchHeaderButton.tsx b/packages/ui/src/features/autoresearch/AutoresearchHeaderButton.tsx index 6a7b2a75df..2300b10570 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchHeaderButton.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchHeaderButton.tsx @@ -18,7 +18,7 @@ interface AutoresearchHeaderButtonProps { /** * Task-header shortcut to the autoresearch dashboard. Only rendered for - * tasks that have a run — autoresearch tasks are created from the composer, + * tasks that have a run. Autoresearch tasks are created from the composer, * not retrofitted onto existing tasks. */ export function AutoresearchHeaderButton({ diff --git a/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx new file mode 100644 index 0000000000..4e826d19b7 --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchObservability.stories.tsx @@ -0,0 +1,93 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { AcpMessage } from "@posthog/shared"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { AutoresearchObservability } from "./AutoresearchObservability"; + +const STARTED_AT = Date.parse("2026-07-10T14:00:00Z"); + +const run: AutoresearchRun = { + id: "run-observability", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: 300, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce dashboard loading time.", + }, + status: "running", + metricName: "dashboard bundle", + metricUnit: "KiB", + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [], + iterations: [], + startedAt: STARTED_AT, + endedAt: STARTED_AT + 10 * 60_000, + endReason: null, + interruptedReason: null, + lastError: null, +}; + +function event(ts: number, update: Record): AcpMessage { + return { + type: "acp_message", + ts, + message: { + jsonrpc: "2.0", + method: "session/update", + params: { update }, + }, + } as AcpMessage; +} + +const events = [ + event(STARTED_AT + 30_000, { + sessionUpdate: "agent_message_chunk", + content: { + type: "text", + text: "```autoresearch\ntype: plan\nhypothesis: eager modal imports dominate the dashboard entry bundle\nplan: lazy load dashboard modals and rerun the bundle measurement\napproach: code splitting\n```", + }, + }), + event(STARTED_AT + 60_000, { + sessionUpdate: "tool_call", + title: "Search dashboard imports", + kind: "search", + status: "completed", + }), + event(STARTED_AT + 3 * 60_000, { + sessionUpdate: "tool_call", + title: "Edit DashboardScene", + kind: "edit", + status: "completed", + }), + event(STARTED_AT + 7 * 60_000, { + sessionUpdate: "tool_call", + title: "Measure dashboard bundle", + kind: "execute", + status: "in_progress", + }), +]; + +const meta: Meta = { + title: "Autoresearch/Observability", + component: AutoresearchObservability, + decorators: [ + (Story) => ( +
    + +
    + ), + ], +}; + +export default meta; +type Story = StoryObj; + +export const ActiveExperiment: Story = { args: { run, events } }; + +export const WaitingForPlan: Story = { args: { run, events: [] } }; diff --git a/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx b/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx new file mode 100644 index 0000000000..f047768fff --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchObservability.tsx @@ -0,0 +1,182 @@ +import { + Binoculars, + Code, + Compass, + Lightbulb, + ListChecks, + MagnifyingGlass, + TestTube, +} from "@phosphor-icons/react"; +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { AcpMessage } from "@posthog/shared"; +import { Badge, Progress, Text } from "@radix-ui/themes"; +import { useEffect, useMemo, useState } from "react"; +import { formatDuration } from "../sessions/components/GeneratingIndicator"; +import { + type AutoresearchActivityKind, + analyzeAutoresearchActivity, +} from "./autoresearchActivity"; + +export function AutoresearchObservability({ + run, + events, +}: { + run: AutoresearchRun; + events: AcpMessage[]; +}) { + const now = useLiveNow(run.endedAt === null); + const activity = useMemo( + () => analyzeAutoresearchActivity(events, run.startedAt, run.endedAt, now), + [events, now, run.endedAt, run.startedAt], + ); + const lastIteration = run.iterations.at(-1); + const hypothesis = + activity.currentPlan?.hypothesis ?? lastIteration?.hypothesis; + const plan = activity.currentPlan?.plan ?? lastIteration?.plan; + const approach = activity.currentPlan?.approach ?? lastIteration?.approach; + + return ( +
    +
    + } + title="Current experiment" + /> + + + {approach && ( +
    + + {approach} + +
    + )} +
    + +
    + } title="Observed time" /> +
    + {(Object.keys(activity.timeByKind) as AutoresearchActivityKind[]).map( + (kind) => ( + + ), + )} +
    +
    + +
    + } title="Live activity" /> + {activity.items.length === 0 ? ( + + Waiting for the first observable tool action. + + ) : ( +
      + {activity.items.map((item) => ( +
    1. + + + {item.label} + + {item.active && Active} +
    2. + ))} +
    + )} +
    +
    + ); +} + +function SectionTitle({ + icon, + title, +}: { + icon: React.ReactNode; + title: string; +}) { + return ( +
    + {icon} + + {title} + +
    + ); +} + +function Detail({ label, value }: { label: string; value: string }) { + return ( +
    + + {label} + + + {value} + +
    + ); +} + +const TIME_LABEL: Record = { + research: "Research", + implementation: "Implementation", + measurement: "Commands and measurement", + reasoning: "Reasoning and coordination", +}; + +function TimeRow({ + kind, + value, + total, +}: { + kind: AutoresearchActivityKind; + value: number; + total: number; +}) { + const percentage = Math.round((value / total) * 100); + return ( +
    +
    + {TIME_LABEL[kind]} + + {formatDuration(value, 0)} + +
    + +
    + ); +} + +function ActivityIcon({ kind }: { kind: AutoresearchActivityKind }) { + if (kind === "research") return ; + if (kind === "implementation") return ; + if (kind === "measurement") return ; + return ; +} + +function useLiveNow(live: boolean): number { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!live) return; + const interval = window.setInterval(() => setNow(Date.now()), 1_000); + return () => window.clearInterval(interval); + }, [live]); + return now; +} diff --git a/packages/ui/src/features/autoresearch/AutoresearchPanel.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchPanel.stories.tsx index cc20962a30..990c525bc7 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchPanel.stories.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchPanel.stories.tsx @@ -5,15 +5,33 @@ import type { import { Flex } from "@radix-ui/themes"; import type { Meta, StoryObj } from "@storybook/react-vite"; import { RunStats } from "./AutoresearchPanel"; +import { AutoresearchRuntimeStats } from "./AutoresearchRuntimeStats"; import { IterationsTable } from "./IterationsTable"; import { MetricChart } from "./MetricChart"; +import { PreBaselineState } from "./PreBaselineState"; /** - * The run dashboard's presentational stack — stat cards, metric chart, and - * iterations table — as `AutoresearchPanel` composes them, minus the header + * The run dashboard presentational stack includes stat cards, the metric + * chart, and the iterations table. It matches `AutoresearchPanel` without header * controls and dialogs (which need a live service). */ function RunDashboard({ run }: { run: AutoresearchRun }) { + if (run.iterations.length === 0) { + return ( + + + + + ); + } + return ( @@ -24,6 +42,16 @@ function RunDashboard({ run }: { run: AutoresearchRun }) { metricName={run.metricName ?? "the metric"} unit={run.metricUnit} /> + = {}): AutoresearchRun => ({ phase: null, originalModel: null, originalEffort: null, + researchFindings: [], iterations: [], startedAt: BASE_AT, endedAt: null, @@ -101,7 +130,7 @@ const meta: Meta = { export default meta; type Story = StoryObj; -/** A minimize run mid-flight: noisy progress, best-so-far frontier, target line. */ +/** A minimize run in progress with noisy results, the best frontier, and a target. */ export const MinimizeInProgress: Story = { args: { run: run({ @@ -123,7 +152,7 @@ export const MinimizeInProgress: Story = { }, }; -/** A completed maximize run with no target — the loop spent its budget. */ +/** A completed maximize run with no target. The loop spent its budget. */ export const MaximizeCompleted: Story = { args: { run: run({ @@ -158,7 +187,7 @@ export const MaximizeCompleted: Story = { }, }; -/** Before the first metric report arrives: empty cards, chart, and table. */ +/** Before the first metric report arrives: active baseline measurement. */ export const NoIterationsYet: Story = { args: { run: run() }, }; diff --git a/packages/ui/src/features/autoresearch/AutoresearchPanel.test.tsx b/packages/ui/src/features/autoresearch/AutoresearchPanel.test.tsx new file mode 100644 index 0000000000..5de566d94d --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchPanel.test.tsx @@ -0,0 +1,131 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import { Theme } from "@radix-ui/themes"; +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { PreBaselineState } from "./PreBaselineState"; + +function makeRun(overrides: Partial = {}): AutoresearchRun { + return { + id: "run-1", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: null, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce memory usage.", + }, + status: "running", + metricName: null, + metricUnit: null, + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [], + iterations: [], + startedAt: 0, + endedAt: null, + endReason: null, + interruptedReason: null, + lastError: null, + ...overrides, + }; +} + +function renderState( + run: AutoresearchRun, + sessionActivity: React.ComponentProps< + typeof PreBaselineState + >["sessionActivity"], +) { + return render( + + + , + ); +} + +describe("PreBaselineState", () => { + it("shows active baseline progress and loading dashboard placeholders", () => { + renderState(makeRun(), { + status: "connected", + isPromptPending: true, + isCompacting: false, + }); + + expect(screen.getByText("Establishing the baseline")).toBeVisible(); + expect( + screen.getByRole("status", { name: "Loading autoresearch metrics" }), + ).toBeVisible(); + expect(screen.getByText("0 / 10")).toBeVisible(); + }); + + it("surfaces codebase research while the baseline is pending", () => { + renderState( + makeRun({ + researchFindings: [ + { + index: 1, + summary: "Mapped the execution path", + finding: "The metric is computed in the workspace server.", + nextStep: "Trace the benchmark command", + area: "workspace server", + at: 1, + }, + ], + }), + { + status: "connected", + isPromptPending: true, + isCompacting: false, + }, + ); + + expect(screen.getByText("Researching the codebase")).toBeVisible(); + expect(screen.getByText("Codebase research")).toBeVisible(); + expect(screen.getByText("Mapped the execution path")).toBeVisible(); + expect( + screen.getByText("The metric is computed in the workspace server."), + ).toBeVisible(); + expect(screen.getByText("Next: Trace the benchmark command")).toBeVisible(); + const metrics = screen.getByRole("status", { + name: "Loading autoresearch metrics", + }); + const research = screen.getByText("Codebase research"); + expect( + metrics.compareDocumentPosition(research) & + Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); + }); + + it("explains when baseline collection is paused", () => { + renderState(makeRun({ status: "paused" }), null); + + expect(screen.getByText("Baseline measurement paused")).toBeVisible(); + expect( + screen.queryByRole("status", { name: "Loading autoresearch metrics" }), + ).not.toBeInTheDocument(); + }); + + it("does not imply loading after a run ends without a report", () => { + renderState( + makeRun({ + status: "failed", + endReason: "missing-report", + endedAt: 1, + lastError: "The agent did not report a metric.", + }), + null, + ); + + expect( + screen.getByText("Run ended before the baseline was reported"), + ).toBeVisible(); + expect( + screen.getByText("No metric report was recorded for this run."), + ).toBeVisible(); + }); +}); diff --git a/packages/ui/src/features/autoresearch/AutoresearchPanel.tsx b/packages/ui/src/features/autoresearch/AutoresearchPanel.tsx index a0571f5273..1433bfc546 100644 --- a/packages/ui/src/features/autoresearch/AutoresearchPanel.tsx +++ b/packages/ui/src/features/autoresearch/AutoresearchPanel.tsx @@ -15,17 +15,22 @@ import { EmptyTitle, } from "@posthog/quill"; import { MetricCard } from "@posthog/quill-charts"; +import type { AcpMessage } from "@posthog/shared"; import { Badge, Button, Callout, Flex, Select, Text } from "@radix-ui/themes"; import { useEffect, useMemo, useState } from "react"; +import { useContextUsage } from "../sessions/hooks/useContextUsage"; import { getConfigOptionByCategory, useSessionStore, } from "../sessions/sessionStore"; import { usePendingPermissionsForTask } from "../sessions/useSession"; import { AutoresearchConfigDialog } from "./AutoresearchConfigDialog"; +import { AutoresearchObservability } from "./AutoresearchObservability"; +import { AutoresearchRuntimeStats } from "./AutoresearchRuntimeStats"; import { IterationsTable } from "./IterationsTable"; import { MetricChart } from "./MetricChart"; import { metricNumberFormat, withMetricUnit } from "./metricFormat"; +import { PreBaselineState } from "./PreBaselineState"; import { type AutoresearchModelOption, stageValueLabel, @@ -34,6 +39,8 @@ import { import { useAutoresearchEnabled } from "./useAutoresearchEnabled"; import { useAutoresearchRuns } from "./useAutoresearchStore"; +const EMPTY_SESSION_EVENTS: AcpMessage[] = []; + const STATUS_BADGE: Record< AutoresearchRunStatus, { @@ -92,6 +99,24 @@ export function AutoresearchPanel({ taskId }: AutoresearchPanelProps) { const session = taskRunId ? state.sessions[taskRunId] : undefined; return getConfigOptionByCategory(session?.configOptions, "thought_level"); }); + const sessionActivity = useSessionStore((state) => { + const taskRunId = state.taskIdIndex[taskId]; + const session = taskRunId ? state.sessions[taskRunId] : undefined; + return session + ? { + status: session.status, + isPromptPending: session.isPromptPending, + isCompacting: session.isCompacting, + } + : null; + }); + const sessionEvents = useSessionStore((state) => { + const taskRunId = state.taskIdIndex[taskId]; + return taskRunId + ? (state.sessions[taskRunId]?.events ?? EMPTY_SESSION_EVENTS) + : EMPTY_SESSION_EVENTS; + }); + const contextUsage = useContextUsage(sessionEvents); const modelOptions = useMemo( () => toStageSelectOptions(modelOption), [modelOption], @@ -100,7 +125,7 @@ export function AutoresearchPanel({ taskId }: AutoresearchPanelProps) { () => toStageSelectOptions(thoughtOption), [thoughtOption], ); - // What the session is actually on right now — the loop switches these + // What the session is actually on right now. The loop switches these // between stages, and this reflects the switches live. const liveModel = modelOption?.type === "select" ? (modelOption.currentValue ?? null) : null; @@ -113,6 +138,13 @@ export function AutoresearchPanel({ taskId }: AutoresearchPanelProps) { const selectedRun = (selectedRunId && runs.find((run) => run.id === selectedRunId)) || latestRun; + const selectedRunUsage = + selectedRun?.id === latestRun?.id && + (selectedRun?.status === "running" || + selectedRun?.status === "paused" || + selectedRun?.status === "interrupted") + ? contextUsage + : null; // A persisted panel tab can outlive access to the feature (web, or the // flag turned off). With runs already in the store, keep the dashboard @@ -143,7 +175,7 @@ export function AutoresearchPanel({ taskId }: AutoresearchPanelProps) { No autoresearch run This task wasn't created in autoresearch mode. Start one from the - new-task composer: arm the Autoresearch toggle, describe what to + new task composer: arm the Autoresearch toggle, describe what to optimize and how to measure it, and submit. @@ -166,19 +198,31 @@ export function AutoresearchPanel({ taskId }: AutoresearchPanelProps) { onNewRun={() => setDialogOpen(true)} /> - - - + {selectedRun.iterations.length === 0 ? ( + + ) : ( + <> + + + + {selectedRun.endedAt !== null && } + + )} + + {runs.map((candidate, index) => ( - Run {index + 1} — {STATUS_BADGE[candidate.status].label} + Run {index + 1}: {STATUS_BADGE[candidate.status].label} ))} @@ -327,21 +371,21 @@ function RunHeader({ {liveEffort ? ` · ${stageValueLabel(liveEffort, effortOptions) ?? liveEffort} effort` : ""} - {isSplit && run.phase ? ` — ${run.phase} phase` : ""} + {isSplit && run.phase ? ` (${run.phase} phase)` : ""}
    )} {run.status === "interrupted" && ( {INTERRUPTION_LABEL[run.interruptedReason ?? ""] ?? "Loop interrupted"} - {run.lastError ? ` — ${run.lastError}` : ""}. Resumes automatically; + {run.lastError ? `. ${run.lastError}` : ""}. Resumes automatically; Resume retries now. )} {run.endReason && ( {END_REASON_LABEL[run.endReason] ?? run.endReason} - {run.lastError ? ` — ${run.lastError}` : ""} + {run.lastError ? `. ${run.lastError}` : ""} )} @@ -380,7 +424,7 @@ export function RunStats({ run }: { run: AutoresearchRun }) { const unit = run.metricUnit; const formatMetricValue = (value: number) => Number.isNaN(value) - ? "—" + ? "Not available" : withMetricUnit(metricNumberFormat.format(value), unit); const cardClassName = "rounded-md border border-(--gray-5) p-3"; @@ -423,6 +467,49 @@ export function RunStats({ run }: { run: AutoresearchRun }) { ); } +export function RunSummary({ run }: { run: AutoresearchRun }) { + const summary = summarizeRun(run); + const bestIteration = run.iterations.find( + (iteration) => iteration.index === summary.best?.index, + ); + const approaches = Array.from( + new Set( + run.iterations.flatMap((iteration) => + iteration.approach ? [iteration.approach] : [], + ), + ), + ); + return ( +
    + + Run summary + + + {summary.best + ? `Best result was iteration ${summary.best.index} at ${withMetricUnit(metricNumberFormat.format(summary.best.value), run.metricUnit)}.` + : "The run ended without a metric result."} + {summary.improvementFromBaseline !== null + ? ` Change from baseline: ${withMetricUnit(metricNumberFormat.format(summary.improvementFromBaseline), run.metricUnit)}.` + : ""} + + {bestIteration?.hypothesis && ( + + Best hypothesis: {bestIteration.hypothesis} + + )} + {approaches.length > 0 && ( +
    + {approaches.map((approach) => ( + + {approach} + + ))} +
    + )} +
    + ); +} + function stageText( model: string | null, effort: string | null, diff --git a/packages/ui/src/features/autoresearch/AutoresearchResults.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchResults.stories.tsx new file mode 100644 index 0000000000..00759d8cca --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchResults.stories.tsx @@ -0,0 +1,110 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import { Flex } from "@radix-ui/themes"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { RunStats, RunSummary } from "./AutoresearchPanel"; +import { IterationsTable } from "./IterationsTable"; + +const STARTED_AT = Date.parse("2026-07-10T14:00:00Z"); +const run: AutoresearchRun = { + id: "run-results", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: 300, + maxIterations: 3, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce dashboard loading time.", + }, + status: "completed", + metricName: "dashboard bundle", + metricUnit: "KiB", + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [], + iterations: [ + { + index: 1, + value: 3850.7, + bestValue: 3850.7, + delta: null, + summary: "Established the baseline", + hypothesis: "The eager graph defines the current cost", + plan: "Measure the marginal dashboard bundle", + approach: "baseline", + at: STARTED_AT + 5 * 60_000, + }, + { + index: 2, + value: 3320.4, + bestValue: 3320.4, + delta: -530.3, + summary: "Lazy loaded dashboard modals", + hypothesis: "Modal imports dominate the eager route", + plan: "Split modal imports and remeasure", + approach: "code splitting", + at: STARTED_AT + 15 * 60_000, + }, + { + index: 3, + value: 3208.1, + bestValue: 3208.1, + delta: -112.3, + summary: "Deferred the insight editor", + hypothesis: "The editor is unused during initial dashboard render", + plan: "Load the editor when add insight opens", + approach: "lazy loading", + at: STARTED_AT + 25 * 60_000, + }, + ], + startedAt: STARTED_AT, + endedAt: STARTED_AT + 26 * 60_000, + endReason: "max-iterations", + interruptedReason: null, + lastError: null, +}; + +function Results({ run }: { run: AutoresearchRun }) { + return ( + + + + + + ); +} + +const meta: Meta = { + title: "Autoresearch/Results and Summary", + component: Results, + decorators: [ + (Story) => ( +
    + +
    + ), + ], +}; + +export default meta; +type Story = StoryObj; + +export const CompletedRun: Story = { args: { run } }; + +export const BaselineJustRecorded: Story = { + args: { + run: { + ...run, + status: "running", + endedAt: null, + iterations: [run.iterations[0]], + }, + }, +}; diff --git a/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.stories.tsx b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.stories.tsx new file mode 100644 index 0000000000..ef41f6f41d --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.stories.tsx @@ -0,0 +1,55 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { AutoresearchRuntimeStats } from "./AutoresearchRuntimeStats"; + +const STARTED_AT = Date.parse("2026-07-10T14:00:00Z"); +const run: AutoresearchRun = { + id: "run-runtime", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: null, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce dashboard loading time.", + }, + status: "completed", + metricName: null, + metricUnit: null, + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [], + iterations: [], + startedAt: STARTED_AT, + endedAt: STARTED_AT + 26 * 60_000 + 17_000, + endReason: "max-iterations", + interruptedReason: null, + lastError: null, +}; + +const meta: Meta = { + title: "Autoresearch/Runtime Stats", + component: AutoresearchRuntimeStats, +}; + +export default meta; +type Story = StoryObj; + +export const WithContextUsage: Story = { + args: { + run, + usage: { + used: 175_000, + size: 1_000_000, + percentage: 18, + cost: null, + breakdown: null, + }, + }, +}; + +export const BeforeUsageUpdate: Story = { args: { run, usage: null } }; diff --git a/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.test.tsx b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.test.tsx new file mode 100644 index 0000000000..6e21a313fa --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.test.tsx @@ -0,0 +1,75 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import { Theme } from "@radix-ui/themes"; +import { render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { AutoresearchRuntimeStats } from "./AutoresearchRuntimeStats"; + +function makeRun(overrides: Partial = {}): AutoresearchRun { + return { + id: "run-1", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: null, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce memory usage.", + }, + status: "running", + metricName: null, + metricUnit: null, + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [], + iterations: [], + startedAt: 1_000, + endedAt: null, + endReason: null, + interruptedReason: null, + lastError: null, + ...overrides, + }; +} + +function renderStats( + run: AutoresearchRun, + usage: React.ComponentProps["usage"], +) { + return render( + + + , + ); +} + +describe("AutoresearchRuntimeStats", () => { + afterEach(() => vi.useRealTimers()); + + it("shows elapsed time and live usage reported by the runtime", () => { + vi.useFakeTimers(); + vi.setSystemTime(66_000); + renderStats(makeRun(), { + used: 42_800, + size: 200_000, + percentage: 21, + cost: { amount: 1.284, currency: "USD" }, + breakdown: null, + }); + + expect(screen.getByText("1m 05s")).toBeVisible(); + expect(screen.getByText("43K / 200K")).toBeVisible(); + expect(screen.getByText("21% of current window")).toBeVisible(); + }); + + it("uses the end time and explains unavailable usage", () => { + renderStats(makeRun({ endedAt: 31_000, status: "completed" }), null); + + expect(screen.getByText("30s")).toBeVisible(); + expect(screen.getByText("Waiting")).toBeVisible(); + expect(screen.getByText("Total run time")).toBeVisible(); + }); +}); diff --git a/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.tsx b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.tsx new file mode 100644 index 0000000000..1e2d040373 --- /dev/null +++ b/packages/ui/src/features/autoresearch/AutoresearchRuntimeStats.tsx @@ -0,0 +1,88 @@ +import { Clock, Gauge } from "@phosphor-icons/react"; +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { ContextUsage } from "@posthog/core/sessions/contextUsage"; +import { Text } from "@radix-ui/themes"; +import { useEffect, useState } from "react"; +import { formatDuration } from "../sessions/components/GeneratingIndicator"; +import { formatTokensCompact } from "../sessions/contextColors"; + +export function AutoresearchRuntimeStats({ + run, + usage, +}: { + run: AutoresearchRun; + usage: ContextUsage | null; +}) { + const elapsed = useRunElapsed(run); + const contextValue = usage + ? usage.size > 0 + ? `${formatTokensCompact(usage.used)} / ${formatTokensCompact(usage.size)}` + : formatTokensCompact(usage.used) + : "Waiting"; + const contextDetail = usage + ? usage.size > 0 + ? `${usage.percentage}% of current window` + : "Current context" + : "No usage update yet"; + return ( +
    + } + label="Elapsed" + value={formatDuration(elapsed, 0)} + detail={ + run.endedAt === null ? "Since this run started" : "Total run time" + } + /> + } + label="Context tokens" + value={contextValue} + detail={contextDetail} + /> +
    + ); +} + +function RuntimeMetric({ + icon, + label, + value, + detail, +}: { + icon: React.ReactNode; + label: string; + value: string; + detail: string; +}) { + return ( +
    +
    + {icon} + {label} +
    + + {value} + + + {detail} + +
    + ); +} + +function useRunElapsed(run: AutoresearchRun): number { + const [now, setNow] = useState(() => Date.now()); + const live = run.endedAt === null; + + useEffect(() => { + if (!live) return; + const interval = window.setInterval(() => setNow(Date.now()), 1_000); + return () => window.clearInterval(interval); + }, [live]); + + return Math.max(0, (run.endedAt ?? now) - run.startedAt); +} diff --git a/packages/ui/src/features/autoresearch/IterationsTable.tsx b/packages/ui/src/features/autoresearch/IterationsTable.tsx index e964a4dd75..9193fd5973 100644 --- a/packages/ui/src/features/autoresearch/IterationsTable.tsx +++ b/packages/ui/src/features/autoresearch/IterationsTable.tsx @@ -87,13 +87,20 @@ export function IterationsTable({ - - {iteration.summary ?? "—"} - +
    + {iteration.approach && ( + + {iteration.approach} + + )} + + {iteration.summary ?? "None"} + +
    diff --git a/packages/ui/src/features/autoresearch/MetricChart.tsx b/packages/ui/src/features/autoresearch/MetricChart.tsx index d20bc3a7b6..94caafcd22 100644 --- a/packages/ui/src/features/autoresearch/MetricChart.tsx +++ b/packages/ui/src/features/autoresearch/MetricChart.tsx @@ -21,7 +21,7 @@ interface MetricChartProps { } /** - * Metric value per iteration (solid, with dots) plus the best-so-far + * Metric value per iteration, shown as a solid line with dots, plus the best * frontier (dashed) and the optional target line. */ export function MetricChart({ @@ -32,7 +32,7 @@ export function MetricChart({ unit, }: MetricChartProps) { const theme = useChartTheme(); - // Canvas colors must be concrete — `var(--…)` strings don't paint. The + // Canvas colors must be concrete because CSS variable strings do not paint. The // theme's palette is already resolved from CSS variables. const valueColor = theme.colors[0] ?? "#1d4aff"; const bestColor = theme.axisColor ?? "#8b8d98"; @@ -89,7 +89,7 @@ export function MetricChart({ showCrosshair: true, yTickFormatter: (value) => withMetricUnit(formatChartValue(value), unit), - // Keep an off-scale target visible instead of clipping it. + // Keep a target outside the current scale visible instead of clipping it. valueDomain: targetValue === null ? undefined : { include: [targetValue] }, }} diff --git a/packages/ui/src/features/autoresearch/PreBaselineState.stories.tsx b/packages/ui/src/features/autoresearch/PreBaselineState.stories.tsx new file mode 100644 index 0000000000..337e3be50d --- /dev/null +++ b/packages/ui/src/features/autoresearch/PreBaselineState.stories.tsx @@ -0,0 +1,79 @@ +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { PreBaselineState } from "./PreBaselineState"; + +const run: AutoresearchRun = { + id: "run-research", + config: { + taskId: "task-1", + direction: "minimize", + targetValue: null, + maxIterations: 10, + implementModel: null, + measureModel: null, + implementEffort: null, + measureEffort: null, + instructions: "Reduce dashboard loading time.", + }, + status: "running", + metricName: null, + metricUnit: null, + phase: null, + originalModel: null, + originalEffort: null, + researchFindings: [ + { + index: 1, + area: "build", + summary: "Located the production bundle measurement", + finding: + "The dashboard marginal bundle can be isolated from esbuild metadata.", + nextStep: "Establish the baseline bundle size", + at: 1, + }, + { + index: 2, + area: "frontend", + summary: "Mapped eager dashboard imports", + finding: "Dashboard modals load before users open them.", + nextStep: "Inspect modal boundaries", + at: 2, + }, + ], + iterations: [], + startedAt: Date.parse("2026-07-10T14:00:00Z"), + endedAt: null, + endReason: null, + interruptedReason: null, + lastError: null, +}; + +const meta: Meta = { + title: "Autoresearch/Research Map", + component: PreBaselineState, +}; + +export default meta; +type Story = StoryObj; + +export const FindingsByCodeArea: Story = { + args: { + run, + sessionActivity: { + status: "connected", + isPromptPending: true, + isCompacting: false, + }, + }, +}; + +export const EstablishingBaseline: Story = { + args: { + run: { ...run, researchFindings: [] }, + sessionActivity: { + status: "connected", + isPromptPending: true, + isCompacting: false, + }, + }, +}; diff --git a/packages/ui/src/features/autoresearch/PreBaselineState.tsx b/packages/ui/src/features/autoresearch/PreBaselineState.tsx new file mode 100644 index 0000000000..0b4782a5a4 --- /dev/null +++ b/packages/ui/src/features/autoresearch/PreBaselineState.tsx @@ -0,0 +1,224 @@ +import { MagnifyingGlass } from "@phosphor-icons/react"; +import type { AutoresearchRun } from "@posthog/core/autoresearch/schemas"; +import { Badge, Skeleton, Spinner, Text } from "@radix-ui/themes"; + +export interface SessionActivity { + status: "connecting" | "connected" | "disconnected" | "error"; + isPromptPending: boolean; + isCompacting: boolean; +} + +export function PreBaselineState({ + run, + sessionActivity, +}: { + run: AutoresearchRun; + sessionActivity: SessionActivity | null; +}) { + const live = run.status === "running" || run.status === "interrupted"; + const activity = baselineActivity(run, sessionActivity); + + return ( +
    +
    + {live && } +
    + + {activity.title} + + + {activity.description} + +
    +
    + + {live ? ( + + ) : ( +
    + + No metric report was recorded for this run. + +
    + )} + + {run.researchFindings.length > 0 && } +
    + ); +} + +function baselineActivity( + run: AutoresearchRun, + session: SessionActivity | null, +): { title: string; description: string } { + if (run.status === "paused") { + return { + title: "Baseline measurement paused", + description: "Resume the run to collect the first metric value.", + }; + } + if (run.status === "interrupted") { + return { + title: "Reconnecting before baseline measurement", + description: + "Autoresearch will resume automatically when the agent is available.", + }; + } + if (run.status !== "running") { + return { + title: "Run ended before the baseline was reported", + description: run.lastError ?? "The agent did not return a metric value.", + }; + } + if (!session || session.status === "connecting") { + return { + title: "Connecting to the agent", + description: + "The baseline measurement starts when the task session is ready.", + }; + } + if (session.isCompacting) { + return { + title: "Preparing agent context", + description: + "The baseline measurement will continue after context compaction.", + }; + } + if (session.isPromptPending) { + if (run.researchFindings.length > 0) { + const latest = run.researchFindings[run.researchFindings.length - 1]; + return { + title: "Researching the codebase", + description: + latest?.nextStep ?? + "The agent is continuing its investigation before measuring the baseline.", + }; + } + return { + title: "Establishing the baseline", + description: + "The agent is running the measurement defined in the task prompt.", + }; + } + return { + title: "Waiting for the first metric report", + description: + "The dashboard will populate when the agent reports the baseline value.", + }; +} + +function ResearchFindings({ run }: { run: AutoresearchRun }) { + const groups = Map.groupBy( + run.researchFindings, + (finding) => finding.area ?? "General", + ); + return ( +
    +
    + + + Codebase research + + + {run.researchFindings.length}{" "} + {run.researchFindings.length === 1 ? "finding" : "findings"} + +
    +
    + {Array.from(groups.entries()).map(([area, findings]) => ( +
    +
    + + {area} + + + {findings.length} + +
    +
      + {findings.map((finding) => ( +
    1. + + {finding.summary} + + + {finding.finding} + + {finding.nextStep && ( + + Next: {finding.nextStep} + + )} +
    2. + ))} +
    +
    + ))} +
    +
    + ); +} + +function BaselineDashboardSkeleton({ + maxIterations, +}: { + maxIterations: number; +}) { + return ( + +
    + {[ + ["Best", "w-16"], + ["Last", "w-16"], + ["Iterations", "w-12"], + ["Target", "w-14"], + ].map(([title, width]) => ( +
    + + {title} + + {title === "Iterations" ? ( + + 0 / {maxIterations} + + ) : ( + + )} +
    + ))} +
    + +
    +
    + + + +
    +
    + {["h-8", "h-12", "h-16", "h-20", "h-24"].map((height) => ( + + ))} +
    +
    + +
    +
    + + + + +
    +
    +
    + ); +} diff --git a/packages/ui/src/features/autoresearch/autoresearchActivity.test.ts b/packages/ui/src/features/autoresearch/autoresearchActivity.test.ts new file mode 100644 index 0000000000..5aa03940dc --- /dev/null +++ b/packages/ui/src/features/autoresearch/autoresearchActivity.test.ts @@ -0,0 +1,99 @@ +import type { AcpMessage } from "@posthog/shared"; +import { describe, expect, it } from "vitest"; +import { analyzeAutoresearchActivity } from "./autoresearchActivity"; + +function updateEvent(ts: number, update: Record): AcpMessage { + return { + type: "acp_message", + ts, + message: { + jsonrpc: "2.0", + method: "session/update", + params: { update }, + }, + } as AcpMessage; +} + +describe("analyzeAutoresearchActivity", () => { + it("extracts the current plan and classifies observable work", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "agent_message_chunk", + content: { + type: "text", + text: "```autoresearch\ntype: plan\nhypothesis: selectors cause rerenders\nplan: memoize selectors and benchmark\napproach: rendering\n```", + }, + }), + updateEvent(3_000, { + sessionUpdate: "tool_call", + title: "Search for selectors", + kind: "search", + status: "completed", + }), + updateEvent(5_000, { + sessionUpdate: "tool_call", + title: "Edit selector module", + kind: "edit", + status: "completed", + }), + updateEvent(8_000, { + sessionUpdate: "tool_call", + title: "Run benchmark", + kind: "execute", + status: "in_progress", + }), + ]; + + const result = analyzeAutoresearchActivity(events, 1_000, null, 11_000); + + expect(result.currentPlan).toEqual({ + hypothesis: "selectors cause rerenders", + plan: "memoize selectors and benchmark", + approach: "rendering", + }); + expect(result.items.map((item) => item.kind)).toEqual([ + "measurement", + "implementation", + "research", + ]); + expect(result.items[0]).toMatchObject({ + label: "Run benchmark", + active: true, + }); + expect(result.timeByKind).toEqual({ + reasoning: 2_000, + research: 2_000, + implementation: 3_000, + measurement: 3_000, + }); + }); + + it("excludes activity after a historical run ended", () => { + const events = [ + updateEvent(2_000, { + sessionUpdate: "tool_call", + title: "Run benchmark", + kind: "execute", + status: "completed", + }), + updateEvent(6_000, { + sessionUpdate: "tool_call", + title: "Later manual edit", + kind: "edit", + status: "completed", + }), + ]; + + const result = analyzeAutoresearchActivity(events, 1_000, 4_000, 10_000); + + expect(result.items).toEqual([ + expect.objectContaining({ label: "Run benchmark" }), + ]); + expect(result.timeByKind).toEqual({ + reasoning: 1_000, + research: 0, + implementation: 0, + measurement: 2_000, + }); + }); +}); diff --git a/packages/ui/src/features/autoresearch/autoresearchActivity.ts b/packages/ui/src/features/autoresearch/autoresearchActivity.ts new file mode 100644 index 0000000000..4699e7d3b3 --- /dev/null +++ b/packages/ui/src/features/autoresearch/autoresearchActivity.ts @@ -0,0 +1,115 @@ +import { parsePlanReport } from "@posthog/core/autoresearch/prompts"; +import type { AcpMessage } from "@posthog/shared"; + +export type AutoresearchActivityKind = + | "research" + | "implementation" + | "measurement" + | "reasoning"; + +export interface AutoresearchActivityItem { + id: string; + kind: AutoresearchActivityKind; + label: string; + at: number; + active: boolean; +} + +export interface AutoresearchActivitySnapshot { + currentPlan: ReturnType; + items: AutoresearchActivityItem[]; + timeByKind: Record; +} + +export function analyzeAutoresearchActivity( + events: AcpMessage[], + startedAt: number, + endedAt: number | null, + now: number, +): AutoresearchActivitySnapshot { + const relevant = events.filter( + (event) => + event.ts >= startedAt && (endedAt === null || event.ts <= endedAt), + ); + const items: AutoresearchActivityItem[] = []; + const agentText: string[] = []; + + for (const event of relevant) { + const message = event.message; + if (!("method" in message) || message.method !== "session/update") continue; + const params = message.params as + | { + update?: { + sessionUpdate?: string; + title?: string; + kind?: string | null; + status?: string | null; + content?: { type?: string; text?: string }; + }; + } + | undefined; + const update = params?.update; + if (!update) continue; + + if ( + update.sessionUpdate === "agent_message_chunk" && + update.content?.type === "text" && + update.content.text + ) { + agentText.push(update.content.text); + continue; + } + + if (update.sessionUpdate !== "tool_call") continue; + const kind = activityKindForTool(update.kind); + items.push({ + id: `${event.ts}:${update.title ?? "tool"}`, + kind, + label: update.title || activityLabel(kind), + at: event.ts, + active: update.status === "in_progress" || update.status === "pending", + }); + } + + const currentPlan = parsePlanReport(agentText.join("")); + const end = endedAt ?? now; + const boundaries = items.map((item) => ({ at: item.at, kind: item.kind })); + const timeByKind: Record = { + research: 0, + implementation: 0, + measurement: 0, + reasoning: 0, + }; + let cursor = startedAt; + let kind: AutoresearchActivityKind = "reasoning"; + for (const boundary of boundaries) { + timeByKind[kind] += Math.max(0, boundary.at - cursor); + cursor = boundary.at; + kind = boundary.kind; + } + timeByKind[kind] += Math.max(0, end - cursor); + + return { + currentPlan, + items: items.slice(-8).reverse(), + timeByKind, + }; +} + +function activityKindForTool(kind?: string | null): AutoresearchActivityKind { + if (kind === "edit" || kind === "delete" || kind === "move") { + return "implementation"; + } + if (kind === "execute") return "measurement"; + if (kind === "read" || kind === "search" || kind === "fetch") { + return "research"; + } + return "reasoning"; +} + +function activityLabel(kind: AutoresearchActivityKind): string { + if (kind === "implementation") return "Editing code"; + if (kind === "measurement") return "Running a command"; + if (kind === "research") return "Inspecting the codebase"; + return "Working"; +} diff --git a/packages/ui/src/features/autoresearch/autoresearchDraftStore.ts b/packages/ui/src/features/autoresearch/autoresearchDraftStore.ts index 6629e44672..877e7fe9b0 100644 --- a/packages/ui/src/features/autoresearch/autoresearchDraftStore.ts +++ b/packages/ui/src/features/autoresearch/autoresearchDraftStore.ts @@ -2,7 +2,7 @@ import type { AutoresearchDraftConfig } from "@posthog/core/autoresearch/schemas import { create } from "zustand"; /** - * Autoresearch mode armed on a new-task composer, keyed by the composer's + * Autoresearch mode armed on a new task composer, keyed by the composer * draft session id. Submitting the composer turns the draft into a run. */ interface AutoresearchDraftState { diff --git a/packages/ui/src/features/autoresearch/metricFormat.test.ts b/packages/ui/src/features/autoresearch/metricFormat.test.ts index d8b919efb7..c871ed22b7 100644 --- a/packages/ui/src/features/autoresearch/metricFormat.test.ts +++ b/packages/ui/src/features/autoresearch/metricFormat.test.ts @@ -16,7 +16,7 @@ describe("deltaTone", () => { describe("formatMetricDelta", () => { it.each([ - [null, "kB", "—"], + [null, "kB", "Baseline"], [10.5, "kB", "+10.5 kB"], [-3, null, "-3"], [2, "%", "+2%"], diff --git a/packages/ui/src/features/autoresearch/metricFormat.ts b/packages/ui/src/features/autoresearch/metricFormat.ts index c60c18db6a..d02ccb9686 100644 --- a/packages/ui/src/features/autoresearch/metricFormat.ts +++ b/packages/ui/src/features/autoresearch/metricFormat.ts @@ -45,12 +45,12 @@ export function deltaTone( return isImprovement(delta, 0, direction) ? "improved" : "worsened"; } -/** Signed delta with unit ("+1.5 kB"); "—" for the baseline iteration. */ +/** Signed delta with unit ("+1.5 kB"); "Baseline" for the first iteration. */ export function formatMetricDelta( delta: number | null, unit: string | null, ): string { - if (delta === null) return "—"; + if (delta === null) return "Baseline"; return withMetricUnit( `${delta > 0 ? "+" : ""}${metricNumberFormat.format(delta)}`, unit, diff --git a/packages/ui/src/features/autoresearch/stageModels.tsx b/packages/ui/src/features/autoresearch/stageModels.tsx index 38ac36587d..624e9970e5 100644 --- a/packages/ui/src/features/autoresearch/stageModels.tsx +++ b/packages/ui/src/features/autoresearch/stageModels.tsx @@ -10,7 +10,7 @@ export interface AutoresearchModelOption { } /** - * Sentinel for "no stage model" — Radix `Select.Item` cannot have an empty + * Sentinel for "no stage model" because Radix `Select.Item` cannot have an empty * value, so the "leave the session model alone" choice needs a placeholder. * Shared so the composer strip and the dashboard dialog can't drift onto * different sentinels. @@ -18,7 +18,7 @@ export interface AutoresearchModelOption { export const NO_STAGE_MODEL = "__no_stage_model__"; /** - * Derive stage select options from a session config option — works for the + * Derive stage select options from a session config option. This works for the * `model` option and the `thought_level` (effort) option alike. */ export function toStageSelectOptions( diff --git a/packages/ui/src/features/message-editor/components/ModeSelector.tsx b/packages/ui/src/features/message-editor/components/ModeSelector.tsx index f56a8466ae..39f37db2e4 100644 --- a/packages/ui/src/features/message-editor/components/ModeSelector.tsx +++ b/packages/ui/src/features/message-editor/components/ModeSelector.tsx @@ -105,7 +105,7 @@ export function ModeSelector({ label: "Autoresearch", ...autoresearch, icon: , - className: "text-violet-11", + className: "text-muted-foreground", }); } diff --git a/packages/ui/src/features/task-detail/components/TaskInput.tsx b/packages/ui/src/features/task-detail/components/TaskInput.tsx index 490bb8ce9c..c152024f96 100644 --- a/packages/ui/src/features/task-detail/components/TaskInput.tsx +++ b/packages/ui/src/features/task-detail/components/TaskInput.tsx @@ -732,12 +732,9 @@ export function TaskInput({ implementEffort: currentReasoningLevel ?? null, measureEffort: currentReasoningLevel ?? null, }); - // The loop iterates unattended, so a permission prompt would stall it. - // Default to the most hands-off mode available: bypass when the user has - // enabled it, otherwise accept-edits. - const autonomousMode = allowBypassPermissions - ? "bypassPermissions" - : "acceptEdits"; + // Autoresearch needs to apply edits without stopping for each change, but + // it should not silently inherit the broader bypass-permissions mode. + const autonomousMode = "acceptEdits"; if (modeOption && isValidConfigValue(modeOption, autonomousMode)) { setConfigOption(modeOption.id, autonomousMode); } @@ -749,7 +746,6 @@ export function TaskInput({ sessionId, currentModel, currentReasoningLevel, - allowBypassPermissions, modeOption, setConfigOption, workspaceMode, @@ -1194,7 +1190,7 @@ export function TaskInput({ {autoresearchDraft && ( -
    +