diff --git a/evals/delivery-presentation.ts b/evals/delivery-presentation.ts index f9fe9797..1f0568be 100644 --- a/evals/delivery-presentation.ts +++ b/evals/delivery-presentation.ts @@ -704,7 +704,12 @@ export function missingAssuranceDisclosures(text: string): string[] { .replace(/[.*+?^${}()|[\]\\]/g, "\\$&") .replace(/ /g, "\\s+"); const canonicalMatches = [ - ...body.matchAll(new RegExp(`(?:^|\\n)${words}(?=$|\\n)`, "g")), + ...body.matchAll( + new RegExp( + `(?:^|\\n)${index === 2 ? words.replace("an\\s+archive", "(?:an|the)\\s+archive") : words}(?=$|\\n)`, + "g", + ), + ), ]; const canonical = canonicalMatches.length > 0; const subjectWords = disclosure.subject diff --git a/skills/flow-plan/SKILL.md b/skills/flow-plan/SKILL.md index fd714a06..58ac1fa1 100644 --- a/skills/flow-plan/SKILL.md +++ b/skills/flow-plan/SKILL.md @@ -34,9 +34,10 @@ without rediscovering the goal. mixed continuation plus unrelated work, and a replacement goal as new-scope. - Delivery handoff: report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance - conclusion and external action authority. Retain all three assurance limitations, - blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit - them. For requested + conclusion and external action authority. Copy each supplied + `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. + Retain blockers, unfinished IDs and observed command, exit and nonpass facts. + Concise formatting may not omit them. For requested full detail or a missing summary, use the retained close response's `report`. Map IDs only from `outcomeSummary`/`terminalFindings`. Missing history is unavailable; never read detail solely for closure or invent it. diff --git a/skills/flow-run/SKILL.md b/skills/flow-run/SKILL.md index 20c8fc9c..e88b3861 100644 --- a/skills/flow-run/SKILL.md +++ b/skills/flow-run/SKILL.md @@ -39,8 +39,10 @@ Work on exactly one approved feature. Delivery handoff: report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance -conclusion and external action authority. Retain all three assurance limitations, -blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit them. +conclusion and external action authority. Copy each supplied +`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. +Retain blockers, unfinished IDs and observed command, exit and nonpass facts. +Concise formatting may not omit them. For requested full detail or a missing summary, use the retained close response's `report`. Map IDs only diff --git a/skills/flow/SKILL.md b/skills/flow/SKILL.md index 9f0a6c7c..1c7ec5bc 100644 --- a/skills/flow/SKILL.md +++ b/skills/flow/SKILL.md @@ -107,8 +107,10 @@ only the `archiveRetry` of a durably accepted close. Rejected revision conflict: refresh compact, confirm the same session/goal, then build a fresh request. Report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance -conclusion and external action authority. Retain all three assurance limitations, -blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit them. +conclusion and external action authority. Copy each supplied +`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. +Retain blockers, unfinished IDs and observed command, exit and nonpass facts. +Concise formatting may not omit them. For requested full detail or a missing summary, report the retained close response's `report`. Report external prerequisites only from terminal text; otherwise mark them unavailable. Create no other ledger or diff --git a/tests/delivery-disclosure-contract.test.ts b/tests/delivery-disclosure-contract.test.ts new file mode 100644 index 00000000..9c43c06d --- /dev/null +++ b/tests/delivery-disclosure-contract.test.ts @@ -0,0 +1,133 @@ +import { expect, test } from "bun:test"; +import { missingAssuranceDisclosures } from "../evals/delivery-presentation.js"; +import { deliveryIssues } from "../evals/delivery-scenario-checks.js"; +import { getFlowGuidance } from "../src/guidance/catalog.js"; +import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; +import saved from "./fixtures/delivery-flow-zero-count-answer.json" with { + type: "json", +}; + +const actual = + "Implemented in `src/parser.mjs`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n- `parse(null)` returns `''`; strings retain trimming behavior.\n- Only `src/parser.mjs` changed. `scripts/verify.mjs` remains unchanged.\n- Validation: `node scripts/verify.mjs` passed with exit code **0**.\n- Independent review passed with **no findings**.\n\n### Flow completion\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.\n\nSession completed and archived: **1 of 1 features complete**, no unfinished features or blockers. Assurance: **completion supported**, with all four assurance checks satisfied. External action authority: not granted.\n\nAssurance limitations:\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Scope, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; the archive does not attest the current workspace."; +const partial = + "Scope, requirement coverage, test adequacy, and review substance remain model judgments."; +const complete = + "Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments."; +const freshness = + "Freshness holds when review is accepted; the archive does not attest the current workspace."; +const canonicalFreshness = freshness.replace("the archive", "an archive"); +const coverageIssue = + "Missing assurance disclosure: coverage and review substance remain model judgments."; +const freshnessIssue = + "Missing assurance disclosure: accepted evidence does not attest later workspace changes."; +const conflict = "Assurance disclosures contain a conflicting current claim."; +const expected = { + closure: "completed" as const, + presentation: "summary" as const, + gate: "node scripts/verify.mjs", + allowedPaths: ["src/parser.mjs"], +}; +function fixture(finalText: string) { + const input = autoQualifiedOutcome("single", { + goal: saved.goal, + featureId: saved.featureId, + }); + const close = input.allCalls.find( + (call) => call.tool === "flow_session_close", + ); + if (!close || typeof close.output !== "object" || close.output === null) + throw new Error("Missing close fixture."); + const data = (close.output as { workflowData: { delivery: unknown } }) + .workflowData; + data.delivery = structuredClone(saved.delivery); + return { ...input, finalText }; +} + +test("unchanged retained handoff still rejects its genuinely omitted model judgments", () => { + expect(deliveryIssues(fixture(actual), expected)).toEqual([coverageIssue]); +}); +test("complete judgment topics and definite archive compose in whole checker", () => { + expect( + deliveryIssues(fixture(actual.replace(partial, complete)), expected), + ).toEqual([]); +}); +test("canonical archive article still accepts complete handoff", () => { + expect( + deliveryIssues( + fixture( + actual + .replace(partial, complete) + .replace(freshness, canonicalFreshness), + ), + expected, + ), + ).toEqual([]); +}); +for (const topic of ["Goal alignment, ", "evidence completeness, "]) { + test(`Goal line cannot replace the omitted disclosure ${topic}`, () => { + expect( + missingAssuranceDisclosures( + actual + .replace(partial, complete.replace(topic, "")) + .replace(freshness, canonicalFreshness), + ), + ).toEqual([coverageIssue]); + }); +} +for (const invalid of [ + "Freshness holds when review is accepted; the archive attests the current workspace.", + "Freshness will hold when review is accepted; the archive does not attest the current workspace.", + "Freshness holds when review is accepted; the archive does not attest the current workspace if validation passes.", + "Freshness holds when review is accepted; the archive does not attest the current workspace or authorize release.", +]) { + test(`freshness remains closed and truthful ${invalid}`, () => { + expect( + missingAssuranceDisclosures( + actual.replace(partial, complete).replace(freshness, invalid), + ), + ).toContain(freshnessIssue); + }); +} +for (const ignored of [ + `Example: "${freshness}"`, + `## Historical handoff\n${freshness}`, + `Goal: ${freshness}`, +]) { + test(`freshness cannot be supplied in ignored scope ${ignored}`, () => { + expect( + missingAssuranceDisclosures( + actual.replace(partial, complete).replace(`- ${freshness}`, ignored), + ), + ).toContain(freshnessIssue); + }); +} +for (const statement of [canonicalFreshness, freshness]) { + test(`Markdown blockquote formatting preserves visible disclosure ${statement}`, () => { + expect( + missingAssuranceDisclosures( + actual + .replace(partial, complete) + .replace(`- ${freshness}`, `> ${statement}`), + ), + ).toEqual([]); + }); +} +test("valid definite archive disclosure does not hide contradictory current claim", () => { + expect( + missingAssuranceDisclosures( + `${actual.replace(partial, complete)}\nThe archive attests the current workspace.`, + ), + ).toEqual([conflict]); +}); +for (const id of ["flow", "flow-plan", "flow-run"] as const) { + test(`delivered ${id} guidance preserves each supplied limitation unchanged`, () => { + const guidance = getFlowGuidance(id).content.replace(/\s+/g, " "); + expect(guidance).toContain( + "Copy each supplied `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list.", + ); + expect(guidance).toContain("`workflowData.delivery.summary.lines`"); + expect(guidance).toMatch( + /For requested full detail or a missing summary, (?:use|report) the retained close response's `report`\./, + ); + }); +}