From 5e228c77383eceb19f95a8200d5a94a30681a351 Mon Sep 17 00:00:00 2001 From: vriesd Date: Wed, 7 Oct 2026 22:59:09 +0200 Subject: [PATCH 1/4] test(flow): expose disclosure omission and article grading mismatch --- tests/delivery-disclosure-contract.test.ts | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/delivery-disclosure-contract.test.ts diff --git a/tests/delivery-disclosure-contract.test.ts b/tests/delivery-disclosure-contract.test.ts new file mode 100644 index 00000000..aedc52ef --- /dev/null +++ b/tests/delivery-disclosure-contract.test.ts @@ -0,0 +1,120 @@ +import { expect, test } from "bun:test"; +import { missingAssuranceDisclosures } from "../evals/delivery-presentation.js"; +import { deliveryIssues } from "../evals/delivery-scenario-checks.js"; +import { getFlowGuidance } from "../src/guidance/catalog.js"; +import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; +import saved from "./fixtures/delivery-flow-zero-count-answer.json" with { + type: "json", +}; + +const actual = + "Implemented in `src/parser.mjs`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n- `parse(null)` returns `''`; strings retain trimming behavior.\n- Only `src/parser.mjs` changed. `scripts/verify.mjs` remains unchanged.\n- Validation: `node scripts/verify.mjs` passed with exit code **0**.\n- Independent review passed with **no findings**.\n\n### Flow completion\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.\n\nSession completed and archived: **1 of 1 features complete**, no unfinished features or blockers. Assurance: **completion supported**, with all four assurance checks satisfied. External action authority: not granted.\n\nAssurance limitations:\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Scope, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; the archive does not attest the current workspace."; +const partial = + "Scope, requirement coverage, test adequacy, and review substance remain model judgments."; +const complete = + "Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments."; +const freshness = + "Freshness holds when review is accepted; the archive does not attest the current workspace."; +const canonicalFreshness = freshness.replace("the archive", "an archive"); +const coverageIssue = + "Missing assurance disclosure: coverage and review substance remain model judgments."; +const freshnessIssue = + "Missing assurance disclosure: accepted evidence does not attest later workspace changes."; +const conflict = "Assurance disclosures contain a conflicting current claim."; +const expected = { + closure: "completed" as const, + presentation: "summary" as const, + gate: "node scripts/verify.mjs", + allowedPaths: ["src/parser.mjs"], +}; +function fixture(finalText: string) { + const input = autoQualifiedOutcome("single", { + goal: saved.goal, + featureId: saved.featureId, + }); + const close = input.allCalls.find( + (call) => call.tool === "flow_session_close", + ); + if (!close || typeof close.output !== "object" || close.output === null) + throw new Error("Missing close fixture."); + const data = (close.output as { workflowData: { delivery: unknown } }) + .workflowData; + data.delivery = structuredClone(saved.delivery); + return { ...input, finalText }; +} + +test("unchanged retained handoff still rejects its genuinely omitted model judgments", () => { + expect(deliveryIssues(fixture(actual), expected)).toEqual([coverageIssue]); +}); +test("complete judgment topics and definite archive compose in whole checker", () => { + expect( + deliveryIssues(fixture(actual.replace(partial, complete)), expected), + ).toEqual([]); +}); +test("canonical archive article still accepts complete handoff", () => { + expect( + deliveryIssues( + fixture( + actual + .replace(partial, complete) + .replace(freshness, canonicalFreshness), + ), + expected, + ), + ).toEqual([]); +}); +for (const topic of ["Goal alignment, ", "evidence completeness, "]) { + test(`Goal line cannot replace the omitted disclosure ${topic}`, () => { + expect( + missingAssuranceDisclosures( + actual + .replace(partial, complete.replace(topic, "")) + .replace(freshness, canonicalFreshness), + ), + ).toEqual([coverageIssue]); + }); +} +for (const invalid of [ + "Freshness holds when review is accepted; the archive attests the current workspace.", + "Freshness will hold when review is accepted; the archive does not attest the current workspace.", + "Freshness holds when review is accepted; the archive does not attest the current workspace if validation passes.", + "Freshness holds when review is accepted; the archive does not attest the current workspace or authorize release.", +]) { + test(`freshness remains closed and truthful ${invalid}`, () => { + expect( + missingAssuranceDisclosures( + actual.replace(partial, complete).replace(freshness, invalid), + ), + ).toContain(freshnessIssue); + }); +} +for (const ignored of [ + `> ${freshness}`, + `Example: "${freshness}"`, + `## Historical handoff\n${freshness}`, + `Goal: ${freshness}`, +]) { + test(`freshness cannot be supplied in ignored scope ${ignored}`, () => { + expect( + missingAssuranceDisclosures( + actual.replace(partial, complete).replace(freshness, ignored), + ), + ).toContain(freshnessIssue); + }); +} +test("valid definite archive disclosure does not hide contradictory current claim", () => { + expect( + missingAssuranceDisclosures( + `${actual.replace(partial, complete)}\nThe archive attests the current workspace.`, + ), + ).toEqual([conflict]); +}); +test("delivered guidance preserves each supplied limitation unchanged", () => { + const guidance = getFlowGuidance("flow-run").content.replace(/\s+/g, " "); + expect(guidance).toContain( + "Copy each supplied `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list.", + ); + expect(guidance).toContain( + "For requested full detail or a missing summary, use the retained close response's `report`.", + ); +}); From bc5eabf92b47de3e0f4702b48ea1b1592099000a Mon Sep 17 00:00:00 2001 From: vriesd Date: Wed, 7 Oct 2026 23:02:43 +0200 Subject: [PATCH 2/4] fix(flow): preserve supplied assurance limitation statements --- evals/delivery-presentation.ts | 7 ++++++- skills/flow-run/SKILL.md | 6 ++++-- tests/delivery-disclosure-contract.test.ts | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/evals/delivery-presentation.ts b/evals/delivery-presentation.ts index f9fe9797..1f0568be 100644 --- a/evals/delivery-presentation.ts +++ b/evals/delivery-presentation.ts @@ -704,7 +704,12 @@ export function missingAssuranceDisclosures(text: string): string[] { .replace(/[.*+?^${}()|[\]\\]/g, "\\$&") .replace(/ /g, "\\s+"); const canonicalMatches = [ - ...body.matchAll(new RegExp(`(?:^|\\n)${words}(?=$|\\n)`, "g")), + ...body.matchAll( + new RegExp( + `(?:^|\\n)${index === 2 ? words.replace("an\\s+archive", "(?:an|the)\\s+archive") : words}(?=$|\\n)`, + "g", + ), + ), ]; const canonical = canonicalMatches.length > 0; const subjectWords = disclosure.subject diff --git a/skills/flow-run/SKILL.md b/skills/flow-run/SKILL.md index 20c8fc9c..e88b3861 100644 --- a/skills/flow-run/SKILL.md +++ b/skills/flow-run/SKILL.md @@ -39,8 +39,10 @@ Work on exactly one approved feature. Delivery handoff: report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance -conclusion and external action authority. Retain all three assurance limitations, -blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit them. +conclusion and external action authority. Copy each supplied +`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. +Retain blockers, unfinished IDs and observed command, exit and nonpass facts. +Concise formatting may not omit them. For requested full detail or a missing summary, use the retained close response's `report`. Map IDs only diff --git a/tests/delivery-disclosure-contract.test.ts b/tests/delivery-disclosure-contract.test.ts index aedc52ef..0b601f2f 100644 --- a/tests/delivery-disclosure-contract.test.ts +++ b/tests/delivery-disclosure-contract.test.ts @@ -89,7 +89,6 @@ for (const invalid of [ }); } for (const ignored of [ - `> ${freshness}`, `Example: "${freshness}"`, `## Historical handoff\n${freshness}`, `Goal: ${freshness}`, @@ -97,11 +96,22 @@ for (const ignored of [ test(`freshness cannot be supplied in ignored scope ${ignored}`, () => { expect( missingAssuranceDisclosures( - actual.replace(partial, complete).replace(freshness, ignored), + actual.replace(partial, complete).replace(`- ${freshness}`, ignored), ), ).toContain(freshnessIssue); }); } +for (const statement of [canonicalFreshness, freshness]) { + test(`Markdown blockquote formatting preserves visible disclosure ${statement}`, () => { + expect( + missingAssuranceDisclosures( + actual + .replace(partial, complete) + .replace(`- ${freshness}`, `> ${statement}`), + ), + ).toEqual([]); + }); +} test("valid definite archive disclosure does not hide contradictory current claim", () => { expect( missingAssuranceDisclosures( From 4eb856b0792e3219c828ebcad6274f828c0b6a0b Mon Sep 17 00:00:00 2001 From: vriesd Date: Wed, 7 Oct 2026 23:19:00 +0200 Subject: [PATCH 3/4] test(flow): cover disclosure copying across handoff skills --- tests/delivery-disclosure-contract.test.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/delivery-disclosure-contract.test.ts b/tests/delivery-disclosure-contract.test.ts index 0b601f2f..9c43c06d 100644 --- a/tests/delivery-disclosure-contract.test.ts +++ b/tests/delivery-disclosure-contract.test.ts @@ -119,12 +119,15 @@ test("valid definite archive disclosure does not hide contradictory current clai ), ).toEqual([conflict]); }); -test("delivered guidance preserves each supplied limitation unchanged", () => { - const guidance = getFlowGuidance("flow-run").content.replace(/\s+/g, " "); - expect(guidance).toContain( - "Copy each supplied `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list.", - ); - expect(guidance).toContain( - "For requested full detail or a missing summary, use the retained close response's `report`.", - ); -}); +for (const id of ["flow", "flow-plan", "flow-run"] as const) { + test(`delivered ${id} guidance preserves each supplied limitation unchanged`, () => { + const guidance = getFlowGuidance(id).content.replace(/\s+/g, " "); + expect(guidance).toContain( + "Copy each supplied `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list.", + ); + expect(guidance).toContain("`workflowData.delivery.summary.lines`"); + expect(guidance).toMatch( + /For requested full detail or a missing summary, (?:use|report) the retained close response's `report`\./, + ); + }); +} From f25987c862b6e0ff74bb5b140e308b49af70e679 Mon Sep 17 00:00:00 2001 From: vriesd Date: Wed, 7 Oct 2026 23:19:45 +0200 Subject: [PATCH 4/4] fix(flow): preserve disclosures across all handoff skills --- skills/flow-plan/SKILL.md | 7 ++++--- skills/flow/SKILL.md | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/skills/flow-plan/SKILL.md b/skills/flow-plan/SKILL.md index fd714a06..58ac1fa1 100644 --- a/skills/flow-plan/SKILL.md +++ b/skills/flow-plan/SKILL.md @@ -34,9 +34,10 @@ without rediscovering the goal. mixed continuation plus unrelated work, and a replacement goal as new-scope. - Delivery handoff: report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance - conclusion and external action authority. Retain all three assurance limitations, - blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit - them. For requested + conclusion and external action authority. Copy each supplied + `workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. + Retain blockers, unfinished IDs and observed command, exit and nonpass facts. + Concise formatting may not omit them. For requested full detail or a missing summary, use the retained close response's `report`. Map IDs only from `outcomeSummary`/`terminalFindings`. Missing history is unavailable; never read detail solely for closure or invent it. diff --git a/skills/flow/SKILL.md b/skills/flow/SKILL.md index 9f0a6c7c..1c7ec5bc 100644 --- a/skills/flow/SKILL.md +++ b/skills/flow/SKILL.md @@ -107,8 +107,10 @@ only the `archiveRetry` of a durably accepted close. Rejected revision conflict: refresh compact, confirm the same session/goal, then build a fresh request. Report `workflowData.delivery.summary.lines`. Keep the supplied Goal line unchanged. Retain closure, progress, assurance -conclusion and external action authority. Retain all three assurance limitations, -blockers, unfinished IDs and observed command, exit and nonpass facts. Concise formatting may not omit them. +conclusion and external action authority. Copy each supplied +`workflowData.delivery.assurance.limitations` statement unchanged, including its full topic list. +Retain blockers, unfinished IDs and observed command, exit and nonpass facts. +Concise formatting may not omit them. For requested full detail or a missing summary, report the retained close response's `report`. Report external prerequisites only from terminal text; otherwise mark them unavailable. Create no other ledger or