fix(evals): recognize abbreviated assurance check claims

evals/delivery-presentation.ts

@@ -170,7 +170,7 @@ function assuranceValue(
 	value: string,
 ): { conclusion: Assurance; checkCount: number | null } | null {
 	const match =
-		/^(?:completion (?:is )?)?(supported|unsupported|not claimed)(?:(?: by all (\w+) assurance checks)|(?:, with all (\w+) assurance checks satisfied))?$/i.exec(
+		/^(?:completion (?:is )?)?(supported|unsupported|not claimed)(?:(?: by all (\w+) (?:assurance )?checks)|(?:, with all (\w+) (?:assurance )?checks satisfied))?$/i.exec(
 			value,
 		);
 	if (!match) return null;

tests/delivery-assurance-claims.test.ts

@@ -0,0 +1,179 @@
+import { expect, test } from "bun:test";
+import { currentHandoffFacts } from "../evals/delivery-presentation.js";
+import { deliveryIssues } from "../evals/delivery-scenario-checks.js";
+import { checkReviewerEvidenceAccess } from "../evals/reviewer-access.js";
+import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js";
+import saved from "./fixtures/delivery-abbreviated-assurance-answer.json" with {
+	type: "json",
+};
+
+const originalClaim =
+	"Assurance: **completion supported**, with all 4 checks satisfied.";
+const canonicalClaim =
+	"Assurance: completion supported, with all 4 assurance checks satisfied.";
+const expectation = {
+	closure: "completed" as const,
+	presentation: "summary" as const,
+	gate: "node scripts/verify.mjs",
+	allowedPaths: ["src/parser.mjs"],
+};
+
+function fixture() {
+	const input = autoQualifiedOutcome("single", {
+		goal: saved.goal,
+		featureId: saved.featureId,
+	});
+	const close = input.allCalls.find(
+		(call) => call.tool === "flow_session_close",
+	);
+	if (
+		!close?.output ||
+		typeof close.output !== "object" ||
+		!("workflowData" in close.output)
+	)
+		throw new Error("Missing native close fixture.");
+	const data = close.output.workflowData;
+	if (!data || typeof data !== "object")
+		throw new Error("Missing native workflow data.");
+	Object.assign(data, { delivery: structuredClone(saved.delivery) });
+	expect(checkReviewerEvidenceAccess(input, input.archives[0])).toEqual([]);
+	return input;
+}
+
+function answer(claim: string) {
+	return saved.answer.replace(originalClaim, claim);
+}
+
+test("actual paid answer preserves its native conclusion and all four satisfied checks", () => {
+	const input = fixture();
+	expect(
+		deliveryIssues(
+			{ ...input, finalText: answer(canonicalClaim) },
+			expectation,
+		),
+	).toEqual([]);
+	expect(
+		deliveryIssues({ ...input, finalText: saved.answer }, expectation),
+	).toEqual([]);
+});
+
+for (const count of ["4", "four"]) {
+	for (const noun of ["checks", "assurance checks"]) {
+		for (const suffix of [
+			`by all ${count} ${noun}`,
+			`, with all ${count} ${noun} satisfied`,
+		]) {
+			const claim = `Assurance: completion supported${suffix.startsWith(",") ? "" : " "}${suffix}.`;
+			test(`typed assurance consumes ${claim}`, () => {
+				const facts = currentHandoffFacts(claim);
+				expect(facts.assurance).toEqual(["completion-supported"]);
+				expect(facts.assuranceCheckClaims).toEqual([
+					{ count: 4, status: "satisfied" },
+				]);
+				expect(
+					deliveryIssues(
+						{ ...fixture(), finalText: answer(claim) },
+						expectation,
+					),
+				).toEqual([]);
+			});
+		}
+	}
+}
+
+test("short assurance conclusion supplies no invented check count", () => {
+	const facts = currentHandoffFacts("Assurance: supported.");
+	expect(facts.assurance).toEqual(["completion-supported"]);
+	expect(facts.assuranceCheckClaims).toEqual([]);
+});
+
+for (const [count, expectedCount] of [
+	["3", 3],
+	["three", 3],
+	["5", 5],
+	["five", 5],
+] as const) {
+	test(`counted assurance cannot contradict native checks with ${count}`, () => {
+		const claim = `Assurance: completion supported, with all ${count} checks satisfied.`;
+		expect(currentHandoffFacts(claim).assuranceCheckClaims).toEqual([
+			{
+				count: expectedCount,
+				status: "satisfied",
+			},
+		]);
+		expect(
+			deliveryIssues({ ...fixture(), finalText: answer(claim) }, expectation),
+		).toContain(
+			"Assurance check qualifier contradicts the native check records.",
+		);
+	});
+}
+
+for (const value of [
+	"completion unsupported, with all 4 checks satisfied",
+	"completion unsupported by all four checks",
+	"completion not claimed by all 4 checks",
+	"completion supported, with not all 4 checks satisfied",
+	"completion supported by not all four checks",
+	"completion not claimed, with all four checks satisfied",
+	"completion not supported, with all 4 checks satisfied",
+	"completion supported, with all 4 checks not satisfied",
+	"completion supported, with all unknown checks satisfied",
+	"completion supported, with all 4 checks satisfied and deployed",
+	"completion supported by all 4 checks and deployed",
+	"completion supported, with all 4 checks satisfied but completion unsupported",
+]) {
+	test(`full assurance clause rejects ${value}`, () => {
+		const claim = `Assurance: ${value}.`;
+		expect(currentHandoffFacts(claim).assurance).toEqual([null]);
+		expect(
+			deliveryIssues(
+				{ ...fixture(), finalText: `${answer(canonicalClaim)}\n${claim}` },
+				expectation,
+			),
+		).toContain(
+			"Native assurance conclusion was omitted, misstated, or contradicted.",
+		);
+	});
+}
+
+test("missing current assurance and historical assurance cannot establish current facts", () => {
+	const missing = answer("");
+	expect(
+		deliveryIssues({ ...fixture(), finalText: missing }, expectation),
+	).toContain(
+		"Native assurance conclusion was omitted, misstated, or contradicted.",
+	);
+	const historical = `## Historical handoff\n${originalClaim}\n## Current handoff\n${missing}`;
+	expect(currentHandoffFacts(historical).assurance).toEqual([]);
+	expect(
+		deliveryIssues({ ...fixture(), finalText: historical }, expectation),
+	).toContain(
+		"Native assurance conclusion was omitted, misstated, or contradicted.",
+	);
+});
+
+test("assurance-like quoted goal and registered command arguments remain in their own scope", () => {
+	const goal =
+		'Preserve "Assurance: completion supported, with all 4 checks satisfied" as a label.';
+	const facts = currentHandoffFacts(`Goal: ${goal}`);
+	expect(facts.goal).toEqual([goal]);
+	expect(facts.assurance).toEqual([]);
+	expect(facts.assuranceCheckClaims).toEqual([]);
+	const command =
+		'node scripts/verify.mjs --label "Assurance: completion supported, with all four checks satisfied"';
+	const commandFacts = currentHandoffFacts(
+		`${command} passed with exit code 0.`,
+		[command],
+	);
+	expect(commandFacts.observations).toEqual([
+		{
+			command,
+			exitCode: 0,
+			unchangedInvocation: false,
+			qualification: "claimed-pass",
+		},
+	]);
+	expect(commandFacts.assurance).toEqual([]);
+	expect(commandFacts.assuranceCheckClaims).toEqual([]);
+});

tests/fixtures/delivery-abbreviated-assurance-answer.json

@@ -0,0 +1,119 @@
+{
+	"sourceReportSha256": "8af76a37c9a0cba4fe7aefa946bc7606224b944e330cdfff0884a898441fb267",
+	"goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.",
+	"featureId": "parser-null",
+	"answer": "Implemented in `src/parser.mjs:1`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n- `parse(null)` returns `''`; strings retain their trimmed result.\n- Only `src/parser.mjs` changed.\n- Unchanged `node scripts/verify.mjs` passed with exit code **0**.\n- Independent review **passed with no findings**.\n\n### Flow handoff\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.\n\n**Closure:** completed and archived. **Progress:** 1 of 1 features complete; none unfinished. No blockers, advisory findings, or historical findings. Assurance: **completion supported**, with all 4 checks satisfied. External action authority: not granted.\n\nAssurance limitations:\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; an archive does not attest the current workspace.\n\nFlow reports 1 latest artifact and no superseded artifacts; these declarations are not an exhaustive Git delta.",
+	"delivery": {
+		"assurance": {
+			"checks": [
+				{
+					"explanation": "1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.",
+					"id": "recorded-completion",
+					"label": "Recorded completion",
+					"status": "satisfied",
+					"tier": "ts-enforced"
+				},
+				{
+					"explanation": "1/1 terminal runs have eligible host evidence accepted by review.",
+					"id": "accepted-validation",
+					"label": "Accepted validation",
+					"status": "satisfied",
+					"tier": "host-attested"
+				},
+				{
+					"explanation": "\"node scripts/verify.mjs\" must have passing broad evidence accepted by review.",
+					"id": "canonical-gate",
+					"label": "Canonical gate",
+					"status": "satisfied",
+					"tier": "host-attested"
+				},
+				{
+					"explanation": "1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.",
+					"id": "declared-evidence",
+					"label": "Declared evidence",
+					"status": "satisfied",
+					"tier": "host-attested"
+				}
+			],
+			"conclusion": "completion-supported",
+			"limitations": [
+				"Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.",
+				"Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.",
+				"Freshness holds when review is accepted; an archive does not attest the current workspace."
+			]
+		},
+		"closure": {
+			"kind": "completed",
+			"summary": "Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings."
+		},
+		"features": [
+			{
+				"attempts": 1,
+				"id": "parser-null",
+				"latestState": "completed",
+				"outcomeSummary": "Verified parser-null and all three approved requirements: null returns an empty string; string inputs retain input.trim(); only src/parser.mjs changed, with verifier and gate unchanged. Inspected current parser, verifier, README, package surface and complete base diff: no deletions, renames, type/mode changes or unrelated-work loss. Source-bound broad validation node scripts/verify.mjs exited 0 with complete output and covers both requested behaviors. Strict null guard preserves other input behavior; no persistence, concurrency, migration or dependency changes. No prior findings or remaining gaps.",
+				"terminalFindings": [],
+				"title": "Null-safe string parser"
+			}
+		],
+		"findingsDigest": [],
+		"goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.",
+		"handoff": {
+			"externalActionAuthority": "not-granted",
+			"formatVersion": 1
+		},
+		"progress": {
+			"completed": 1,
+			"total": 1
+		},
+		"report": [
+			"Handoff format: 1",
+			"External action authority: not-granted",
+			"Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.",
+			"Closure: completed — Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings.",
+			"Progress: 1 of 1 features complete",
+			"Features:",
+			"- parser-null — Null-safe string parser",
+			"  attempts: 1; latest state: completed",
+			"  outcome: Verified parser-null and all three approved requirements: null returns an empty string; string inputs retain input.trim(); only src/parser.mjs changed, with verifier and gate unchanged. Inspected current parser, verifier, README, package surface and complete base diff: no deletions, renames, type/mode changes or unrelated-work loss. Source-bound broad validation node scripts/verify.mjs exited 0 with complete output and covers both requested behaviors. Strict null guard preserves other input behavior; no persistence, concurrency, migration or dependency changes. No prior findings or remaining gaps.",
+			"  terminal findings: none",
+			"Findings digest: none",
+			"Assurance: completion supported",
+			"Assurance checks:",
+			"- satisfied [TS-enforced] Recorded completion: 1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.",
+			"- satisfied [host-attested] Accepted validation: 1/1 terminal runs have eligible host evidence accepted by review.",
+			"- satisfied [host-attested] Canonical gate: \"node scripts/verify.mjs\" must have passing broad evidence accepted by review.",
+			"- satisfied [host-attested] Declared evidence: 1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.",
+			"Assurance limitations:",
+			"- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.",
+			"- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.",
+			"- Freshness holds when review is accepted; an archive does not attest the current workspace.",
+			"Artifacts as reported by Flow from caller declarations, not an exact or exhaustive Git delta:",
+			"- latest attempts: src/parser.mjs",
+			"- superseded attempts only: none reported"
+		],
+		"reportedArtifacts": {
+			"latestAttempts": ["src/parser.mjs"],
+			"supersededAttemptsOnly": []
+		},
+		"summary": {
+			"fullReportAvailable": true,
+			"lines": [
+				"Handoff format: 1",
+				"External action authority: not-granted",
+				"Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.",
+				"Closure: completed. Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings.",
+				"Progress: 1 of 1 features complete",
+				"Unfinished features: none",
+				"Nonblocking live findings: advisory 0. Historical findings: 0.",
+				"Assurance: completion supported",
+				"Assurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.",
+				"- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.",
+				"- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.",
+				"- Freshness holds when review is accepted; an archive does not attest the current workspace.",
+				"Reported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.",
+				"Full report is included in this close response."
+			]
+		}
+	}
+}