diff --git a/evals/delivery-presentation.ts b/evals/delivery-presentation.ts index fad4a3a7..eedc061b 100644 --- a/evals/delivery-presentation.ts +++ b/evals/delivery-presentation.ts @@ -170,7 +170,7 @@ function assuranceValue( value: string, ): { conclusion: Assurance; checkCount: number | null } | null { const match = - /^(?:completion (?:is )?)?(supported|unsupported|not claimed)(?:(?: by all (\w+) assurance checks)|(?:, with all (\w+) assurance checks satisfied))?$/i.exec( + /^(?:completion (?:is )?)?(supported|unsupported|not claimed)(?:(?: by all (\w+) (?:assurance )?checks)|(?:, with all (\w+) (?:assurance )?checks satisfied))?$/i.exec( value, ); if (!match) return null; diff --git a/tests/delivery-assurance-claims.test.ts b/tests/delivery-assurance-claims.test.ts new file mode 100644 index 00000000..024af901 --- /dev/null +++ b/tests/delivery-assurance-claims.test.ts @@ -0,0 +1,179 @@ +import { expect, test } from "bun:test"; +import { currentHandoffFacts } from "../evals/delivery-presentation.js"; +import { deliveryIssues } from "../evals/delivery-scenario-checks.js"; +import { checkReviewerEvidenceAccess } from "../evals/reviewer-access.js"; +import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; +import saved from "./fixtures/delivery-abbreviated-assurance-answer.json" with { + type: "json", +}; + +const originalClaim = + "Assurance: **completion supported**, with all 4 checks satisfied."; +const canonicalClaim = + "Assurance: completion supported, with all 4 assurance checks satisfied."; +const expectation = { + closure: "completed" as const, + presentation: "summary" as const, + gate: "node scripts/verify.mjs", + allowedPaths: ["src/parser.mjs"], +}; + +function fixture() { + const input = autoQualifiedOutcome("single", { + goal: saved.goal, + featureId: saved.featureId, + }); + const close = input.allCalls.find( + (call) => call.tool === "flow_session_close", + ); + if ( + !close?.output || + typeof close.output !== "object" || + !("workflowData" in close.output) + ) + throw new Error("Missing native close fixture."); + const data = close.output.workflowData; + if (!data || typeof data !== "object") + throw new Error("Missing native workflow data."); + Object.assign(data, { delivery: structuredClone(saved.delivery) }); + expect(checkReviewerEvidenceAccess(input, input.archives[0])).toEqual([]); + return input; +} + +function answer(claim: string) { + return saved.answer.replace(originalClaim, claim); +} + +test("actual paid answer preserves its native conclusion and all four satisfied checks", () => { + const input = fixture(); + expect( + deliveryIssues( + { ...input, finalText: answer(canonicalClaim) }, + expectation, + ), + ).toEqual([]); + expect( + deliveryIssues({ ...input, finalText: saved.answer }, expectation), + ).toEqual([]); +}); + +for (const count of ["4", "four"]) { + for (const noun of ["checks", "assurance checks"]) { + for (const suffix of [ + `by all ${count} ${noun}`, + `, with all ${count} ${noun} satisfied`, + ]) { + const claim = `Assurance: completion supported${suffix.startsWith(",") ? "" : " "}${suffix}.`; + test(`typed assurance consumes ${claim}`, () => { + const facts = currentHandoffFacts(claim); + expect(facts.assurance).toEqual(["completion-supported"]); + expect(facts.assuranceCheckClaims).toEqual([ + { count: 4, status: "satisfied" }, + ]); + expect( + deliveryIssues( + { ...fixture(), finalText: answer(claim) }, + expectation, + ), + ).toEqual([]); + }); + } + } +} + +test("short assurance conclusion supplies no invented check count", () => { + const facts = currentHandoffFacts("Assurance: supported."); + expect(facts.assurance).toEqual(["completion-supported"]); + expect(facts.assuranceCheckClaims).toEqual([]); +}); + +for (const [count, expectedCount] of [ + ["3", 3], + ["three", 3], + ["5", 5], + ["five", 5], +] as const) { + test(`counted assurance cannot contradict native checks with ${count}`, () => { + const claim = `Assurance: completion supported, with all ${count} checks satisfied.`; + expect(currentHandoffFacts(claim).assuranceCheckClaims).toEqual([ + { + count: expectedCount, + status: "satisfied", + }, + ]); + expect( + deliveryIssues({ ...fixture(), finalText: answer(claim) }, expectation), + ).toContain( + "Assurance check qualifier contradicts the native check records.", + ); + }); +} + +for (const value of [ + "completion unsupported, with all 4 checks satisfied", + "completion unsupported by all four checks", + "completion not claimed by all 4 checks", + "completion supported, with not all 4 checks satisfied", + "completion supported by not all four checks", + "completion not claimed, with all four checks satisfied", + "completion not supported, with all 4 checks satisfied", + "completion supported, with all 4 checks not satisfied", + "completion supported, with all unknown checks satisfied", + "completion supported, with all 4 checks satisfied and deployed", + "completion supported by all 4 checks and deployed", + "completion supported, with all 4 checks satisfied but completion unsupported", +]) { + test(`full assurance clause rejects ${value}`, () => { + const claim = `Assurance: ${value}.`; + expect(currentHandoffFacts(claim).assurance).toEqual([null]); + expect( + deliveryIssues( + { ...fixture(), finalText: `${answer(canonicalClaim)}\n${claim}` }, + expectation, + ), + ).toContain( + "Native assurance conclusion was omitted, misstated, or contradicted.", + ); + }); +} + +test("missing current assurance and historical assurance cannot establish current facts", () => { + const missing = answer(""); + expect( + deliveryIssues({ ...fixture(), finalText: missing }, expectation), + ).toContain( + "Native assurance conclusion was omitted, misstated, or contradicted.", + ); + const historical = `## Historical handoff\n${originalClaim}\n## Current handoff\n${missing}`; + expect(currentHandoffFacts(historical).assurance).toEqual([]); + expect( + deliveryIssues({ ...fixture(), finalText: historical }, expectation), + ).toContain( + "Native assurance conclusion was omitted, misstated, or contradicted.", + ); +}); + +test("assurance-like quoted goal and registered command arguments remain in their own scope", () => { + const goal = + 'Preserve "Assurance: completion supported, with all 4 checks satisfied" as a label.'; + const facts = currentHandoffFacts(`Goal: ${goal}`); + expect(facts.goal).toEqual([goal]); + expect(facts.assurance).toEqual([]); + expect(facts.assuranceCheckClaims).toEqual([]); + const command = + 'node scripts/verify.mjs --label "Assurance: completion supported, with all four checks satisfied"'; + const commandFacts = currentHandoffFacts( + `${command} passed with exit code 0.`, + [command], + ); + expect(commandFacts.observations).toEqual([ + { + command, + exitCode: 0, + unchangedInvocation: false, + qualification: "claimed-pass", + }, + ]); + expect(commandFacts.assurance).toEqual([]); + expect(commandFacts.assuranceCheckClaims).toEqual([]); +}); diff --git a/tests/fixtures/delivery-abbreviated-assurance-answer.json b/tests/fixtures/delivery-abbreviated-assurance-answer.json new file mode 100644 index 00000000..3be6facf --- /dev/null +++ b/tests/fixtures/delivery-abbreviated-assurance-answer.json @@ -0,0 +1,119 @@ +{ + "sourceReportSha256": "8af76a37c9a0cba4fe7aefa946bc7606224b944e330cdfff0884a898441fb267", + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "featureId": "parser-null", + "answer": "Implemented in `src/parser.mjs:1`:\n\n```js\nexport function parse(input) { return input === null ? '' : input.trim(); }\n```\n\n- `parse(null)` returns `''`; strings retain their trimmed result.\n- Only `src/parser.mjs` changed.\n- Unchanged `node scripts/verify.mjs` passed with exit code **0**.\n- Independent review **passed with no findings**.\n\n### Flow handoff\nGoal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.\n\n**Closure:** completed and archived. **Progress:** 1 of 1 features complete; none unfinished. No blockers, advisory findings, or historical findings. Assurance: **completion supported**, with all 4 checks satisfied. External action authority: not granted.\n\nAssurance limitations:\n- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.\n- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.\n- Freshness holds when review is accepted; an archive does not attest the current workspace.\n\nFlow reports 1 latest artifact and no superseded artifacts; these declarations are not an exhaustive Git delta.", + "delivery": { + "assurance": { + "checks": [ + { + "explanation": "1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "id": "recorded-completion", + "label": "Recorded completion", + "status": "satisfied", + "tier": "ts-enforced" + }, + { + "explanation": "1/1 terminal runs have eligible host evidence accepted by review.", + "id": "accepted-validation", + "label": "Accepted validation", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "\"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "id": "canonical-gate", + "label": "Canonical gate", + "status": "satisfied", + "tier": "host-attested" + }, + { + "explanation": "1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "id": "declared-evidence", + "label": "Declared evidence", + "status": "satisfied", + "tier": "host-attested" + } + ], + "conclusion": "completion-supported", + "limitations": [ + "Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "Freshness holds when review is accepted; an archive does not attest the current workspace." + ] + }, + "closure": { + "kind": "completed", + "summary": "Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings." + }, + "features": [ + { + "attempts": 1, + "id": "parser-null", + "latestState": "completed", + "outcomeSummary": "Verified parser-null and all three approved requirements: null returns an empty string; string inputs retain input.trim(); only src/parser.mjs changed, with verifier and gate unchanged. Inspected current parser, verifier, README, package surface and complete base diff: no deletions, renames, type/mode changes or unrelated-work loss. Source-bound broad validation node scripts/verify.mjs exited 0 with complete output and covers both requested behaviors. Strict null guard preserves other input behavior; no persistence, concurrency, migration or dependency changes. No prior findings or remaining gaps.", + "terminalFindings": [], + "title": "Null-safe string parser" + } + ], + "findingsDigest": [], + "goal": "Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "handoff": { + "externalActionAuthority": "not-granted", + "formatVersion": 1 + }, + "progress": { + "completed": 1, + "total": 1 + }, + "report": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "Closure: completed — Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings.", + "Progress: 1 of 1 features complete", + "Features:", + "- parser-null — Null-safe string parser", + " attempts: 1; latest state: completed", + " outcome: Verified parser-null and all three approved requirements: null returns an empty string; string inputs retain input.trim(); only src/parser.mjs changed, with verifier and gate unchanged. Inspected current parser, verifier, README, package surface and complete base diff: no deletions, renames, type/mode changes or unrelated-work loss. Source-bound broad validation node scripts/verify.mjs exited 0 with complete output and covers both requested behaviors. Strict null guard preserves other input behavior; no persistence, concurrency, migration or dependency changes. No prior findings or remaining gaps.", + " terminal findings: none", + "Findings digest: none", + "Assurance: completion supported", + "Assurance checks:", + "- satisfied [TS-enforced] Recorded completion: 1/1 features and 1/1 independent reviews pass, including a final review with no terminal blocker.", + "- satisfied [host-attested] Accepted validation: 1/1 terminal runs have eligible host evidence accepted by review.", + "- satisfied [host-attested] Canonical gate: \"node scripts/verify.mjs\" must have passing broad evidence accepted by review.", + "- satisfied [host-attested] Declared evidence: 1/1 declared obligations have accepted evidence on their declared host; observed gates do not claim a pass.", + "Assurance limitations:", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Artifacts as reported by Flow from caller declarations, not an exact or exhaustive Git delta:", + "- latest attempts: src/parser.mjs", + "- superseded attempts only: none reported" + ], + "reportedArtifacts": { + "latestAttempts": ["src/parser.mjs"], + "supersededAttemptsOnly": [] + }, + "summary": { + "fullReportAvailable": true, + "lines": [ + "Handoff format: 1", + "External action authority: not-granted", + "Goal: Make parse(input) safely handle null and preserve trimmed strings, changing only src/parser.mjs and keeping node scripts/verify.mjs and its script unchanged.", + "Closure: completed. Changed only src/parser.mjs with explicit null guard; unchanged node scripts/verify.mjs passed and independent review passed with no findings.", + "Progress: 1 of 1 features complete", + "Unfinished features: none", + "Nonblocking live findings: advisory 0. Historical findings: 0.", + "Assurance: completion supported", + "Assurance checks: 4 satisfied, 0 not applicable, 0 unsatisfied.", + "- Artifact paths and the canonical gate are caller declarations; Flow validates binding, not completeness or fitness.", + "- Goal alignment, scope discipline, evidence completeness, requirement coverage, test adequacy, and review substance remain model judgments.", + "- Freshness holds when review is accepted; an archive does not attest the current workspace.", + "Reported artifacts: 1 latest, 0 superseded only. Caller declarations, not an exact or exhaustive Git delta.", + "Full report is included in this close response." + ] + } + } +}