fix(evals): distinguish deferred capture from reviewed completion

evals/cassette.ts

@@ -16,6 +16,7 @@
 // nothing that could be a credential is ever written into one, and the recording
 // host's absolute paths are replaced by a token rather than baked in.
 
+import { z } from "zod";
 import {
 	commandUsesManagedJUnitPath,
 	MANAGED_JUNIT_PATH,
@@ -277,6 +278,35 @@ export function capturedValidationIdentity(
 	return { id: marker.id, revision: marker.recordedRevision };
 }
 
+const CapturedResult = z
+	.object({
+		id: z.string().min(1).max(256),
+		scope: z.enum(["focused", "broad"]),
+		intent: z.literal("pass"),
+		passed: z.literal(true),
+		observed: z.literal(false),
+		recordedRevision: z.number().int().safe().positive(),
+		assertions: z
+			.array(
+				z
+					.object({ name: z.string().min(1), status: z.literal("passed") })
+					.strict(),
+			)
+			.optional(),
+		fullOutputDigest: z
+			.string()
+			.regex(/^sha256:[a-f0-9]{64}$/)
+			.optional(),
+	})
+	.strict();
+export function capturedValidationResult(
+	output: string,
+): z.infer<typeof CapturedResult> | null {
+	if (output.split("[flow-validation]").length !== 2) return null;
+	const parsed = CapturedResult.safeParse(validationMarker(output));
+	return parsed.success ? parsed.data : null;
+}
+
 function validationReport(output: string): string | null {
 	const assertions = validationMarker(output)?.assertions;
 	if (!Array.isArray(assertions)) return null;

evals/delivery-presentation.ts

@@ -48,6 +48,16 @@ type CurrentHandoffFacts = {
 	auxiliaryCounts: AuxiliaryCount[];
 	assuranceCheckClaims: { count: number; status: "satisfied" }[];
 	unavailableProofPlatforms: string[];
+	unavailableCommands: {
+		command: string;
+		targetPlatform: string;
+		hostPlatform: string;
+	}[];
+	independentReview: (
+		| { kind: "passed"; findings: "none" | "not-claimed" }
+		| { kind: "not-performed" }
+		| null
+	)[];
 	observations: CommandObservation[];
 	unsupported: string[];
 };
@@ -273,7 +283,7 @@ function closureStatement(
 	value: string,
 ): { closure: Closure; unavailablePlatform: string | null } | null {
 	const match =
-		/^(completed|complete|deferred|abandoned)(?: and archived(?: the Flow session)?)?(?: because (macOS|darwin|Linux|Windows) validation is unavailable)?$/i.exec(
+		/^(completed|complete|deferred|abandoned)(?: and archived(?: (?:the |this )?(?:current )?(?:Flow )?(?:session|workflow))?)?(?: because (macOS|darwin|Linux|Windows) validation is unavailable)?$/i.exec(
 			value,
 		);
 	if (!match) return null;
@@ -295,6 +305,10 @@ function commandStatusAssertion(text: string): boolean {
 		text,
 	);
 }
+function platformValue(value: string) {
+	const lower = value.toLowerCase();
+	return lower === "macos" ? "darwin" : lower === "windows" ? "win32" : lower;
+}
 function sentenceBoundary(text: string, start = 0) {
 	let quote: string | null = null;
 	for (let index = start; index < text.length; index++) {
@@ -334,6 +348,21 @@ function commandResultValue(line: string, commands: readonly string[]) {
 			.find((value) => value !== null);
 		if (!matched) continue;
 		const rawBody = matched[1] ?? "";
+		const availability =
+			/^ on (macOS|darwin|Linux|Windows); unavailable on this (macOS|darwin|Linux|Windows) host\.?$/i.exec(
+				rawBody,
+			);
+		if (availability)
+			return {
+				observation: null,
+				unavailable: {
+					command,
+					targetPlatform: platformValue(availability[1] ?? ""),
+					hostPlatform: platformValue(availability[2] ?? ""),
+				},
+				remainder: "",
+				source: line,
+			};
 		let boundary = sentenceBoundary(rawBody);
 		while (
 			boundary.end < rawBody.length &&
@@ -473,6 +502,8 @@ export function currentHandoffFacts(
 		auxiliaryCounts: [],
 		assuranceCheckClaims: [],
 		unavailableProofPlatforms: [],
+		unavailableCommands: [],
+		independentReview: [],
 		observations: [],
 		unsupported: [],
 	};
@@ -486,6 +517,8 @@ export function currentHandoffFacts(
 		}
 		const commandRecord = commandResultValue(line, observationCommands);
 		if (commandRecord) {
+			if ("unavailable" in commandRecord && commandRecord.unavailable)
+				facts.unavailableCommands.push(commandRecord.unavailable);
 			if (commandRecord.observation) {
 				facts.observations.push(commandRecord.observation);
 				if (commandRecord.observation.qualification === null)
@@ -507,9 +540,30 @@ export function currentHandoffFacts(
 		) {
 			facts.unsupported.push(line);
 		}
+		if (/^(?:[^:]+:\s*)?(?:node|bun) \S+.*\bunavailable\b/i.test(line))
+			facts.unsupported.push(line);
 		for (const segment of line.split(/;|\.\s+(?=[A-Z])/)) {
 			const claim = segment.trim().replace(/\.$/, "");
 			if (!claim) continue;
+			const review = /^Independent review(?::|\s)\s*(.*)$/i.exec(claim);
+			if (review) {
+				const value = review[1] ?? "";
+				facts.independentReview.push(
+					/^(?:was )?not performed$/i.test(value)
+						? { kind: "not-performed" }
+						: /^(?:has |was )?(?:passed|passed with no findings)$/i.test(value)
+							? {
+									kind: "passed",
+									findings: /with no findings$/i.test(value)
+										? "none"
+										: "not-claimed",
+								}
+							: null,
+				);
+				if (facts.independentReview.at(-1) === null)
+					facts.unsupported.push(claim);
+				continue;
+			}
 			const handoff = closureProgressValue(claim);
 			if (handoff) {
 				if ("closure" in handoff) facts.closure.push(handoff.closure ?? null);