fix(evals): separate command integrity and prioritize delivery gates

docs/release-qualification.md

@@ -49,13 +49,13 @@ second external failure or an unallowed ask leaves a gap. Product and evaluator
 failures never activate reserves. Evaluator failure is `NOT VERIFIED`;
 persistence failure stops without a finalized report.
 
-Repository code owns the release catalog; persisted `catalog.json` must match.
-Versions 9.1.0 and 9.2.0 use 38 primary cells and eight reserves on GPT-6 Sol.
-Version 9.3.0 uses 48 primary cells and nine reserves. Version 9.4.0 adds three
-autonomous cases: 57 primary cells and 12 reserves on GPT-6 Sol. Version 9.5.0
-retains those twelve cases on GPT-6.1 Sol: 57 primary cells and 12 reserves.
-Version 9.6.0 adds five delivery cases: 72 primary cells and 17 reserves.
-Other versions use 96 primary cells and 18 reserves. Narrowed or merged reports
+Repository code owns catalog order; persisted `catalog.json` must match.
+Primary/reserve cells on GPT-6 Sol are 38/8 in 9.1.0 and 9.2.0, 48/9 in
+9.3.0, and 57/12 in 9.4.0. The 9.4.0 profile adds three autonomous cases.
+Version 9.5.0 keeps those twelve cases on GPT-6.1 Sol, with 57/12 cells.
+Version 9.6.0 adds five delivery cases, collects them first, and uses 72/17
+cells. Other versions use 96/18. Changed order requires a new freeze and
+approval. Existing campaigns stay immutable; narrowed or merged reports
 cannot qualify.
 
 Reported but ungated: reviewer findings/silent passes, refusals, operational counts,

evals/delivery-presentation.ts

@@ -5,6 +5,42 @@ type Assurance =
 	| "completion-supported"
 	| "completion-unsupported"
 	| "completion-not-claimed";
+const COMMAND_INTEGRITY = [
+	{
+		subject: "its script",
+		copulas: ["is", "was", "remains", "remained"],
+		predicate: "unchanged",
+		value: "script-unchanged",
+	},
+	{
+		subject: "its script and invocation",
+		copulas: ["are"],
+		predicate: "unchanged",
+		value: "script-and-invocation-unchanged",
+	},
+] as const;
+type CommandIntegrity =
+	| "not-claimed"
+	| (typeof COMMAND_INTEGRITY)[number]["value"];
+type CommandObservation = {
+	command: string;
+	exitCode: number | null;
+	integrity: CommandIntegrity;
+	qualification: "observation" | "does-not-claim-pass" | "claimed-pass" | null;
+};
+function commandIntegrityValue(
+	clause: string,
+): Exclude<CommandIntegrity, "not-claimed"> | null {
+	const text = clause.toLowerCase();
+	for (const rule of COMMAND_INTEGRITY)
+		if (
+			rule.copulas.some(
+				(copula) => text === `${rule.subject} ${copula} ${rule.predicate}`,
+			)
+		)
+			return rule.value;
+	return null;
+}
 type CurrentHandoffFacts = {
 	closure: (Closure | null)[];
 	assurance: (Assurance | null)[];
@@ -17,16 +53,7 @@ type CurrentHandoffFacts = {
 	}[];
 	assuranceCheckClaims: { count: number; status: "satisfied" }[];
 	unavailableProofPlatforms: string[];
-	observations: {
-		command: string;
-		exitCode: number | null;
-		unchangedInvocation: boolean;
-		qualification:
-			| "observation"
-			| "does-not-claim-pass"
-			| "claimed-pass"
-			| null;
-	}[];
+	observations: CommandObservation[];
 	unsupported: string[];
 };
 export function presentationText(text: string): string {
@@ -319,11 +346,11 @@ function parseCommandResult(
 	unterminatedQuote: boolean,
 ) {
 	const body = rawBody.trim().replace(/^(?::\s*|[—–]\s*|-\s+)/, "");
-	const invalid = {
+	const invalid: CommandObservation = {
 		command,
 		exitCode: null,
 		qualification: null,
-		unchangedInvocation: false,
+		integrity: "not-claimed",
 	};
 	if (unterminatedQuote) return invalid;
 	if (commands.some((other) => body.includes(other))) return invalid;
@@ -340,11 +367,11 @@ function parseCommandResult(
 	const exitCode =
 		rawExit.toLowerCase() === "unavailable" ? null : Number(rawExit);
 	if (exitCode !== null && !Number.isSafeInteger(exitCode)) return invalid;
-	const malformed = {
+	const malformed: CommandObservation = {
 		command,
 		exitCode,
 		qualification: null,
-		unchangedInvocation: false,
+		integrity: "not-claimed",
 	};
 	const metadata = value[4] ?? "";
 	if (
@@ -361,29 +388,42 @@ function parseCommandResult(
 		| "does-not-claim-pass"
 		| "claimed-pass"
 		| null = value[1] ? "claimed-pass" : value[2] ? "observation" : null;
-	let unchangedInvocation = false;
+	let integrity: CommandIntegrity = "not-claimed";
 	for (const qualifier of parts) {
 		if (
 			/^(?:this observation does not claim a pass|this does not claim the command passed)$/i.test(
 				qualifier,
 			)
 		) {
-			if (qualification !== "claimed-pass")
-				qualification = "does-not-claim-pass";
+			if (qualification === "claimed-pass") return malformed;
+			qualification = "does-not-claim-pass";
 		} else if (
 			/^(?:this (?:command|observation)|it) (?:passed|succeeded)$/i.test(
 				qualifier,
 			)
-		)
+		) {
+			if (
+				qualification === "observation" ||
+				qualification === "does-not-claim-pass"
+			)
+				return malformed;
 			qualification = "claimed-pass";
-		else if (
-			qualification === "claimed-pass" &&
-			/^Its script and invocation are unchanged$/i.test(qualifier)
-		)
-			unchangedInvocation = true;
-		else return malformed;
+		} else {
+			const claim = commandIntegrityValue(qualifier);
+			if (
+				!claim ||
+				(claim === "script-and-invocation-unchanged" &&
+					qualification !== "claimed-pass")
+			)
+				return malformed;
+			if (
+				integrity === "not-claimed" ||
+				claim === "script-and-invocation-unchanged"
+			)
+				integrity = claim;
+		}
 	}
-	return { command, exitCode, qualification, unchangedInvocation };
+	return { command, exitCode, qualification, integrity };
 }
 export function currentHandoffFacts(
 	text: string,

evals/delivery-scenario-checks.ts

@@ -1,4 +1,6 @@
+import { posix } from "node:path";
 import { z } from "zod";
+import { isArtifactPath } from "../src/domain/artifact.js";
 import { canonicalJson } from "./canonical-json.js";
 import {
 	currentHandoffFacts,
@@ -9,6 +11,75 @@ import {
 import type { ScenarioGradeInput } from "./grader-input.js";
 import { checkReviewerEvidenceAccess } from "./reviewer-access.js";
 
+function invokingScript(command: string): string | null {
+	const words: string[] = [];
+	let value = "";
+	let started = false;
+	let quote: "'" | '"' | null = null;
+	for (let index = 0; index < command.length; index++) {
+		const character = command[index] ?? "";
+		if (/\r|\n|\0/.test(character)) return null;
+		if (quote === "'") {
+			if (character === "'") quote = null;
+			else if (words.length < 2) value += character;
+			continue;
+		}
+		if (character === "\\") {
+			const next = command[index + 1];
+			if (next === undefined || /\r|\n|\0/.test(next)) return null;
+			started = true;
+			if (quote === '"' && !['"', "\\", "$", "`"].includes(next)) {
+				if (words.length < 2) value += "\\";
+			} else {
+				if (words.length < 2) value += next;
+				index++;
+			}
+			continue;
+		}
+		if (quote === '"') {
+			if (character === '"') quote = null;
+			else {
+				if (character === "$" || character === "`") return null;
+				if (words.length < 2) value += character;
+			}
+			continue;
+		}
+		if (character === "'" || character === '"') {
+			quote = character;
+			started = true;
+			continue;
+		}
+		if (/[;|&<>`$*?[\]{}~#()]/.test(character)) return null;
+		if (/\s/.test(character)) {
+			if (started && words.length < 2) words.push(value);
+			value = "";
+			started = false;
+		} else {
+			started = true;
+			if (words.length < 2) value += character;
+		}
+	}
+	if (quote) return null;
+	if (started && words.length < 2) words.push(value);
+	const runner = words[0];
+	const script = words[1];
+	if (
+		!runner ||
+		!script ||
+		!["node", "bun"].includes(runner) ||
+		script.startsWith("-")
+	)
+		return null;
+	if (
+		(runner === "node" && script === "inspect") ||
+		(runner === "bun" && !/\.(?:[cm]?[jt]s|[jt]sx)$/.test(script))
+	)
+		return null;
+	if (script.split("/").includes("..")) return null;
+	const path = posix.normalize(script);
+	return isArtifactPath(path) ? path : null;
+}
+
 export type DeliveryExpectation = Readonly<{
 	closure: "completed" | "deferred";
 	presentation: "summary" | "full" | "idle";
@@ -458,17 +529,28 @@ export function deliveryIssues(
 			issues.push(
 				"Claimed command pass lacks matching accepted complete source evidence.",
 			);
-		if (
-			result.unchangedInvocation &&
-			(result.command !== expected.gate ||
-				input.workspaceChanges?.kind !== "observed" ||
-				input.workspaceChanges.paths.some((path) =>
-					result.command.split(/\s+/).includes(path),
-				))
-		)
+	}
+	for (const result of facts.observations) {
+		if (result.integrity === "not-claimed") continue;
+		const script = invokingScript(result.command);
+		const immutableScript =
+			script !== null &&
+			input.workspaceChanges?.kind === "observed" &&
+			!input.workspaceChanges.paths.includes(script);
+		if (result.integrity === "script-unchanged") {
+			if (!immutableScript)
+				issues.push(
+					"Unchanged script claim lacks immutable workspace evidence.",
+				);
+		} else if (
+			result.qualification !== "claimed-pass" ||
+			result.command !== expected.gate ||
+			!immutableScript
+		) {
 			issues.push(
 				"Unchanged invocation claim does not match the gate and immutable script paths.",
 			);
+		}
 	}
 	if (facts.unsupported.length)
 		issues.push("Unsupported or conflicting current handoff assertions.");

evals/release-policy.ts

@@ -143,7 +143,6 @@ if (!prospectiveParsed.ok)
 const AUTO_RELEASE_CATALOG = prospectiveParsed.value;
 
 const deliveryParsed = parseCaseCatalog([
-	...AUTO_RELEASE_CATALOG,
 	...[
 		"delivery-summary-completed",
 		"delivery-summary-deferred",
@@ -161,6 +160,7 @@ const deliveryParsed = parseCaseCatalog([
 		minPassRate: 1,
 		reviewerPromotionRecordSha256: null,
 	})),
+	...AUTO_RELEASE_CATALOG,
 ]);
 if (!deliveryParsed.ok) throw new Error("Delivery release policy is invalid.");
 const DELIVERY_RELEASE_CATALOG = deliveryParsed.value;

tests/delivery-assurance-claims.test.ts

@@ -170,7 +170,7 @@ test("assurance-like quoted goal and registered command arguments remain in thei
 		{
 			command,
 			exitCode: 0,
-			unchangedInvocation: false,
+			integrity: "not-claimed",
 			qualification: "claimed-pass",
 		},
 	]);