diff --git a/.github/workflows/incoming-context-eval.yml b/.github/workflows/incoming-context-eval.yml new file mode 100644 index 0000000000000..05931646e6910 --- /dev/null +++ b/.github/workflows/incoming-context-eval.yml @@ -0,0 +1,40 @@ +name: Incoming Context Eval + +on: + pull_request: + paths: + - "evals/incoming-context/**" + - ".github/workflows/incoming-context-eval.yml" + push: + branches: [main] + paths: + - "evals/incoming-context/**" + - ".github/workflows/incoming-context-eval.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: incoming-context-eval-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + name: Incoming context behavior tests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: evals/incoming-context + - name: Install evaluator dependencies + run: | + python3 -m venv "$RUNNER_TEMP/incoming-context-venv" + "$RUNNER_TEMP/incoming-context-venv/bin/python" -m pip install -r evals/incoming-context/requirements.txt + - name: Test evaluator without model inference + working-directory: evals/incoming-context + env: + TIKTOKEN_CACHE_DIR: ${{ runner.temp }}/tiktoken + run: '"$RUNNER_TEMP/incoming-context-venv/bin/python" -m unittest discover -p "test_*.py"' diff --git a/evals/incoming-context/.gitignore b/evals/incoming-context/.gitignore new file mode 100644 index 0000000000000..bcec818032af8 --- /dev/null +++ b/evals/incoming-context/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +.venv/ +cases*.json +analysis.json +runs/ +blind*/ +*.local.json diff --git a/evals/incoming-context/README.md b/evals/incoming-context/README.md new file mode 100644 index 0000000000000..f76883c684c67 --- /dev/null +++ b/evals/incoming-context/README.md @@ -0,0 +1,119 @@ +# Local incoming-context evaluation + +Offline evaluation for removing text from a new tool result before a model first sees it. This does not enable production pruning or modify provider history. + +## Run synthetic checks + +```sh +cd evals/incoming-context +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +.venv/bin/python -m unittest discover -p 'test_*.py' +``` + +The Incoming Context Eval workflow runs these behavior tests on changes to this directory, without weights or model inference. + +Local model inference additionally needs the Apple Silicon `mlx-decision` runtime and separately downloaded weights. Inspect `evaluate.py run --help` for model, batching, deadline and size options. HTTP inference accepts loopback endpoints only, bypasses environment proxies, and refuses redirects. + +## What the selector sees + +One scoring request is one contract. `REQUEST_VERSION` (`incoming-state-v2:keep-prompt-v2`) names the state and prompt. Comparisons also match the causal-context, selection-policy and protection versions. The current eligibility rules are `causal-context-v4`, `selection-evidence-v4` and `json-span-v4-depth64+evidence+source-v2`; results with older or missing identities are not current measurements. `rerender`/`sweep` preserve the input identities and never stamp newer ones. + +- `state.recent_user_context` — every sourced human instruction strictly before the result, oldest first, verbatim. `causal_context` collects `message` records with role `user`, skips `assistant` and tool records entirely, and refuses harness envelopes, developer-role messages, opaque content and unknown wrappers instead of promoting them to instructions. Compacted history contributes only retained, client-authored user inputs that carry verified provenance; otherwise the case abstains for an ambiguous goal. Nothing is summarized. Generated handoff headers and historical replay wrappers make the goal ambiguous: omitted history cannot be replaced by generated text presented as a human instruction. Such cases retain their original output without scoring. +- `state.tool_call` — the resolved originating call: name plus full command or arguments body, never truncated. +- `state.candidate_tool_output.blocks` — every block of the pending result, including protected boundary and diagnostic blocks, each with a positional `index`, its text-leaf `path` and its exact text. Question batching groups questions only; a block is never hidden from the state because another request asks about it. +- `questions` — one `noul` question per **eligible** block, using one fixed prompt for both batching modes. A block is eligible only when it is neither a protected boundary/diagnostic block nor already retained by a protection rule. The prompt asks about one block at a time and states that no sibling is guaranteed to survive, so redundancy is never by itself a reason to drop; a later source review, a requested audit and negative evidence are named as reasons to keep. + +There is no assistant plan, task-progress or conversation-summary field, because the source records never contribute assistant content to the goal and generating one would need inference the evaluator does not have. The available instructions and the full originating call are the whole causal state; adding a summary would be a new context schema with its own failure modes, and the defect this revision repairs did not require it. + +Budgets are charged the real serialized request, including HTTP's model field. `--batch-tokens` measures the complete request — the full state plus that group's questions — and a question is added only while the whole request still fits. When even one question would exceed the budget the case abstains as `oversize` and nothing is sent: a state cannot be shrunk by regrouping, and a request is never shortened to fit it. `--max-tokens` is the transport's independent ceiling for the same request. A result that would expand never replaces the original, and `apply_incoming` re-checks that the historical prefix bytes are unchanged. + +`--prompt v1|v2` selects the question text; `v1` exists only so the prompt change can be measured apart from the state change. The identity of the chosen text is in every receipt. + +## Retention that happens before any question + +Three rules can keep a block without asking a model. They only ever add retention, they are recorded per block with an explicit reason, and none of them is a saving: + +- **`requested source`** — a fully recognized read selects a file named by a sourced read/review/audit instruction. Supported forms include read-like tools with one path, literal `exec_command` arguments, repeated `text(await tools.exec_command({...}))` calls, and read-only shell lists using `;` or `&&`. The shell subset accepts explicit file operands, `cd ... &&`, numeric `head`/`tail` limits and fixed `sed -n 'N,Mp'` ranges. If any selected file is requested, the entire result is kept: concatenated output is never attributed to individual commands. Dynamic JavaScript, unknown commands, mutation, pipes, redirects, substitutions and globs make no source-retention claim. The recorded experiments used the older single-file rule; this broader recognition has synthetic coverage but has not been measured against those private recordings. +- **`json_container`** — a complete JSON object or array decoded with the stdlib `json.JSONDecoder.raw_decode` at a _standalone_ start (line start, or after `=`/`:`/`>`/`|`). Scanning resumes after each decoded value, so JSONL and several printed values are all found. Scalars are never containers, and inline brackets in argument lists (`execve(..., ["/a"], 0x..)`) are not standalone starts. Every block the container spans on that text leaf is kept, with the span recorded in the block's own coordinates. +- **`json_container_unverified`** — a JSON-shaped start that cannot be decoded. A leaf that opens such a container keeps the whole leaf (a truncated document); a mid-text one keeps only the block that carries it. Code braces (`if (x) {`, `fn main() {`) are not JSON-shaped and never match. + +Structural retention is a property of the bytes, not a judgement about semantics: nothing is inferred from text that merely looks like an audit or a schema. Requested evidence terms keep their earlier narrow behaviour. + +Hard-kept and protected blocks are never sent to the selector, so their decision row carries `p_keep: null`, `proposed: null`, `hard_keep: true` and a `hard_reason`/`hard_source`. A `null` proposal is not a keep: `proposed` is only ever `drop`/`keep` for a block the model actually scored, and `effective` is the only field that says what happened. If no eligible block remains, nothing is sent and the original result is retained. + +## Evaluate private recordings + +Supply an analysis manifest containing `census.sessions` entries with `origin: "T3 Code"` and a local `path` to each rollout JSONL. Generate cases with `evaluate.py prepare --analysis analysis.json --out cases.json`. Pass `--cases cases.json` explicitly to subsequent commands. Keep source recordings, manifests, model weights, result files and blind packets outside Git. + +Prepared cases pin the exact source record and all preceding bytes. Source reads revalidate those hashes even on cache hits, and a recording changed during evaluation is rejected. Legacy manifests lack the full-prefix hashes; their context is still reconstructed from the current source, and frozen scores cannot be reused after that context changes. + +The evaluator reconstructs causal instructions and poll origins from records preceding each result. Missing or ambiguous context retains the whole result without a scoring request, including unreadable attachments or corrections, malformed history, partially sourced compaction and interactive stdin absent from the originating command. Named evidence adds retention constraints. These heuristics are conservative and do not prove semantic safety. + +The scoring state represents text only. Results with hidden JSON-envelope metadata, nontext siblings, or unrepresented diagnostic fields retain their full output without inference. Deeply nested JSON that cannot be decoded also retains its text. Truncated responses, duplicate answer fields, and answer IDs that differ from the request are unusable. Case timing includes source loading, context reconstruction, rendering and accounting; HTTP timeouts are bounded by the remaining deadline, and an expired result is retained whole. MLX initialization failures abstain, and changing the model or load options reloads the runtime. These rules can reduce coverage and change timing compared with older measurements. + +`run` records model input, scores, effective decisions, elapsed time, per-request state and request token counts, and local prefix receipts. Totals separate protected, hard-kept, context-abstained and eligible blocks and tokens, count questions actually asked against eligible blocks, and count every scoring attempt including failed ones. `sweep` and `rerender` derive new thresholds from frozen scores; guards count as zero effective drops. Threshold fitting with `equalize.py` accepts matching development cohorts only. Stability comparisons check source and context identity, and latency groups separate runtime and budget configurations. Changes to context, prompt, state or batching require fresh inference. `blind_packet.py` validates every candidate against the same source blocks and causal context, checks retained-output hashes, and randomizes candidate order independently for each case. Invalid or empty builds leave an existing review packet untouched. Save judgments before opening the separate identity map. Reviewer-only enrichment must not be represented as model input. + +## Frozen results cannot inherit a newer policy + +Every result records `request_version`, `state_version`, `prompt_version`, `policy_version` and `protection_version`. `sweep`/`rerender` reuse a frozen result's hard-keep records only when its context, selection-policy and protection identities are current. A result recorded under another policy (or before this version existed) is reported with `protection_current: false` and `stale_protection`, keeps its original output at every threshold, and reports no derived savings. Old artifacts are never rewritten and never silently upgraded: re-thresholding a v2 measurement cannot loan it the JSON or source rules it was not measured with. + +## Current limits + +Synthetic checks cover causal cutoffs, poll resolution, source changes, retention, failure paths, accounting, the whole-result state, request budgets, prefix invariance, version provenance, JSON span decoding, and literal composed reads. They cannot cover a real provider interception point or real provider cache hits. No model is approved for live pruning. + +The recorded development sample contains 40 checkpoints; its prior context census abstained on 29. The measured protection policy had 15 hard-kept blocks (13 requested evidence, 2 structural), 58 eligible blocks and only one case with an effective removal. The sampled test split was exhausted, so the fresh sample used 8 never-measured development checkpoints and is **not** a holdout. `requested source` made no claim on real calls because their harness wrappers contain composed commands. These counts and measurements have not been rerun under the current eligibility rules. Private recordings and judgments remain outside Git and do not establish semantic safety. + +### Experimental decoded-line adapter + +`output_lines.decode(raw, producer)` accepts the explicitly identified bare +`t3.exec.v1` envelope with exactly one completed (`exit_code`) or running +(`session_id`) status form. `render(decoded, keep)` accepts half-open original +line ranges, coalesces omissions, and changes only the JSON output string token. +`omitted_text` verifies the original digest before recovering an omitted range. +No-op rendering preserves the original envelope exactly; unknown shapes abstain. +The producer identifier is a caller assertion, not inferred provenance. + +This adapter is not connected to the decision engine or a live provider. +Markers identify a source digest and line range; a production retrieval service +and event locator are still required. Character counts are not token savings, +and the evaluator must enforce token non-expansion before applying any selection. + +`line_experiment.py` provides offline `freeze`/`run`/`compare` and +`freeze-edits`/`run-edits`/`compare-edits` commands for per-line and complete-edit +experiments. Comparisons write JSON. Real transcript manifests and results stay +outside the repository. Complete-edit v2 preserves the full parent result, including headings and sibling +outputs, while replacing only the selected envelope text. Frozen plans bind parent +identity, leaf coordinates, required evidence, exact request serialization order, +and candidate token counts. Model approval is recorded separately from effective +acceptance; an expanding replacement is always rejected. Controls without verified +required lines are refused. + +Frozen context, calls and request grouping are verified +before dispatch; the request cap is enforced before each call. Local MLX calls +have no enforced wall-clock timeout. Model-loading failures are recorded as +rejections, not omitted from the result. + +JSON admission uses one iterative structural inspector with a maximum container depth of +64 (the root counts as one). Strings and escaped delimiters do not add depth. Over-depth +output stays whole; envelope adapters abstain, and source records and call arguments are +unavailable rather than decoded. Defensive decoder catches remain. The protection version +owns this eligibility change; causal extraction, source matching, prompt and frozen-input +formats retain their existing versions. Changed leaf or request identities require refreezing. +Historical measurements in RESULTS have not been rerun under this policy. + +Complete-edit runs and block receipts carry `execution-v1` evidence owned by that invocation. +Real dispatch records the coupled loaded model/configuration/receipt, including on cache +hits. A load alone does not attest execution. Injected scorers, patched loaders, preflight +abstentions, failed loads and unattested HTTP responses cannot borrow a cached receipt. +Receipts snapshot requested controls, actual applied/refused load options, runtime versions, +machine/Python metadata and dispatch/failure counts. They are compact evidence, not a request +journal; source paths and model labels are not weight-content hashes. Complete-edit runs do +not warm up. Block runs report their requested warmup and measured warmup duration separately. + +`compare-edits` preserves each execution receipt and reports `comparable`, +`different-controls`, or `unverified`, with differing controls or missing evidence stated. +Missing/unknown runtime evidence, failures and historical receipts remain descriptive data, +not controlled measurements. Inconsistent requested/loaded sources or multiple runtime +configurations are rejected. Machine differences qualify timing comparisons. Model identities +are expected comparison dimensions, while load controls and runtime versions must match. diff --git a/evals/incoming-context/RESULTS.md b/evals/incoming-context/RESULTS.md new file mode 100644 index 0000000000000..5f6076c5de8bd --- /dev/null +++ b/evals/incoming-context/RESULTS.md @@ -0,0 +1,396 @@ +# Local evaluation checkpoints + +These recorded checkpoints precede the current `causal-context-v3`, +`selection-evidence-v4` and `json-span-v3+evidence+source-v2` repairs. Current case +timing includes source loading, context reconstruction, rendering and accounting. +Coverage, savings and latency have not been rerun +under the combined rules; the figures below describe the named historical +checkpoints, not validation of the current evaluator. + +## Recorded v3 protection-policy checkpoint + +Development experiment; no candidate is approved for live pruning. + +### What the recorded revision added + +Retention that runs before any question, with an explicit reason per block: + +- **Complete JSON object/array spans.** A stdlib `json.JSONDecoder.raw_decode` is attempted only + at standalone container starts (line start, or after `=`/`:`/`>`/`|`); scanning resumes after + each decoded value. Scalars are never containers, and inline argument brackets + (`execve(..., ["/a"], 0x..)`) are not starts. Every block a container spans on one text leaf + is kept, with the span recorded in that block's coordinates. A JSON-shaped start that fails to + decode keeps its whole leaf when it opens the leaf, and only its own block when it is mid-text. + Nothing is retained because text merely looks like an audit or a schema. +- **Requested source.** Exactly one file must resolve from the originating call (single + `path`/`file_path`/`file`, or a one-operand `cat`/`bat`/`nl`/`head`/`tail` with no composition) + _and_ be named by a sourced instruction with a read/review intent. Every unresolved shape + abstains from the claim instead of guessing. +- **No question about a retained block.** Hard-kept and protected blocks are never sent, so an + unscored block records `p_keep: null`, `proposed: null`, `hard_keep: true`, and no usage. A + `null` proposal is the absence of a decision, not a keep; `effective` remains the only field + that states what happened. With no eligible block left, nothing is sent. +- **Protection identity.** Results record `POLICY_VERSION` (`selection-evidence-v2`) and + `PROTECTION_VERSION` (`json-span-v2+evidence+source-v1`). `sweep`/`rerender` reuse a frozen + result's hard-keep records only when its protection identity is current; anything else is + reported `protection_current: false` and keeps its original output at every threshold. Frozen + v1/v2 artifacts therefore yield 0 % when rethresholded instead of silently inheriting rules + they were never measured with. +- **Accounting.** Totals partition blocks into protected / hard-kept (structure, evidence, + source) / context-abstained / eligible; they also count eligible tokens, questions actually + asked against eligible blocks, every scoring attempt including failures, and the protected + share of tokens. + +### Development regressions (this sample is consumed) + +Both datasets below were already used to develop v2, so they are regression checks, not fresh +evidence. Same prompts (`keep-prompt-v2`), same thresholds, same batching, same local weights. + +| Corpus | Config | Net saved | Dropped / proposed | Scored | Asked questions | Calls | Case p50 | p95 | +| ------------------- | ---------- | --------: | -----------------: | -----: | --------------: | ----: | -------: | -----: | +| dev40 v2 (frozen) | d1-3B 0.25 | 0 % | 0/2 | 73 | 73 | 6 | 5.4 ms | 3.60 s | +| dev40 v3 | d1-3B 0.25 | 0 % | 0/0 | 58 | 58 | 5 | 5.3 ms | 2.76 s | +| dev40 v2 (frozen) | pplx 0.10 | 11.03 % | 48/48 | 73 | 73 | 6 | 5.3 ms | 7.94 s | +| dev40 v3 | pplx 0.10 | 10.71 % | 47/47 | 58 | 58 | 5 | 5.3 ms | 5.57 s | +| holdout v2 (frozen) | d1-3B 0.25 | 0 % | 0/0 | 11 | 11 | 4 | 0.53 s | 3.51 s | +| holdout v3 | d1-3B 0.25 | 0 % | 0/0 | 8 | 8 | 2 | 8.2 ms | 3.03 s | +| holdout v2 (frozen) | pplx 0.10 | 3.67 % | 3/4 | 11 | 11 | 4 | 0.95 s | 6.77 s | +| holdout v3 | pplx 0.10 | 3.67 % | 3/3 | 8 | 8 | 2 | 8.9 ms | 5.61 s | + +Reading it: the protection removes 15 of 73 questions on dev40 and 3 of 11 on the holdout. +pplx dev40 savings decrease from 11.03% to 10.71%; holdout savings remain unchanged. +The holdout proposal retained by requested evidence is now never asked, which is why +4/3 becomes 3/3 with the same removed blocks. d1 still +proposes nothing at 0.25: its only dev40 proposals were evidence blocks, and evidence blocks are +no longer scored. + +The decisive holdout case is unchanged. v3 removes exactly the same blocks (8, 9, 12 of 53) as +v2, including the two the root judged a major source-evidence loss. Structural retention did not +touch them (they are Rust source, not JSON) and requested-source abstained on the call shape, so +**this revision does not fix that regression class**; it makes the policy narrower and cheaper and +the bookkeeping honest, not safer on that case. + +### Requested source: implemented but inert on this workload + +Every originating call in the measured corpora is a harness wrapper around composed commands +(`text(await tools.exec_command({cmd: "..."})); text(await tools.exec_command({cmd: "..."}))`), +often with several commands per call. No exact single-file selection can be proven from that +shape, and mapping blocks back to one command is impossible because the outputs are concatenated. +The rule therefore made **0 claims** (`requested_source_blocks = 0`) on dev40, the previous +holdout and the fresh set. It is covered by synthetic tests; it is not evidence on real calls. A +provider call shape that exposes a single file read was required by that revision. +The current evaluator recognizes a bounded subset of literal wrapper calls and read-only +shell lists, retaining the whole combined result when a requested file is selected. The +historical zero-claim counts above have not been rerun with that recognition. + + + +### Fresh, never-measured checkpoints (not a holdout) + +The untouched `test` split has no unused checkpoint with verified causal context: of its 24 +unused checkpoints, 16 abstain as `goal_ambiguous:compacted_history` and 8 as +`goal_missing:no_human_instruction`, and an abstained checkpoint sends no request. The previous +holdout consumed every context-available test checkpoint. Rather than reuse one, selection moved +to **unused sessions of the development split**: 23 sessions / 92 checkpoints were never +measured by any prior artifact, 16 have verified context and 8 of those have eligible blocks. The +rule and the case file were frozen before any v3 score existed (`runs/evidence-v3/freeze.json`): + +``` +fresh-cases.json sha256 073475037d9c2431558745f81fe87672b33b879b7b10eed9adde3c119b8c9c2e +fresh-manifest.json sha256 8a085345dd64886ec49e19a3212f0f93d8ef128fa57cbf8341510ba7e6aabbe9 +``` + +8 checkpoints from 8 sessions, 50,891 tokens, 139 blocks, 68 eligible, 3 mixed JSON + trace, one +with no hard-kept block at all. Rule: prefer mixed JSON+trace, then most eligible blocks, at most +one checkpoint per session, then size extremes. + +| Configuration | Net saved | Dropped / proposed | Asked | Calls | Evidence kept | Structural kept | Case p50 | p95 | +| ----------------------------- | --------: | -----------------: | ----: | ----: | ------------: | --------------: | -------: | ------: | +| Keep original (control) | 0 % | 0/0 | 0 | 0 | 6 | 3 | 5.2 ms | 14.0 ms | +| Exact duplicate (control) | 0 % | 0/0 | 0 | 0 | 6 | 3 | 5.1 ms | 14.0 ms | +| d1-3B, 0.25 | 5.43 % | 8/8 | 72 | 8 | 6 | 3 | 9.8 s | 19.3 s | +| pplx-decider-flash-v1.2, 0.10 | 5.46 % | 7/7 | 72 | 8 | 6 | 3 | 11.5 s | 18.6 s | + +The recorded census lists 68 eligible blocks while this table lists 72 asked questions. +That discrepancy has not been reconciled against the private run artifacts; these counts +must not be treated as a validated coverage measurement. + +Three of the eight cases lose content (one under d1, two different ones under pplx); the other +five keep everything, so this set does show positive removals next to keep-all behaviour. The +largest removal is 9,180 → 8,072 tokens. No request failed and none was refused as oversize. + +This is a **fresh development sample**, not a holdout and not an estimate of production savings: +it was selected partly for structural contrast, the sessions are development-split, and eight +checkpoints cannot bound risk. + +### Bounded cache-option comparison (d1 only) + +The installed runtime documents `prefix_cache_gb` for d1, Clef, Base Labs and H2O-Lightning, so +one case (17,922 tokens, two requests) was run on d1 with the option on and off, twice each, and +nothing else changed: + +| `prefix_cache_gb` | case total | request 1 | request 2 | decisions | +| ----------------- | ----------------: | ----------------: | ------------------: | ---------- | +| 0 (off) | 28.61 s / 28.69 s | 15.30 s / 15.39 s | 13.23 s / 13.23 s | `e3a29f6d` | +| 2.0 (default) | 17.26 s / 17.24 s | 15.35 s / 15.31 s | **1.84 s / 1.87 s** | `e3a29f6d` | + +The second request reuses the first request's state prefix, so the same work is 7× faster; the +case total drops by 40 %. Decisions are byte-identical (`after_sha256`), and the historical +prefix is unchanged in every run (`prefix_bytes` 411,147 with the same `prefix_sha256`). Cache +reuse is therefore a runtime property only, never a selection rule. pplx does not accept the +option: the receipt records `load_options_refused: [noul_floor, prefix_cache_gb]` and it was not +forced. No cache layer was invented. + +### Blind packet for this revision + +`blind-evidence-v3/` (local scratch) holds three anonymous packets built from these runs: + +- `packet/` — the three fresh cases that actually lose content (one under d1, two others under + pplx), 3 cases × 4 labels = 12 labels, removed block range 0–8; +- `fresh-intact/` — three fresh cases where nothing is removed, 12 labels, so a reviewer can + check that controls are intact; +- `dev40/packet/` — the single large development case that carries the whole dev40 saving, 4 labels. + +No candidate identity, model, score, threshold, latency or savings total is in a packet; the +identity maps were written by the builder and have not been opened. Root owns the judgments. + +### Accounting on dev40 under the final policy + +Blocks partition exactly: 204 protected + 15 hard-kept (2 structural, 13 requested evidence) + +106 context-abstained + 58 eligible = 383. The selector was asked 58 questions in 5 calls, so +`unasked_eligible_questions` is 0 and `eligible_blocks - asked_questions` can never be negative. +`failed_blocks` is 0, not the 106 that a naive "abstain" count reports, because context +abstention is a policy keep rather than an inference failure; `failed_requests` is also 0, and +`actual_calls` matches `completed_requests` plus failures. Every unscored block has +`p_keep: null`, `proposed: null` and no usage, so a guard can never be read as a model keep. + +### Decision and next gate + +Do not promote either candidate. This revision makes retention narrower, cheaper and +auditable; it does not make the known failure class safe: + +- the holdout case that the root judged a major source-evidence loss still loses exactly the same + three blocks, because the requested-source rule cannot resolve the real call shape; +- dev40 savings still come from one large result, and the fresh sample has only eight selected + checkpoints, so these figures describe workloads rather than a validated policy; +- the fresh sample is development-split and selected, not a holdout. + +Before promotion: a provider call shape that exposes a single file read (so requested-source can +fire), a provider-owned interception point, and context-available held-out sessions that were +never inspected. The structural rule is the only part of this revision that changes decisions on +real data, and it changes 2 of 383 dev40 blocks and 0 of 83 holdout blocks. + +## Prior checkpoint: repaired full-result state (v2) + +Development experiment; neither configuration is approved for live pruning. + +### What this revision repairs + +The previous evaluator showed the selector only the blocks it was currently asking about. It asked +about a group at a time but built each request's state from that group alone, and the group excluded +protected blocks. A protected sibling (boundary or diagnostic) was therefore invisible to the model +judging a neighbouring block, and blocks questioned in another request were invisible in this one. +The docstring claimed the state carried the whole pending result; it did not. + +Two further defects were found while repairing it, both in the selector contract rather than the +state: + +- The prompt said to answer "no" when the retained result already shows the block's content, but + the selector only ever sees the full original result. Nothing guarantees a sibling is kept, so + joint drops could erase every copy of a fact. The shipped prompt now answers per block and states + that no sibling is a witness. No dependency-group or witness machinery was added. +- The batch budget was charged an estimate of the target blocks, not the request that is actually + sent. It now measures the complete serialized request — state plus that group's questions — + and abstains when even one question exceeds the budget instead of repeating an oversized state + once per block. + +A separate audit of the causal context answered the other half of the question: the source never +contributes assistant content, so no assistant plan or task-progress state exists and none is +supplied. `causal_context` collects every readable `user` message before the checkpoint (1 to 6 per +scored development case, all with `human` provenance) plus the full originating call, and refuses +harness envelopes or developer-role messages rather than promoting them. A summary layer would add a +new failure mode, not recover missing state. See [README.md](README.md#what-the-selector-sees). + +Every measured result records `request_version`, `state_version`, `prompt_version` and the prompt +digest at the top level and inside each case receipt. A run reports `request_version` only when every +case carries the run's own identity; `rerender`/`sweep` carry the input's version through. Frozen v1 +scores therefore stay labeled v1. + +## Separating the state change from the prompt change + +State and prompt changed together, so the two effects are not identified from a single before/after. +The evaluator keeps both prompt texts (`--prompt v1|v2`) to measure them separately. The subset is +the 5 development cases that have verified context and at least one unprotected block — the only +cases where a selector runs — and every cell below uses the same cases, thresholds (d1 0.25, +pplx 0.10), and model settings. The frozen v1 run used an 8,000-token target-based batch budget; v2 uses a 32,000-token complete-request budget. Grouping and visible state therefore differ across the frozen-state comparison. Only the two v2-state prompt cells share the new budgeting contract. + +| Cell (37,637 tokens, 73 unprotected blocks) | d1 drops / proposed | d1 saved | pplx drops / proposed | pplx saved | +| ------------------------------------------- | ------------------: | -------: | --------------------: | ---------: | +| v1 prompt + v1 target-only state (frozen) | 47/49 | 39.66% | 37/37 | 31.13% | +| v1 prompt + v2 full-result state | 43/49 | 39.26% | 46/46 | 38.83% | +| v2 prompt + v2 full-result state (shipped) | 0/2 | 0% | 48/48 | 40.43% | + +Reading it: at a fixed old prompt the state change alone leaves d1 removing almost as much (47 → 43 +blocks) and moves pplx up (37 → 46). The prompt change is what collapses d1's removals (43 → 0, both +remaining proposals retained as requested evidence) and nudges pplx further (46 → 48). The whole prompt revision is associated with d1 retaining more; the experiment does not isolate which clause caused it. +These are observational cells with a small sample: they bound the confound, they do not establish +causal effects, and the shipped rows are the only ones that describe the current contract. + +## Development sample under the shipped contract + +40 checkpoints from 40 private sessions, 137,982 tokens, 383 blocks (179 unprotected), 29 cases +abstaining for missing causal context, 204 protected blocks. Local MLX inference, no hosted selector +or judge. + +| Configuration | Net tokens saved | Case p50 | p95 latency | Cases with removals | Dropped / proposed | +| ----------------------------- | ---------------: | -------: | ----------: | ------------------: | -----------------: | +| Keep original (control) | 0% | 5.7 ms | 12.7 ms | 0 | 0/0 | +| Exact duplicate (control) | 0% | 5.9 ms | 12.5 ms | 0 | 0/0 | +| d1-3B, 0.25 | 0% | 5.4 ms | 3596.2 ms | 0 | 0/2 | +| pplx-decider-flash-v1.2, 0.10 | 11.03% | 5.3 ms | 7941.4 ms | 1 | 48/48 | + +d1 proposed two removals and the requested-evidence guard retained both, so it saved nothing. pplx +removed 48 of 49 scored blocks in the single 17,922-token case, leaving 2,706 tokens; that one case +is the entire saving. Coverage is 73 of 179 unprotected blocks scored, 0 oversize refusals and 0 +failed requests. + +Per-request payload (identical for both candidates because grouping depends only on the config): +6 requests, 74,569 whole-request tokens, largest 24,461; block latency p50 477.7 ms (d1) and +1020.3 ms (pplx). The 17.9k-token case costs 17.4 s (d1) and 62.1 s (pplx), and pplx's p99 case +latency is 42.5 s. + +For reference, the rejected checkpoint measured 10.82% (d1) and 8.49% (pplx) at the same thresholds +with a target-only state and the older prompt text. That comparison mixes both changes and is not a +measure of the state repair; the subset table above is the one to read. + +## Fresh holdout + +Selection was frozen before any holdout score was read (`select_holdout.py`, rule in the file), and +the run configuration was frozen with it (`runs/state-v2/config.frozen.txt` and `run_all.sh`): `split +== test`, sessions disjoint from the development subset every prior scored run used (0 of 17 test +sessions overlap), every context-available case with at least one unprotected block, the two +abstention cases with the most unprotected blocks, then the smallest and largest zero-target cases. +Result: 8 cases from 5 sessions, 35,342 tokens, 83 blocks (46 unprotected, 37 protected). Two cases +(33 of the 46 unprotected blocks) abstain for missing causal context and stay in the totals. + +| Configuration | Net saved | Dropped / proposed | Evidence-retained | Case p50 | p95 | +| ----------------------------- | --------: | -----------------: | ----------------: | -------: | ------: | +| Keep original | 0% | 0/0 | 0 | 5.0 ms | 14.4 ms | +| Exact duplicate | 0% | 0/0 | 0 | 4.7 ms | 14.2 ms | +| d1-3B, 0.25 | 0% | 0/0 | 0 | 531.4 ms | 3.51 s | +| pplx-decider-flash-v1.2, 0.10 | 3.67% | 3/4 | 1 | 951.6 ms | 6.77 s | + +11 of 46 unprotected blocks received model scores (4 requests, 16,935 request tokens, largest 7,224). Two additional blocks were retained by evidence constraints, and 33 blocks belong to the two abstained cases. pplx removed 3 blocks from one 5,368-token case and +one more proposal was retained as requested evidence; d1 proposed nothing. The holdout is far too +small to estimate performance, and 3.67% of 35,342 tokens is three blocks, not a result. + +Disclosure on sequence: the holdout was first executed with the earlier prompt text, before the +review identified the redundancy clause as a defect; that execution removed nothing for either +candidate. The prompt fix is an instruction-contract correction, decided without inspecting holdout +scores, and the holdout was rerun with the frozen selection and model settings. The frozen v1 run used an 8,000-token target-based batch budget; v2 uses a 32,000-token complete-request budget. Grouping and visible state therefore differ across the frozen-state comparison. Only the two v2-state prompt cells share the new budgeting contract. Only the shipped-contract +artifact is retained, and the earlier aggregate is recorded here rather than in a file. + +## Prior blind review (v1 contract) + +Root reviewed six anonymous development cases under the earlier contract, saved judgments before +opening the identity map, then unblinded. Three cases had removals; two abstained for unavailable +context. d1 lost graph ordering, host/native closure, environment precedence and relevant tests from +a file explicitly requested for source review, and separately removed executable-control outcomes and +bootstrap-scope records needed for a diagnostic report: two major losses. pplx retained the +structured diagnostic outcome, controls and primary failure while trimming repetitive C/C++ discovery +traces: acceptable for that report, medium confidence. Intact controls lost nothing. + +Those judgments apply to the v1 contract and cannot be transferred: scores for the same blocks differ +under the shipped contract. + +## Blind review for this revision + +`blind-state-v2/` (local scratch) holds one anonymous development case with 4 labels — two candidates +and two controls — seed 20261105, restricted to removal-bearing cases because only one development +case still has an effective removal. One label removes 48 of 53 blocks; the others remove nothing. +No candidate identity, model, score, threshold, latency or total-savings figure is in the packet; +per-block token counts and per-label removed-block counts appear only as block geometry. Save +judgments before opening the identity map. + +Root saved judgments before opening the identity map. The removal-bearing label lost the remaining executable-control outcomes and empty product-call audit needed for the report: major evidence loss. Its identity was inferable from previously disclosed aggregate behavior, so this is anonymous-text review, not an independent model-blind study. The retained labels lost no evidence. + +Root also directly inspected the holdout's three removed blocks after seeing the model identity. One removed the serialization flag and serde skip/default behavior of a type explicitly named in the task; another removed relevant roundtrip test construction. This is a source-evidence failure, not validated 3.67% savings. This inspection was unblinded. + +## Operational note + +One intermediate pplx run failed a request with a transient Metal error +(`kIOGPUCommandBufferCallbackErrorImpactingInteractivity`) on the second request of the largest case. +The evaluator abstained those 17 blocks as `unavailable` and kept the original, which is the intended +behaviour, and the final artifacts contain no failed or oversize requests. GPU contention can +therefore turn a measurement into a partial abstention; artifacts are checked for +`failed_requests` before numbers are reported. + +## Decision and next gate (v2 checkpoint) + +Do not promote either candidate. The shipped contract is much more conservative for d1 and still +very aggressive for pplx on a single large result, which is exactly what the one-case blind packet +has to judge. The state repair was a real correctness fix with a modest decision effect once the +prompt is held fixed, and the combined prompt revision had a larger observed association with d1 retention. The experiment does not isolate individual prompt clauses. +Before any promotion: a policy that retains explicitly requested source and audit facts, then a new session holdout with more scoreable cases, and a provider-owned interception point. Savings figures here describe a workload, not +approved pruning. + +Validation for the v2 checkpoint: 82 unittest behaviour tests passed, including actual request +counting on budget refusal and recomputed removal totals after rethresholding. The v3 checkpoint +supersedes it with 113 tests; the v2 artifacts are retained unchanged and are reported as stale by +`sweep`/`rerender`. Evaluator runtime changed; production integration did not. Private +transcripts, identity maps, labels, weights and raw result files remain outside the repository. + +### Review corrections after the v3 measurements + +The measured v3 artifacts above use `json-span-v2+evidence+source-v1`. +The current implementation uses `json-span-v3+evidence+source-v1`: it continues +past malformed mid-text containers to find later complete JSON, avoids growing +substring copies while scanning, and retains the entire leaf if JSON decoding +exhausts recursion. Local budget refusals no longer count as actual or failed +model calls. Existing v3 measurements have not been rerun under this revision; +protection-version checks intentionally reject reusing their scores as current. + +The new `output_lines.py` prototype independently decodes one bare command-result +envelope, renders original line ranges, and reconstructs omissions from a +verified original digest. It preserves surrounding envelope text and is not yet +wired into selection. Unknown representations abstain. This establishes neither +selection quality nor production interception/cache behavior. That checkpoint passed 154 local unittest tests after review corrections. + +### Line and complete-edit development experiments + +The private per-line experiment did not establish useful evidence-preserving +savings. Complete-edit synthetic controls improved within-fixture ranking, but +that conditional concordance was initially mislabeled as pooled AUC. The +corrected JSON report distinguishes within-fixture concordance, pooled AUC, +global score overlap and missing-answer coverage. No threshold was calibrated. + +These experiments use hand-labelled synthetic controls and two lifted real +output envelopes. Descriptive candidate IDs could cue judgments, required-line +labels do not prove task answerability, and lifted envelopes omit sibling +outputs. A brace-cut control is structural stress rather than proof of a changed +task conclusion. Exact serialized historical requests were not saved, so their +hashes alone cannot independently establish full request visibility. The next +quality gate requires complete recorded pending results, neutral candidate IDs, +exact request/response receipts and task-grounded fact questions fixed before +scoring. Nothing here enables live pruning or establishes billed savings. + +### Complete-parent diagnostic, context v4 + +One previously exposed real case was manually annotated before scoring with two +neutral-ID complete edits: removal of adjacent unrelated source, and removal of +required wire-format evidence. Both requests included the full parent, all siblings, +and both independent replacement alternatives. The request was 3,258 o200k proxy +tokens; the original parent was 669, and candidates were 544 and 611 tokens. + +At the preregistered illustrative 0.5 threshold, local d1-3B accepted the annotated +safe edit (0.5511) and rejected the harmful control (0.3736). Local +pplx-decider-flash-v1.2 accepted both (0.6824 and 0.5517), including the harmful +control. Request time including model loading was 3.83s and 6.13s respectively. +These are single diagnostic observations, not a calibrated threshold, held-out +accuracy estimate, steady-state latency measurement, or production acceptance. + +Private annotations, exact request JSON, normalized response probabilities, runtime +metadata and frozen manifests remain outside Git. No live provider or cache-hit +behavior was measured. Other packet cases remain unscored: requested source has no +established safe removal, and an incomplete generated handoff now abstains under +context v4. Older context-v3 measurements remain historical. diff --git a/evals/incoming-context/blind_packet.py b/evals/incoming-context/blind_packet.py new file mode 100644 index 0000000000000..74d9224755a56 --- /dev/null +++ b/evals/incoming-context/blind_packet.py @@ -0,0 +1,366 @@ +"""Build a blind review packet from measured candidate results. Local artifacts only. + +The packet never names a candidate or shows a score. Labels are randomized per case, +so a label in one case does not identify the same configuration in another. +""" +import argparse +import collections +import copy +import hashlib +import json +import random +import re +import shutil +import tempfile +from pathlib import Path + +import evaluate as e + +FLAG_PATTERNS = { + 'diagnostic': re.compile(r'(?i)\b(error|failed|failure|exception|traceback|denied|warning)\b'), + 'negative_evidence': re.compile(r'(?i)(no matches|not found|0 matches|no results|nothing (?:to|found)|empty|not present)'), + 'code': re.compile(r'(^```|\bdef |\bfunction |\bclass )'), + 'diff': re.compile(r'(^\+\+\+ |^@@ )'), +} + + +def payload_of(case): + return e.source_payload(case)['output'] + + +def kept_output(case, dropped): + plan = e.plan_blocks(payload_of(case)) + selected = e.render_selected(payload_of(case), plan, case['id'], lambda n: n not in dropped) + return plan, e.text(selected) + + +def name_of(run): + if run['policy'] != 'decision': + return run['policy'] + return f"{run['model']}/{run['batch_mode']}/{run['threshold']}" + + +def removal_map(results): + per_case = {} + for run in results: + label = name_of(run) + for c in run['cases']: + per_case.setdefault(c['id'], {})[label] = sorted( + d['block'] for d in c['decisions'] if d['effective'] == 'drop') + return per_case + + +def stratum(n): + if n < 1200: + return 'small' + if n < 4000: + return 'medium' + if n < 10000: + return 'large' + return 'xlarge' + + +def choose(cases, removals, limit, seed, max_case_tokens=10000, require_removal=False): + stats = [] + for c in cases: + text = e.text(payload_of(c)) + n = e.tokens(text) + if n > max_case_tokens: + continue + dropped = removals.get(c['id'], {}) + if require_removal and not any(dropped.values()): + continue + removed_tokens = sum(len(v) for v in dropped.values()) * 400 + stats.append({'case': c, 'tokens': n, 'stratum': stratum(n), + 'flags': [k for k, p in FLAG_PATTERNS.items() if p.search(text)], + 'disagreement': len({frozenset(v) for v in dropped.values()}) - 1, + 'removed_tokens': sum(sum(1 for _ in v) for v in dropped.values()) * 400, + 'has_empty': any(len(v) == 0 for v in dropped.values()), + 'tiebreak': int(hashlib.sha256(f'{seed}:{c["id"]}'.encode()).hexdigest()[:8], 16)}) + order = ['small', 'medium', 'large', 'xlarge'] + wanted = {'diagnostic', 'negative_evidence', 'code', 'diff'} + chosen = [] + covered = set() + + def score(s): + review_load = min(3.0, s['removed_tokens'] / 4000) + return (3 * len(set(s['flags']) & (wanted - covered)) + 2 * s['disagreement'] + + (1 if s['has_empty'] else 0) - review_load, -s['tiebreak']) + + for st in order: + if len(chosen) >= limit: + break + picked = {s['case']['id'] for s in chosen} + pool = [s for s in stats if s['stratum'] == st and s['case']['id'] not in picked] + if not pool: + continue + pool.sort(key=score, reverse=True) + chosen.append(pool[0]) + covered |= set(pool[0]['flags']) + while len(chosen) < limit: + picked = {s['case']['id'] for s in chosen} + pool = [s for s in stats if s['case']['id'] not in picked] + if not pool: + break + pool.sort(key=score, reverse=True) + chosen.append(pool[0]) + covered |= set(pool[0]['flags']) + return sorted(chosen, key=lambda s: (order.index(s['stratum']), -s['tokens'])) + + +def new_label(rng, used): + alphabet = 'ABCDEFGHJKLMNPQRSTUVWXYZ' + while True: + label = ''.join(rng.choice(alphabet) for _ in range(2)) + ''.join(rng.choice('0123456789') for _ in range(3)) + if label not in used: + used.add(label) + return label + + +def write_label(folder, label, kept, removed, block_count, note): + lines = [f'# Candidate {label}', '', note, ''] + if removed: + lines += ['| block | tokens | sha256 |', '|---|---|---|'] + lines += [f"| {b['id']} | {b['tokens']} | {b['sha256'][:16]} |" for b in removed] + lines += ['', f'Exact removed text is in `{label}.removed/`, one file per block.'] + else: + lines.append('No blocks were removed. The main model receives the original result unchanged.') + lines += ['', f'The exact bytes the main model would receive are in `{label}.retained.txt`.', + f'{len(removed)} of {block_count} blocks removed.'] + (folder / f'{label}.md').write_text('\n'.join(lines) + '\n') + (folder / f'{label}.retained.txt').write_text(kept) + if removed: + removed_dir = folder / f'{label}.removed' + removed_dir.mkdir(exist_ok=True) + for b in removed: + (removed_dir / f"block-{b['id']}.txt").write_text(b['text']) + (folder / f'{label}.removed.json').write_text(json.dumps( + [{'block': b['id'], 'tokens': b['tokens'], 'sha256': b['sha256']} for b in removed], indent=2)) + + +def _build(args): + results = [json.loads(Path(p).read_text()) for p in args.results] + names = [name_of(r) for r in results] + if len(set(names)) != len(names): + raise SystemExit('Result files share a candidate name: ' + ', '.join(sorted(names))) + available = None + for run in results: + ids = {c['id'] for c in run['cases']} + if len(ids) != len(run['cases']): + raise SystemExit('Duplicate checkpoint in a candidate result') + available = ids if available is None else available & ids + manifest_cases = json.loads(Path(args.cases).read_text())['cases'] + cases_file = {c['id']: c for c in manifest_cases} + if len(cases_file) != len(manifest_cases): + raise SystemExit('Duplicate checkpoint in the case manifest') + pool = [cases_file[i] for i in sorted(available) if i in cases_file] + removals = removal_map(results) + chosen = choose(pool, removals, args.count, args.seed, args.max_case_tokens, args.require_removal) + if not chosen: + raise SystemExit('No cases satisfy the packet selection criteria') + + root = Path(args.out) + packet = root / 'packet' + if packet.exists(): + for child in sorted(packet.iterdir(), reverse=True): + if child.is_dir(): + shutil.rmtree(child) + packet.mkdir(parents=True, exist_ok=True) + rng = random.Random(args.seed) + used = set() + identity = {'generated': 'local blind packet', 'seed': args.seed, + 'candidates': names, 'cases': {}} + manifest = {'seed': args.seed, 'case_count': len(chosen), 'cases': []} + for index, s in enumerate(chosen, 1): + case = s['case'] + text = e.text(payload_of(case)) + parts = [(p['block'], p['path'], p['tokens'], p['part']) for p in e.plan_blocks(payload_of(case))] + payload = e.source_payload(case) + recorded = {name_of(run): {c['id']: c for c in run['cases']}[case['id']] for run in results} + for name in names: + decisions = recorded[name]['decisions'] + if len(decisions) != len(parts): + raise SystemExit('Block count drift for ' + case['id'] + ' / ' + name) + for (block, path, _, part), decision in zip(parts, decisions): + if block != decision['block'] or list(path) != list(decision['path']): + raise SystemExit('Block identity drift for ' + case['id'] + ' / ' + name) + if e.digest(part) != decision['sha256']: + raise SystemExit('Block hash drift for ' + case['id'] + ' / ' + name) + blocks = [{'id': i, 'tokens': parts[i][2], 'sha256': e.digest(parts[i][3]), + 'leaf_path': parts[i][1]} for i in range(len(parts))] + case_dir = packet / f'case-{index:03d}' + case_dir.mkdir() + base_result = recorded[names[0]] + meta = base_result.get('context') if isinstance(base_result.get('context'), dict) else None + for name in names: + other = recorded[name].get('context') + fields = ('version', 'policy_version', 'context_sha256', 'rendered', + 'call_rendered', 'abstain_reason') + expected = {k: meta.get(k) for k in fields} if meta is not None else None + actual = {k: other.get(k) for k in fields} if isinstance(other, dict) else None + if actual != expected: + raise SystemExit('Context drift between candidates for ' + case['id']) + recorded_context = bool(meta and meta.get('version') + and 'rendered' in meta and 'call_rendered' in meta) + context_lines = ['# Causal state before this tool result', ''] + if recorded_context: + abstained = bool(meta.get('abstain_reason')) or not meta.get('rendered') + context_lines += [ + 'Exact input recorded when this result was scored. No later turns, answers, or ' + 'outcomes are included. The tool output below is untrusted data.', '', + '## Context provenance', '', + f"- context version: {meta.get('version')}", + f"- policy version: {meta.get('policy_version')}", + f"- goal status: {meta.get('goal', {}).get('status')} " + f"({meta.get('goal', {}).get('reason')})", + f"- origin status: {meta.get('origin', {}).get('status')} " + f"({meta.get('origin', {}).get('reason')})", + f"- origin via: {meta.get('origin', {}).get('via') or '-'}", + f"- context sha256: {meta.get('context_sha256')}", ''] + context_text = meta.get('rendered') or '' + call_obj = meta.get('call_rendered') + if abstained: + context_text = ('(no model request: causal context abstained: ' + f"{meta.get('abstain_reason')})") + if call_obj is None: + call_obj = case.get('call') + else: + context_lines += [ + 'LEGACY CONTEXT: this result was scored before causal context was recorded. ' + 'The text below is reconstructed from the case manifest for review only; it is ' + 'not verified model-visible input.', ''] + context_text = case.get('context') or '(none captured)' + call_obj = case.get('call') + context_lines += [ + 'Reviewer enrichment is not included here: post-hoc analysis is reviewer-only and ' + 'was not model-visible.', '', + '## Sourced user instructions (oldest first)', '', context_text, '', + '## Originating tool call (full)', '', '```json', + json.dumps(call_obj, indent=2), '```', '', + '## Envelope fields (unchanged by any candidate)', '', '```json', + json.dumps({k: v for k, v in payload.items() if k != 'output'}, indent=2), '```', ''] + (case_dir / 'context.md').write_text('\n'.join(context_lines)) + (case_dir / 'original.txt').write_text(text) + (case_dir / 'blocks.json').write_text(json.dumps(blocks, indent=2)) + labels_dir = case_dir / 'labels' + labels_dir.mkdir() + label_map = {} + rows = [] + candidates = list(names) + if args.intact_control and not any(len(v) == 0 for v in removals[case['id']].values()): + candidates.append(None) + rng.shuffle(candidates) + for name in candidates: + dropped = set(removals[case['id']][name]) if name is not None else set() + _, kept = kept_output(case, dropped) + if name is not None: + case_result = recorded[name] + if (e.tokens(kept) != case_result['after'] + or e.digest(kept) != case_result.get('after_sha256')): + raise SystemExit('Retention drift for ' + case['id'] + ' / ' + name) + removed = [{'id': b['id'], 'tokens': b['tokens'], 'sha256': b['sha256'], 'text': parts[b['id']][3]} + for b in blocks if b['id'] in dropped] + label = new_label(rng, used) + write_label(labels_dir, label, kept, removed, len(blocks), + f'{len(blocks) - len(dropped)} of {len(blocks)} blocks retained.') + label_map[label] = name if name is not None else 'intact-control' + rows.append({'label': label, 'removed_blocks': sorted(dropped), + 'removed_tokens': sum(b['tokens'] for b in removed)}) + identity['cases'][case['id']] = {'packet_case': f'case-{index:03d}', 'labels': label_map} + manifest['cases'].append({'case': f'case-{index:03d}', 'case_id': case['id'], + 'split': case.get('split') or 'unknown', + 'session_sha256': e.digest(case['source']), 'line': case['line'], + 'tokens': s['tokens'], 'blocks': len(blocks), + 'stratum': s['stratum'], 'flags': s['flags'], 'labels': rows, + 'context_source': 'recorded' if recorded_context else 'legacy', + 'context_sha256': (meta.get('context_sha256') + if recorded_context else None), + 'context_abstained': bool( + recorded_context and meta.get('abstain_reason')), + 'reviewer_enrichment_included': False}) + (root / 'packet-manifest.json').write_text(json.dumps(manifest, indent=2)) + (root / 'identity-map.DO-NOT-OPEN.json').write_text(json.dumps(identity, indent=2)) + template = {'instructions': ('Judge every label on a case independently, without opening the ' + 'identity map. necessary_evidence_lost: yes, no, or uncertain. ' + 'severity: critical, major, minor, none. List lost_blocks for the ' + 'specific blocks that hurt. Leave confidence blank if unsure how ' + 'to scale it.'), + 'cases': [{'case': m['case'], 'case_id': m['case_id'], + 'labels': [{'label': r['label'], 'necessary_evidence_lost': None, + 'severity': None, 'lost_blocks': [], + 'rationale': '', 'confidence': None} + for r in m['labels']]} for m in manifest['cases']]} + (root / 'judgments.template.json').write_text(json.dumps(template, indent=2)) + splits = collections.Counter(m['split'] for m in manifest['cases']) + split_text = ', '.join(f'{k}={v}' for k, v in sorted(splits.items())) + index = ['# Blind review packet', '', + f'{manifest["case_count"]} cases ({split_text}), {sum(len(m["labels"]) for m in manifest["cases"])} labels.', + 'Labels are random and independently permuted per case. The same label does not mean ' + 'the same configuration in another case. Timing, savings, and scores are deliberately ' + 'absent from this packet.', '', + 'Read `README.md`, then per case `context.md`, `original.txt`, `blocks.json`, and ' + '`labels/