eval rounds 2 and 3, round 4 pre-registration; MO-1d, refresh under a whole-value guess, Solid repin ac28157a

compiler/Cargo.toml

@@ -26,7 +26,7 @@ regex = "1"
 serde_json = { version = "1", features = ["preserve_order"] }
 # CP-1b: Solid's compiler, at the commit ref/ runs on (ref/SOLID_NEXT), and the Oxc release it is built against (CP-1c:
 # the program crosses the seam, so the versions must match)
-solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "d2323b077c7dda32ebced7843c2ce2b788a91e8d", default-features = false }
+solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "ac28157aba6d03b806f0043bce70a87d515e953e", default-features = false }
 oxc_allocator = "0.144"
 oxc_ast = "0.144"
 oxc_span = "0.144"

compiler/src/check.rs

@@ -39,7 +39,7 @@ impl Diagnostic {
     }
 }
 
-const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"];
+const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"];
 /// EF-5's cleanup shapes: a `return` of a closure anywhere in the body, a closure as its last expression, or an expression body that is one.
 fn returns_closure(f: &Expr) -> bool {
     fn in_stmts(ss: &[Stmt]) -> bool {
@@ -748,6 +748,8 @@ pub struct Checker<'a> {
     apps: Vec<AppRec<'a>>,
     actions: Vec<ActionRec<'a>>,
     provisions: Vec<Provision<'a>>,
+    /// MO-1d: the symbols import declarations bound, which no declaration in a nested block may shadow.
+    import_syms: HashSet<SId>,
     later: Vec<Later<'a>>,
     comp_syms: HashMap<RId, SId>,
     rows_busy: HashSet<String>,
@@ -819,6 +821,7 @@ impl<'a> Checker<'a> {
             apps: vec![],
             actions: vec![],
             provisions: vec![],
+            import_syms: HashSet::new(),
             later: vec![],
             comp_syms: HashMap::new(),
             rows_busy: HashSet::new(),
@@ -982,13 +985,20 @@ impl<'a> Checker<'a> {
         // 1. names (E-7: one declaration per name in a block)
         for d in decls {
             self.line = d.line() as i64;
-            if let Decl::Import { resolved: Some(_), .. } = d {
-                self.import_rx(d, env);
-                continue;
-            }
-            if let Decl::Import { declared: Some(_), .. } = d {
-                self.import_declared(d, env);
-                continue;
+            if let Decl::Import { names, .. } = d {
+                if matches!(d, Decl::Import { resolved: Some(_), .. }) {
+                    self.import_rx(d, env);
+                } else if matches!(d, Decl::Import { declared: Some(_), .. }) {
+                    self.import_declared(d, env);
+                }
+                if matches!(d, Decl::Import { resolved: Some(_), .. } | Decl::Import { declared: Some(_), .. }) {
+                    for n in names {
+                        if let Some(&s) = self.envs[env].vars.get(&n.alias) {
+                            self.import_syms.insert(s);
+                        }
+                    }
+                    continue;
+                }
             }
             match d {
                 Decl::Import { names, pure, fails, side, line, .. } => {
@@ -997,7 +1007,8 @@ impl<'a> Checker<'a> {
                         s.io = !pure;
                         s.rejects = Some(fails.clone().unwrap_or_else(|| vec!["Error".into()]).into_iter().collect());
                         s.side = *side;
-                        self.bind(env, &n.alias, s);
+                        let s = self.bind(env, &n.alias, s);
+                        self.import_syms.insert(s);
                     }
                     continue;
                 }
@@ -1034,6 +1045,11 @@ impl<'a> Checker<'a> {
                     self.err("DUPLICATE_DECLARATION", "E-7", format!("{} is declared twice in one block", name));
                 }
             }
+            if let Some(shadowed) = self.envs[env].parent.and_then(|p| self.lookup(p, name)) {
+                if self.import_syms.contains(&shadowed) {
+                    self.err("SHADOWED_IMPORT", "MO-1d", format!("{} shadows the imported {}: rename one (`import {{ {} as … }}`)", name, name, name));
+                }
+            }
             let kind = match d {
                 Decl::Fn { .. } => SymKind::Fn,
                 Decl::Component { .. } => SymKind::Comp,

compiler/src/emit.rs

@@ -21,7 +21,7 @@ fn fail<T>(msg: impl Into<String>) -> R<T> {
     Err(CompileError(msg.into()))
 }
 
-const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"];
+const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"];
 fn builtin(n: &str) -> bool {
     BUILTINS.contains(&n)
 }

compiler/src/project.rs

@@ -18,7 +18,7 @@ pub const PRELUDE: &str = include_str!("prelude.ts");
 
 static ROW_WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^((?:io|pending|lane|fails\([^)]*\))\s+)+").unwrap());
 static SOURCE_WORD: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^source\s+").unwrap());
-const BUILTINS: [&str; 12] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "log", "until", "onSettled", "onFlush"];
+const BUILTINS: [&str; 13] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf", "log", "until", "onSettled", "onFlush"];
 fn builtin(n: &str) -> bool {
     BUILTINS.contains(&n)
 }

eval/PREREG-change.md

@@ -0,0 +1,78 @@
+# Round 4: changing an existing app
+
+Pre-registration. Everything in this file was fixed and committed before the round's first agent run.
+
+## Why this round
+
+Rounds 2 and 3 (`results/plain-2026-10-27/`, `results/overlap-2026-10-27/`) had agents write small apps from
+scratch. Neither found a defect gap. This round asks the other half of Ryan's question: does an agent keep an app
+correct when it **changes** one it did not write, where the new feature adds writes that overlap the app's existing
+ones?
+
+## Design
+
+- **Bases**: round 3's frontier-model tracker apps, `results/overlap-2026-10-27/runs/big/tracker-<arm>-<n>`. Run n of
+  each task and arm starts from base n of that arm, for both models. On Solid next @ ac28157a every base passes all
+  of the tracker's visible, hidden and hidden-overlap tests.
+- **Tasks** (`tasks/<task>/brief-plain.md`, the change; `BASE.md`, the tracker's plain brief, which the app must keep
+  meeting). None names a race.
+  - rename: rename the selected issue (`renameIssue`), shown in the list and the detail straight away.
+  - close: close an issue (`closeIssue`); it leaves the list straight away and comes back if declined.
+  - comments: load the selected issue's comments (`fetchComments`) and post one (`addComment`), shown straight away
+    with `(sending…)`.
+- **Tests**: per task `visible.mjs` (the tracker's 4 plus 2 new), `hidden.mjs` (new tests plus the tracker's hidden
+  tests as regressions) and `hidden-overlap.mjs` (new overlap tests plus the tracker's overlap tests). rename 16 + 7,
+  close 15 + 6, comments 17 + 6 hidden and overlap tests.
+- **Prompt**: `harness/prompt-change.md`. Each run keeps scratch files in its own `.scratch` folder (round 3's leak).
+- **Toolchain**: Solid next @ ac28157a (2.0.0-rc.19 plus the #4329 fix, solidjs/solid#4330) in both arms, through
+  `eval/package.json`. The .rx compiler and runtime from this branch: MO-1d `SHADOWED_IMPORT` (taken by default
+  2026-10-27), and `refresh` re-asking the question under a whole-value guess (`ref/compile/rt-web.ts`). The .rx
+  arm gets `docs/spec` and `docs/guide/async.md` as in round 3, unchanged.
+- 3 tasks × 2 arms × 2 models (frontier, Haiku) × 3 runs = 36 fresh runs, 15 `./check` + `./test` each.
+
+Hidden test files, SHA-256 at freezing:
+
+```
+869e02eceb679c5907a67571a26f061157eefd86e8e38a1a7aec8db7d9e65fc0  tasks/rename/hidden.mjs
+532e2caf837f3effaf77c5979af2651ed4d0ec2973b496d76846035d45831403  tasks/rename/hidden-overlap.mjs
+86aebac10d51c9244504e5da3b368c356e6b8c65dfab5d9dbc35b03adb66604d  tasks/close/hidden.mjs
+669c606395e539e031ca7f376304a42607d15ffccb823b291b5d616f30ddce42  tasks/close/hidden-overlap.mjs
+25d241a2309310943ed24d72d26c2a7e638a6feecbd3397b39123ebf5e956dab  tasks/comments/hidden.mjs
+3d95d4274efea4cd7d793140d2daab3f6b087bcd22c6b4ff7e5a8492d4d361ec  tasks/comments/hidden-overlap.mjs
+c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e  tasks/tracker/hidden.mjs
+5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e  tasks/tracker/hidden-overlap.mjs
+```
+
+**References** (`reference/<task>/<arm>`, not subjects) were written on the bases. Both arms pass every test, except
+the .rx comments reference on "a post on one issue does not show on another" (below).
+
+**Tests loosened while writing the references** (before freezing). By H-11 two guesses on one slot from two actions
+are one change, so a declined guess can stay on screen until the other action settles. Ryan (2026-10-28): that is the
+desired behaviour. Four tests were changed to accept it: close's "a declined close keeps an assignment…", "a declined
+assignment keeps a close…", "two closes at once: a declined one does not undo the other", and comments' "two posts at
+once: a declined one does not take the other with it". Each still checks the end state after both settle.
+
+## Hypothesis and test
+
+- **H1**: the React arm ships more defects than the .rx arm.
+- **Primary measure**: defects shipped: a failed build or any failed visible, hidden or hidden-overlap test at the
+  agent's "done" (`node harness/score.mjs <dir> --overlap`), except the one test below. One number per arm, pooled
+  over tasks and models (18 runs each).
+- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React
+  rate is higher and p < 0.05. **H1 reversed** is reported as plainly.
+- **Outside the count, reported alongside**: comments' "a post on one issue does not show on another". The .rx
+  idiom for it (a whole-value guess on the comments source) shows solidjs/solid#4336 (filed 2026-10-28): the guess
+  stays on screen after the selected issue changes, until the post settles. `score.mjs` runs it and lists it per run
+  under `failedAlongside`, for both arms, and does not count it.
+- **Other runtime bugs** found during the round count as defects, as in rounds 2 and 3, and are listed separately
+  with the primary result recomputed without them.
+- **Reported alongside**: the same table per model and per task; base regressions (tracker tests failed) apart from
+  new-feature failures; each failure classified by hand; agent tokens and cycles.
+- Published as is, whatever the outcome.
+
+## Known limits
+
+- Isolation is still by instruction.
+- The bases are frontier-model apps; the small model changes code it would not have written.
+- The new tests were written after round 3's results were known, and tuned on the references.
+- 18 runs per arm detects a large effect only.

eval/PREREG-overlap.md

@@ -0,0 +1,70 @@
+# Round 3: overlapping writes
+
+Pre-registration. Everything in this file was fixed and committed before the round's first agent run.
+
+## Why this round
+
+Round 2 (`results/plain-2026-10-27/`) found no defect gap: with briefs that don't name the races, React shipped 0
+of 18 defects and .rx 1 of 18. Current models guard a single stale answer in React unprompted. What the agents
+reported as unsure, and what no hidden test probed, was **overlapping writes**: two writes in flight at once, and a
+write racing a re-load of the data it changed. Exploratory probes on round 2's finished apps (`probes/`, not a
+test of anything) showed failures there on both sides. This round tests that question on fresh runs.
+
+## What changes from round 2
+
+- **Hidden tests**: each task gets `tasks/<task>/hidden-overlap.mjs`, run in addition to the unchanged
+  `hidden.mjs`. Each test checks something the plain brief already states, in a case where two things are in
+  flight:
+  - search (1): a retry's answer landing after another selection must not show. The brief says the pane shows the
+    selected hit's item.
+  - tracker (2): two issues assigned at once, one declined; and the list loading while an assignment is out. The
+    brief says a declined row "goes back to the assignee it had before" and that the row shows the new assignee
+    "straight away" and "keeps" it if the server accepts.
+  - settings (3): an earlier save failing after a later one succeeded; a later save failing while an earlier one is
+    out; an earlier save landing while a later one is out. The brief says the saved line shows the values being
+    saved with `(saving…)`, the saved settings when a save succeeds, and goes back and shows `save-error` when it
+    fails.
+- **The .rx guide** teaches guessing one row of a list (`optimistic xs[i].field := v`, with `indexOf` to find the
+  row) instead of the whole list. A whole-list guess makes two actions one change (H-11). The example is a generic
+  todo list, not a task's code. **The prelude** gains `indexOf(xs, f)` (Ryan, 2026-10-27), and the .rx prompt's
+  prelude list names it.
+- Everything else is as in round 2: plain briefs, prompt, visible tests, toolchain, budget, 3 tasks × 2 arms ×
+  2 models (frontier, Haiku) × 3 runs = 36 fresh runs.
+
+Hidden test files, SHA-256 at freezing:
+
+```
+5776f8888055c4f5c13261b8637e048ebbb4f919602afb6f294a78ae4339b4d2  tasks/search/hidden-overlap.mjs
+5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e  tasks/tracker/hidden-overlap.mjs
+aa1d42209d188ea4c54220c816d0ffe718e7ebe65c2d9b9d3fbf5fa23949aade  tasks/settings/hidden-overlap.mjs
+1d77a1daa81d15d40045d1f4665f9fdcebbe9abb6356db5b3a6eb8b29d30d0ae  tasks/search/hidden.mjs
+c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e  tasks/tracker/hidden.mjs
+58d2a1daa4a1b8d2ea559ab56bb37ee9193dcf3d31d03e362107e25ee3b106b6  tasks/settings/hidden.mjs
+```
+
+The references are not subjects, and neither passes everything: both fail some overlap tests (`reference/` is
+unchanged from the pilot). The tests were checked to run and to fail for a stated reason, not to be passable by the
+references.
+
+## Hypothesis and test
+
+- **H1**: the React arm ships more defects than the .rx arm.
+- **Primary measure**: defects shipped, where a defect is a failed build or any failed visible, hidden or
+  hidden-overlap test at the agent's "done" (`node harness/score.mjs <dir> --overlap`). One number per arm, pooled
+  over tasks and models (18 runs each).
+- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React rate
+  is higher and p < 0.05. **H1 reversed** (the .rx rate higher, p < 0.05) is reported as plainly.
+- **Reported alongside**: the same table per model and per task; overlap tests failed per run; each failure
+  classified by hand (stale answer, lost rollback, wrong revert, error lost, runtime bug, build failure).
+- **Runtime bugs**: solidjs/solid#4329 (a per-key guess dropped when a re-ask lands mid-action) is live in rc.19 and
+  touches the tracker's "list loading while an assignment is out" test for a .rx app that guesses per row. As in
+  round 2, such a failure counts as a defect in the primary measure (the user gets the bug either way) and is
+  listed separately with the primary result recomputed without it.
+- Published as is, whatever the outcome.
+
+## Known limits
+
+- Isolation is still by instruction.
+- The overlap tests were chosen after the exploratory probes had been run on round 2's apps. They run here only on
+  fresh runs, and four of the six are the probes as they were run.
+- 18 runs per arm detects a large effect only (round 2's PREREG gives the arithmetic).