From 9c38a3f7e3b48f6b8cd800b9b1f5c8a33efed8c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 19:01:09 +0000 Subject: [PATCH 01/11] eval: pre-register round 2, briefs that don't name the races Plain briefs (tasks/*/brief-plain.md), scaffold --brief plain (prompt and flights.js without the race hints, .prompt written), harness/compare.mjs (Fisher's exact test), and guide/async.md examples moved off the tasks' own code. Hidden tests unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-plain.md | 73 ++++++++++++++++++++++++++++++ eval/harness/compare.mjs | 38 ++++++++++++++++ eval/harness/prompt.md | 6 +-- eval/harness/scaffold.mjs | 26 ++++++++--- eval/tasks/search/brief-plain.md | 31 +++++++++++++ eval/tasks/settings/brief-plain.md | 35 ++++++++++++++ eval/tasks/tracker/brief-plain.md | 39 ++++++++++++++++ guide/async.md | 26 +++++------ 8 files changed, 252 insertions(+), 22 deletions(-) create mode 100644 eval/PREREG-plain.md create mode 100644 eval/harness/compare.mjs create mode 100644 eval/tasks/search/brief-plain.md create mode 100644 eval/tasks/settings/brief-plain.md create mode 100644 eval/tasks/tracker/brief-plain.md diff --git a/eval/PREREG-plain.md b/eval/PREREG-plain.md new file mode 100644 index 0000000..51519ae --- /dev/null +++ b/eval/PREREG-plain.md @@ -0,0 +1,73 @@ +# Round 2: briefs that don't name the races + +Pre-registration. Everything in this file was fixed and committed before the round's first agent run. + +## Why this round + +In the pilot (`results/pilot-2026-10-27/`), no arm shipped a defect. The briefs said "answers can arrive in any +order" and "only the answer for the current query may ever show", so the agent knew which races to guard, and it +guarded them in React as well. This round asks the question the pilot couldn't: **when the brief only describes the +screens, does the agent ship a race in React that it doesn't ship in .rx?** + +## What changes from the pilot + +- **Briefs**: `tasks//brief-plain.md`. Each one keeps every screen, text and test id of `brief.md` and drops + every sentence about timing or staleness: "answers take an unknown time and can arrive in any order", "only the + answer for the current … may ever show", "nothing from the previously selected …", "edits the user makes while a + save is in flight stay in the form", "a failure in the list never changes the detail". What is left says what each + screen shows given what the user did and what the server answered. Example: "Two characters or more: call + `search(q)` and show `searching…` until the results arrive." +- **Prompt**: the line about `./test` no longer says the tests control the fake backend's answers, and the agent's + copy of `src/flights.js` loses its header comment (which says tests decide the order answers arrive in). The code + itself is unchanged. `node harness/scaffold.mjs --brief plain` writes all of this, prompt + included (`/.prompt`). +- **The .rx guide's examples** no longer use the tasks' own code: `guide/async.md` showed the search equation and + the settings save action verbatim. They are now a forecast lookup and a profile save, with the same idioms (both + pass `rxc check`). The guide still explains the language's model in general (a stale answer is thrown away, an + action is one change). That is documentation of .rx, the same way the React agent knows React's from training. +- **Two models**: the session's frontier model, and a small model (Haiku), launched as subagents with no memory of + each other. + +## What does not change + +- Hidden tests, byte for byte (SHA-256 as frozen for the pilot): + ``` + 1d77a1daa81d15d40045d1f4665f9fdcebbe9abb6356db5b3a6eb8b29d30d0ae tasks/search/hidden.mjs + c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e tasks/tracker/hidden.mjs + 58d2a1daa4a1b8d2ea559ab56bb37ee9193dcf3d31d03e362107e25ee3b106b6 tasks/settings/hidden.mjs + ``` + Each hidden test checks something the plain brief states: a stale answer on screen contradicts "the results for + the query" or "the selected issue", a reset draft contradicts "saving never changes the draft". No hidden test + needs a sentence that the plain brief dropped. Both references pass every visible and hidden test when scaffolded + with `--brief plain`. +- Visible tests, arms, toolchain (Solid rc.19, React 19.2, TanStack Query 5), the 15-command budget, scoring + (`harness/score.mjs`), and the DV-8 rule (a .rx failure due only to `key` on a projection is a runtime bug, not an + agent defect). + +## Runs + +3 tasks × 2 arms × 2 models × 3 runs = **36 runs**, each a fresh agent in its own directory. A run that ends +without the agent declaring done (a crash or a tool error, not a wrong app) is run again; every such rerun is +reported. + +## Hypothesis and test + +- **H1**: the React arm ships more defects than the .rx arm. +- **Primary measure**: defects shipped, as in the pilot: runs where, at the agent's "done", the build fails or any + visible or hidden test fails. One number per arm, pooled over tasks and models (18 runs each). +- **Primary test**: Fisher's exact test, two-sided, on that 2×2 table, at α = 0.05. H1 holds if the React rate is + higher and p < 0.05. `node harness/compare.mjs ...` computes it from `score.mjs`'s output. +- **Reported alongside, not tested**: the same table for each model and each task; hidden tests failed per run; + `./check`+`./test` cycles; app lines; tokens; each failed hidden test, classified by hand (stale answer, lost + rollback, draft reset, stuck state, wrong text, build failure). +- **Whatever the outcome, it is published as is**, including a null result or a result for React. + +## Known limits + +- Isolation is still by instruction (agents share a machine with this repo). +- Hidden tests were written for the pilot's briefs. This makes them conservative here (they test only what both + briefs say), not tuned to this round. +- 18 runs per arm detects a large effect only. A pooled split like 9/18 vs 2/18 reaches p < 0.05; 6/18 vs 2/18 does + not. A null result here means "no large effect on these three apps", not "no effect". +- .rx failures that are a compiler or runtime bug, not the agent's, are counted as defects in the primary measure + (a user would get the bug either way) and listed separately. diff --git a/eval/harness/compare.mjs b/eval/harness/compare.mjs new file mode 100644 index 0000000..694aee1 --- /dev/null +++ b/eval/harness/compare.mjs @@ -0,0 +1,38 @@ +// Compares the arms over one or more scored run directories (each with the results.json score.mjs wrote): defects +// shipped per arm, and Fisher's exact test (two-sided) on that 2×2 table. +// Usage: node harness/compare.mjs ... or, for a table alone: node harness/compare.mjs --table a n b m +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const lf = n => { let s = 0; for (let i = 2; i <= n; i++) s += Math.log(i); return s; }; +const hyper = (a, b, c, d) => Math.exp(lf(a + b) + lf(c + d) + lf(a + c) + lf(b + d) - lf(a) - lf(b) - lf(c) - lf(d) - lf(a + b + c + d)); + +/** two-sided Fisher's exact p for [[a, b], [c, d]]: the sum over tables with the same margins no likelier than this one */ +export function fisher(a, b, c, d) { + const p0 = hyper(a, b, c, d), r1 = a + b, c1 = a + c, n = a + b + c + d; + let p = 0; + for (let x = Math.max(0, c1 - (n - r1)); x <= Math.min(r1, c1); x++) { + const q = hyper(x, r1 - x, c1 - x, n - r1 - c1 + x); + if (q <= p0 * (1 + 1e-7)) p += q; + } + return Math.min(1, p); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = process.argv.slice(2); + if (args[0] === "--table") { + const [a, n, b, m] = args.slice(1).map(Number); + console.log(`react ${a}/${n} rx ${b}/${m} p = ${fisher(a, n - a, b, m - b).toFixed(4)}`); + } else { + const rows = args.flatMap(d => JSON.parse(readFileSync(path.join(path.resolve(d), "results.json"), "utf8")).map(r => ({ ...r, set: path.basename(d) }))); + const count = (arm, f = () => true) => { const rs = rows.filter(r => r.arm === arm && f(r)); return [rs.filter(r => r.shippedDefect).length, rs.length]; }; + const line = (label, f) => { + const [a, n] = count("react", f), [b, m] = count("rx", f); + console.log(`${label.padEnd(20)} react ${a}/${n} rx ${b}/${m} p = ${n && m ? fisher(a, n - a, b, m - b).toFixed(4) : "-"}`); + }; + line("all"); + for (const s of new Set(rows.map(r => r.set))) line(s, r => r.set === s); + for (const t of new Set(rows.map(r => r.task))) line(t, r => r.task === t); + } +} diff --git a/eval/harness/prompt.md b/eval/harness/prompt.md index 211b03d..3038954 100644 --- a/eval/harness/prompt.md +++ b/eval/harness/prompt.md @@ -1,5 +1,5 @@ You are a developer building a small web app. Work only inside the directory {{DIR}}: read and write files there and -nowhere else{{DOCS_RULE}}. Do not look for other solutions, tests or examples anywhere on this machine. +nowhere else{{DOCS_RULE}}, and keep any scratch files inside it. Do not look for other solutions, tests or examples anywhere on this machine. Read {{DIR}}/BRIEF.md: it is the whole requirement. Build the app in {{LANG_RULE}} @@ -8,8 +8,8 @@ Read {{DIR}}/BRIEF.md: it is the whole requirement. Build the app in {{LANG_RULE Two commands, run from {{DIR}}: - `./check` runs {{CHECKER}}. -- `./test` builds the app and runs the visible tests in a browser against a fake backend whose answers the tests - control (see `tests/`). The visible tests cover only part of the brief: a run is judged by the brief, with more tests. +- `./test` builds the app and runs the visible tests in a browser against a fake backend{{BACKEND_NOTE}} + (see `tests/`). The visible tests cover only part of the brief: a run is judged by the brief, with more tests. You may run `./check` and `./test` at most 15 times in total. When you believe the app does everything the brief says, stop and reply with two or three sentences on what you built and anything you were unsure about. diff --git a/eval/harness/scaffold.mjs b/eval/harness/scaffold.mjs index 8b13de7..fa5efe2 100644 --- a/eval/harness/scaffold.mjs +++ b/eval/harness/scaffold.mjs @@ -1,21 +1,25 @@ // Makes a working directory for one arm of one task: the brief, the API, the visible tests and the commands the // agent may run. With --reference, also copies in the reference solution (eval/reference//). -// Usage: node harness/scaffold.mjs [--reference] -import { cpSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +// --brief uses tasks//brief-.md (e.g. `plain`) instead of brief.md. Writes the agent's prompt to +// /.prompt. +// Usage: node harness/scaffold.mjs [--reference] [--brief ] +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; const here = path.dirname(fileURLToPath(import.meta.url)); const evalDir = path.resolve(here, ".."); -export function scaffold(task, arm, dir, { reference = false } = {}) { +export function scaffold(task, arm, dir, { reference = false, brief = null } = {}) { const t = path.join(evalDir, "tasks", task); dir = path.resolve(dir); mkdirSync(path.join(dir, "src"), { recursive: true }); mkdirSync(path.join(dir, "tests"), { recursive: true }); writeFileSync(path.join(dir, "arm"), `${arm}\n`); - cpSync(path.join(t, "brief.md"), path.join(dir, "BRIEF.md")); + cpSync(path.join(t, brief ? `brief-${brief}.md` : "brief.md"), path.join(dir, "BRIEF.md")); cpSync(path.join(t, "api.js"), path.join(dir, "src/api.js")); - cpSync(path.join(here, "flights.js"), path.join(dir, "src/flights.js")); + // a plain brief names no races, so the agent's copy of the plumbing loses the header that says tests order answers + const flights = readFileSync(path.join(here, "flights.js"), "utf8"); + writeFileSync(path.join(dir, "src/flights.js"), brief === "plain" ? flights.replace(/^(\/\/.*\n)+\n/, "") : flights); cpSync(path.join(t, arm === "rx" ? "api.d.rx" : "api.d.ts"), path.join(dir, `src/${arm === "rx" ? "api.d.rx" : "api.d.ts"}`)); cpSync(path.join(t, "visible.mjs"), path.join(dir, "tests/visible.mjs")); cpSync(path.join(t, "fixtures.mjs"), path.join(dir, "tests/fixtures.mjs")); @@ -33,10 +37,20 @@ export function scaffold(task, arm, dir, { reference = false } = {}) { cpSync(path.join(evalDir, "../guide"), path.join(dir, "docs/guide"), { recursive: true }); } if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true }); + writeFileSync(path.join(dir, ".prompt"), prompt(arm, dir, { plain: brief === "plain" })); return dir; } +export function prompt(arm, dir, { plain = false } = {}) { + const fill = { ...JSON.parse(readFileSync(path.join(here, "prompt-arms.json"), "utf8"))[arm] }; + fill.BACKEND_NOTE = plain ? "" : " whose answers the tests control"; + let text = readFileSync(path.join(here, "prompt.md"), "utf8"); + for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]); + return text; +} + if (import.meta.url === pathToFileURL(process.argv[1]).href) { const [task, arm, dir] = process.argv.slice(2); - console.log(scaffold(task, arm, dir, { reference: process.argv.includes("--reference") })); + const b = process.argv.indexOf("--brief"); + console.log(scaffold(task, arm, dir, { reference: process.argv.includes("--reference"), brief: b > 0 ? process.argv[b + 1] : null })); } diff --git a/eval/tasks/search/brief-plain.md b/eval/tasks/search/brief-plain.md new file mode 100644 index 0000000..17e0c95 --- /dev/null +++ b/eval/tasks/search/brief-plain.md @@ -0,0 +1,31 @@ +# Search with a detail pane + +Build a page where the user searches a catalogue and opens one result in a detail pane. + +## The API (`src/api.js`, given; do not change it) + +- `search(q: string): Promise<{ id: number; name: string }[]>`: the hits for `q`, in order. It may reject with an `Error`. +- `fetchItem(id: number): Promise<{ id: number; name: string; blurb: string } | null>`: one item, or `null` when there is + no item with that id. It may reject with an `Error`. + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. A text input, `query`. What the user types is the query. +2. A results area, `results`. + - Empty query: the results area shows nothing, and nothing is asked. + - A query of one character: the results area shows `too short: ` (for `a`: `too short: a`). Nothing is asked. + - Two characters or more: call `search(q)` and show `searching…` until the results arrive. Results already on + screen stay there until the new ones replace them. + - The results: a line ` results for ` (`2 results for ap`) and one list item per hit, `data-testid="hit"`, + whose text is the hit's name. + - If the search fails: `search failed: `. +3. Clicking a hit selects it. The selected hit's text starts with `> ` (`> apple pie`); the others have no prefix. +4. A detail pane, `detail`. + - Nothing selected: `pick a result`. + - When a hit is selected, call `fetchItem(id)` and show `loading…` until it answers. + - The answer is an item: the pane shows its name and its blurb. + - The answer is `null`: `item not found`. + - It fails: `failed: ` and a button `retry` (`data-testid="retry"`) that calls `fetchItem(id)` again and + shows its answer. diff --git a/eval/tasks/settings/brief-plain.md b/eval/tasks/settings/brief-plain.md new file mode 100644 index 0000000..b0cbc16 --- /dev/null +++ b/eval/tasks/settings/brief-plain.md @@ -0,0 +1,35 @@ +# Settings page + +Build a settings page: it loads the user's settings, lets them edit a draft, and saves it. + +## The API (`src/api.js`, given; do not change it) + +- `loadSettings(): Promise<{ name: string; theme: "light" | "dark"; density: "comfortable" | "compact" }>`. May + reject with an `Error`. +- `saveSettings(s: { name: string; theme: "light" | "dark"; density: "comfortable" | "compact" }): Promise`: + answers the settings as saved. May reject with an `Error`. + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Loading.** Call `loadSettings()` when the page opens. Until it answers, an element `load` shows + `Loading settings…`. If it fails, `load` shows `Could not load settings: ` and a button `retry-load` + calls `loadSettings()` again and shows the settings it answers. +2. **Saved line**, `saved`, once the settings have loaded: `Saved: · · ` + (`Saved: Ann · light · compact`). +3. **The draft.** Once loaded, a form is filled from the loaded settings and then belongs to the user: + - an input `name` (the display name), a counter `name-count` reading `/20`; + - `name-error`, shown only when the name is invalid: `Display name is required` when empty, + `Display name must be at most 20 characters, not ` when longer than 20; + - buttons `theme-light` and `theme-dark` that set the draft's theme; a button `density-toggle` that switches the + draft's density between `comfortable` and `compact`; + - `preview` reading `Preview: / `. +4. **Save**, a button `save`. + - With an invalid name it does nothing (nothing is asked). + - Otherwise it calls `saveSettings(draft)`. Straight away the saved line shows the values being saved followed by + ` (saving…)` (`Saved: Bea · dark · compact (saving…)`), and any previous save error disappears. + - When the save succeeds, the saved line shows the saved settings, without `(saving…)`. + - When it fails, the saved line goes back to what was saved before, without `(saving…)`, and `save-error` shows + `Save failed: `. + - Saving never changes the draft. diff --git a/eval/tasks/tracker/brief-plain.md b/eval/tasks/tracker/brief-plain.md new file mode 100644 index 0000000..981ca16 --- /dev/null +++ b/eval/tasks/tracker/brief-plain.md @@ -0,0 +1,39 @@ +# Issue tracker + +Build an issue tracker: a filtered list of issues beside the selected issue's detail, and a button that assigns the +selected issue to the current user. + +## The API (`src/api.js`, given; do not change it) + +- `fetchIssues(filter: "open" | "closed"): Promise<{ id: number; title: string; assignee: string | null }[]>`. May + reject with an `Error`. +- `fetchIssue(id: number): Promise<{ id: number; title: string; body: string } | null>`: `null` when there is no such + issue. May reject with an `Error`. +- `assign(id: number, user: string): Promise<{ id: number; assignee: string }>`: assigns the issue on the server. May + reject with an `Error` (the server declined). + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Current user.** A text input, `user`, holding the current user's name. It starts as `guest`. +2. **Filter.** Two buttons, `filter-open` and `filter-closed`. The filter starts as `open`. +3. **List**, `list`, of the issues for the current filter, from `fetchIssues(filter)`. + - While they load (when the page opens or the filter changes): `loading issues` (`loading closed issues`). + - Then one row per issue, `data-testid="issue"`, reading `# (<assignee>)`, or `(unassigned)` when the + assignee is `null`: `#3 Crash on save (unassigned)`. + - If it fails: `list error: <message>`. +4. **Detail**, `detail`. + - Nothing selected: `select an issue`. Clicking a row selects its issue. + - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #<id>`. + - An issue: `#<id> <title>` and its body. + - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. + - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. +5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading + `assign to <current user>`. Clicking it: + - shows the current user as the issue's assignee in its list row straight away; + - calls `assign(id, user)`; + - if the server declines, the row goes back to the assignee it had before, and the detail shows the server's + message (`declined`); + - if the server accepts, the row keeps the new assignee. + - An assignment error is cleared when a new assignment starts or another issue is selected. diff --git a/guide/async.md b/guide/async.md index 96bbaee..512a3a1 100644 --- a/guide/async.md +++ b/guide/async.md @@ -13,15 +13,15 @@ want one of the moves below. ## 1. Reading from the server: an equation ``` -results = query == "" ? [] : len(query) < 2 ? throw TooShort(query) : await search(query) +forecast = city == "" ? null : len(city) < 3 ? throw TooShort(city) : await fetchForecast(city) ``` - It is asked again whenever its inputs change. An answer to an older question is thrown away when it lands (R-4), so a stale answer can never show. Nothing to guard. -- An arm that isn't taken asks nothing (E-11): the empty and the too-short query send no request. -- While a new answer is on its way, everything that reads `results` keeps showing the last one (the change is held, - H-2). To show that work is going on, read ahead: `pending? results` is `true` while it's in flight, and - `latest query` is what was typed. +- An arm that isn't taken asks nothing (E-11): the empty and the too-short city send no request. +- While a new answer is on its way, everything that reads `forecast` keeps showing the last one (the change is held, + H-2). To show that work is going on, read ahead: `pending? forecast` is `true` while it's in flight, and + `latest city` is what was typed. - The first load has nothing to show yet: wrap the readers in `loading fallback { <p>loading…</p> } { … }`. To show the fallback again each time a key changes (a new selection, not a refinement), use `loading on(selected)`. @@ -31,22 +31,22 @@ results = query == "" ? [] : len(query) < 2 ? throw TooShort(query) : await sear Give each pane its own, so one pane's failure never touches another. - `x catch (e) => e.message` turns a failure into a value, for showing it inline. - An answer that means "missing" (`null`) is a failure you declare: `export error NotFound(id: number) => …`, and - `found(await fetchItem(id), id)` with `fn found(it, id) => it == null ? throw NotFound(id) : it`. + `found(await fetchUser(id), id)` with `fn found(it, id) => it == null ? throw NotFound(id) : it`. - Retry is `refresh x`. It is quiet (AC-4): the failure stays on screen until the new answer replaces it (H-12). ## 3. Writing to the server: an action ``` -export source saved: Settings = await loadSettings() // a source with a question: it can take a guess +export source profile: Profile = await loadProfile() // a source with a question: it can take a guess export source saveError: string = "" -export action save(next: Settings) { - optimistic saved := next // shows now; reverts by itself if the save fails +export action save(next: Profile) { + optimistic profile := next // shows now; reverts by itself if the save fails optimistic saveError := "" // the old error goes now - affects saved // `pending? saved` is true until the save ends + affects profile // `pending? profile` is true until the save ends try { - let answer = await saveSettings(next) - saved := answer // the truth + let answer = await saveProfile(next) + profile := answer // the truth } catch (e) { saveError := e.message // lands in the same frame as the revert } @@ -72,7 +72,7 @@ by what the screen should do: - **Two actions guessing the same source revert together.** Their guesses are one change (H-11): a declined one goes back when the other ends. Don't rebuild guesses by hand to separate them. - **A click waits for an action it has nothing to do with.** A plain write to a source an action in flight has - written or guessed joins that action's hold (H-2): a `select` that clears the action's error waits for the + written or guessed joins that action's hold (H-2): a click that clears the action's error waits for the server. Keep such a write off the action's sources, or make it an `onFlush` run. - **A failure stays up while its retry is in flight.** That's H-12. To show `loading…` instead, put `loading on(k)` above the `errored`. From 4d70553de09cc1a99c9cb7c78bd805e8de32d902 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 19:40:39 +0000 Subject: [PATCH 02/11] eval: round 2 results (briefs that don't name the races) React 0/18 defects, .rx 1/18 (small model, tracker: two .rx traps). H1 not supported (Fisher p = 1.0). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/results/plain-2026-10-27/README.md | 48 +++ .../results/plain-2026-10-27/results-big.json | 290 +++++++++++++++++ .../plain-2026-10-27/results-small.json | 299 ++++++++++++++++++ .../runs/big/search-react-1/App.tsx | 152 +++++++++ .../runs/big/search-react-1/runlog.jsonl | 2 + .../runs/big/search-react-2/App.tsx | 136 ++++++++ .../runs/big/search-react-2/runlog.jsonl | 2 + .../runs/big/search-react-3/App.tsx | 151 +++++++++ .../runs/big/search-react-3/runlog.jsonl | 2 + .../runs/big/search-rx-1/main.rx | 55 ++++ .../runs/big/search-rx-1/runlog.jsonl | 4 + .../runs/big/search-rx-2/main.rx | 61 ++++ .../runs/big/search-rx-2/runlog.jsonl | 2 + .../runs/big/search-rx-3/main.rx | 57 ++++ .../runs/big/search-rx-3/runlog.jsonl | 5 + .../runs/big/settings-react-1/App.tsx | 140 ++++++++ .../runs/big/settings-react-1/runlog.jsonl | 4 + .../runs/big/settings-react-2/App.tsx | 186 +++++++++++ .../runs/big/settings-react-2/runlog.jsonl | 3 + .../runs/big/settings-react-3/App.tsx | 176 +++++++++++ .../runs/big/settings-react-3/runlog.jsonl | 2 + .../runs/big/settings-rx-1/main.rx | 64 ++++ .../runs/big/settings-rx-1/runlog.jsonl | 5 + .../runs/big/settings-rx-2/main.rx | 70 ++++ .../runs/big/settings-rx-2/runlog.jsonl | 5 + .../runs/big/settings-rx-3/main.rx | 70 ++++ .../runs/big/settings-rx-3/runlog.jsonl | 5 + .../runs/big/tracker-react-1/App.tsx | 184 +++++++++++ .../runs/big/tracker-react-1/runlog.jsonl | 2 + .../runs/big/tracker-react-2/App.tsx | 208 ++++++++++++ .../runs/big/tracker-react-2/runlog.jsonl | 2 + .../runs/big/tracker-react-3/App.tsx | 187 +++++++++++ .../runs/big/tracker-react-3/runlog.jsonl | 2 + .../runs/big/tracker-rx-1/main.rx | 67 ++++ .../runs/big/tracker-rx-1/runlog.jsonl | 2 + .../runs/big/tracker-rx-2/main.rx | 70 ++++ .../runs/big/tracker-rx-2/runlog.jsonl | 2 + .../runs/big/tracker-rx-3/main.rx | 61 ++++ .../runs/big/tracker-rx-3/runlog.jsonl | 3 + .../runs/small/search-react-1/App.tsx | 102 ++++++ .../runs/small/search-react-1/runlog.jsonl | 5 + .../runs/small/search-react-2/App.tsx | 134 ++++++++ .../runs/small/search-react-2/runlog.jsonl | 2 + .../runs/small/search-react-3/App.tsx | 114 +++++++ .../runs/small/search-react-3/runlog.jsonl | 2 + .../runs/small/search-rx-1/main.rx | 65 ++++ .../runs/small/search-rx-1/runlog.jsonl | 4 + .../runs/small/search-rx-2/main.rx | 66 ++++ .../runs/small/search-rx-2/runlog.jsonl | 6 + .../runs/small/search-rx-3/main.rx | 61 ++++ .../runs/small/search-rx-3/runlog.jsonl | 3 + .../runs/small/settings-react-1/App.tsx | 154 +++++++++ .../runs/small/settings-react-1/runlog.jsonl | 4 + .../runs/small/settings-react-2/App.tsx | 172 ++++++++++ .../runs/small/settings-react-2/runlog.jsonl | 2 + .../runs/small/settings-react-3/App.tsx | 157 +++++++++ .../runs/small/settings-react-3/runlog.jsonl | 2 + .../runs/small/settings-rx-1/main.rx | 78 +++++ .../runs/small/settings-rx-1/runlog.jsonl | 5 + .../runs/small/settings-rx-2/main.rx | 71 +++++ .../runs/small/settings-rx-2/runlog.jsonl | 10 + .../runs/small/settings-rx-3/main.rx | 47 +++ .../runs/small/settings-rx-3/runlog.jsonl | 4 + .../runs/small/settings-rx-3/store.rx | 19 ++ .../runs/small/tracker-react-1/App.tsx | 191 +++++++++++ .../runs/small/tracker-react-1/runlog.jsonl | 2 + .../runs/small/tracker-react-2/App.tsx | 215 +++++++++++++ .../runs/small/tracker-react-2/runlog.jsonl | 2 + .../runs/small/tracker-react-3/App.tsx | 178 +++++++++++ .../runs/small/tracker-react-3/runlog.jsonl | 2 + .../runs/small/tracker-rx-1/main.rx | 77 +++++ .../runs/small/tracker-rx-1/runlog.jsonl | 5 + .../runs/small/tracker-rx-2/main.rx | 71 +++++ .../runs/small/tracker-rx-2/runlog.jsonl | 14 + .../runs/small/tracker-rx-3/main.rx | 68 ++++ .../runs/small/tracker-rx-3/runlog.jsonl | 4 + eval/results/plain-2026-10-27/usage.txt | 36 +++ 77 files changed, 4940 insertions(+) create mode 100644 eval/results/plain-2026-10-27/README.md create mode 100644 eval/results/plain-2026-10-27/results-big.json create mode 100644 eval/results/plain-2026-10-27/results-small.json create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/search-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/settings-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/search-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/settings-rx-3/store.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-1/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-2/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-3/App.tsx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-react-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-1/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-2/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-3/main.rx create mode 100644 eval/results/plain-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl create mode 100644 eval/results/plain-2026-10-27/usage.txt diff --git a/eval/results/plain-2026-10-27/README.md b/eval/results/plain-2026-10-27/README.md new file mode 100644 index 0000000..3e27931 --- /dev/null +++ b/eval/results/plain-2026-10-27/README.md @@ -0,0 +1,48 @@ +# Round 2 result: briefs that don't name the races + +Pre-registration: `../../PREREG-plain.md` (commit 9c38a3f, before any run). 36 runs: 3 tasks × 2 arms × 2 models +(the frontier model, and Haiku as the small one) × 3 runs. No run had to be repeated. + +## Primary result: H1 not supported + +| Defects shipped | React | .rx | Fisher, two-sided | +| --- | --- | --- | --- | +| All (pre-registered test) | 0 / 18 | 1 / 18 | p = 1.0 | +| Frontier model | 0 / 9 | 0 / 9 | | +| Small model | 0 / 9 | 1 / 9 | | + +With the race sentences removed from the brief, **every React run still passed every hidden test**, at both model +sizes. The agents guarded stale answers without being told to: a sequence number per request, or TanStack Query keyed +by the query, filter or id. Their own reports say so ("a late answer to an older search … is ignored"). + +## The one defect (.rx, small model, tracker) + +`runs/small/tracker-rx-1/main.rx` fails 2 of 11 hidden tests. Both failures are .rx traps: + +- **A new assignment clears the previous error at once:** fails. The action clears the error with a plain write + (`assignError := ""`), and that write is held until the server answers (AC-1). This is the first trap + `guide/async.md` names. The agent had read the guide and still wrote it. +- **Selecting after not found shows loading, not the old message:** fails. The agent put `errored` *outside* + `loading on(selected)`. A new selection resets the loading boundary but not the error boundary above it, so the old + `not found` stays up. The guide's advice ("put `loading on(k)` above the `errored`") was given for retry, not + for a new selection. + +## Reported alongside + +| Mean per run | React, frontier | .rx, frontier | React, small | .rx, small | +| --- | --- | --- | --- | --- | +| App lines (non-blank, non-comment) | 154 | 58 | 140 | 61 | +| Agent tokens | 47.1k | 68.5k | 55.0k | 91.3k | + +- .rx apps are about 40% the size of the React apps, at both model sizes. +- .rx runs cost about 1.5× (frontier) and 1.7× (small) the tokens. They read the guide and parts of the spec. +- By their own reports, 13 of the 18 .rx runs wrote extra failure-path tests (in `tests/visible.mjs` or a scratch + copy) and removed them afterwards. No React run reported doing this. In every run, `tests/visible.mjs` at the end is + byte-identical to the task's. + +## What this means + +On these three apps, React is not a source of race bugs for current models, even when the brief doesn't name the +races. A defect gap has to come from somewhere else. What the agents' own reports flag as unsure, and the hidden tests +don't probe: overlapping mutations (two assignments or two saves in flight), an optimistic write racing a refetch of +the same list, and changes to code that already exists. See the PR or the thread for the proposed next round. diff --git a/eval/results/plain-2026-10-27/results-big.json b/eval/results/plain-2026-10-27/results-big.json new file mode 100644 index 0000000..a34aef8 --- /dev/null +++ b/eval/results/plain-2026-10-27/results-big.json @@ -0,0 +1,290 @@ +[ + { + "run": "search-react-1", + "task": "search", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-2", + "task": "search", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-3", + "task": "search", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-1", + "task": "search", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-2", + "task": "search", + "arm": "rx", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-3", + "task": "search", + "arm": "rx", + "n": 3, + "checks": 2, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-1", + "task": "settings", + "arm": "react", + "n": 1, + "checks": 3, + "tests": 1, + "cyclesToGreen": 4, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-2", + "task": "settings", + "arm": "react", + "n": 2, + "checks": 2, + "tests": 1, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-3", + "task": "settings", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-1", + "task": "settings", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 4, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-2", + "task": "settings", + "arm": "rx", + "n": 2, + "checks": 2, + "tests": 3, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-3", + "task": "settings", + "arm": "rx", + "n": 3, + "checks": 2, + "tests": 3, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-1", + "task": "tracker", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-2", + "task": "tracker", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-3", + "task": "tracker", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-1", + "task": "tracker", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-2", + "task": "tracker", + "arm": "rx", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-3", + "task": "tracker", + "arm": "rx", + "n": 3, + "checks": 1, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + } +] diff --git a/eval/results/plain-2026-10-27/results-small.json b/eval/results/plain-2026-10-27/results-small.json new file mode 100644 index 0000000..81ca0e7 --- /dev/null +++ b/eval/results/plain-2026-10-27/results-small.json @@ -0,0 +1,299 @@ +[ + { + "run": "search-react-1", + "task": "search", + "arm": "react", + "n": 1, + "checks": 3, + "tests": 2, + "cyclesToGreen": 5, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-2", + "task": "search", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-3", + "task": "search", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-1", + "task": "search", + "arm": "rx", + "n": 1, + "checks": 2, + "tests": 2, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-2", + "task": "search", + "arm": "rx", + "n": 2, + "checks": 3, + "tests": 3, + "cyclesToGreen": 6, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-3", + "task": "search", + "arm": "rx", + "n": 3, + "checks": 1, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-1", + "task": "settings", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-2", + "task": "settings", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-3", + "task": "settings", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-1", + "task": "settings", + "arm": "rx", + "n": 1, + "checks": 2, + "tests": 3, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-2", + "task": "settings", + "arm": "rx", + "n": 2, + "checks": 6, + "tests": 4, + "cyclesToGreen": 5, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-3", + "task": "settings", + "arm": "rx", + "n": 3, + "checks": 2, + "tests": 2, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-1", + "task": "tracker", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-2", + "task": "tracker", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-3", + "task": "tracker", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-1", + "task": "tracker", + "arm": "rx", + "n": 1, + "checks": 3, + "tests": 2, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "9/11", + "shippedDefect": true, + "failedHidden": [ + { + "name": "a new assignment clears the previous error at once", + "error": "expected [detail] not to show \"declined\", it shows \"#3 Crash on savestack trace attachedassign to bodeclined\"" + }, + { + "name": "selecting after not found shows loading, not the old message", + "error": "expected [detail] to show \"loading #1\", it shows \"detail error: issue #3 not foundretry\"" + } + ], + "failedVisible": [] + }, + { + "run": "tracker-rx-2", + "task": "tracker", + "arm": "rx", + "n": 2, + "checks": 7, + "tests": 7, + "cyclesToGreen": 14, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-3", + "task": "tracker", + "arm": "rx", + "n": 3, + "checks": 3, + "tests": 1, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + } +] diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/big/search-react-1/App.tsx new file mode 100644 index 0000000..b676377 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-1/App.tsx @@ -0,0 +1,152 @@ +import { useRef, useState } from "react"; +import type { ReactNode } from "react"; +import { fetchItem, search } from "./api.js"; +import type { Hit, Item } from "./api.js"; + +type Shown = { q: string; hits: Hit[] }; +type SearchState = + | { status: "idle" } + | { status: "loading"; q: string } + | { status: "done"; q: string } + | { status: "error"; q: string; message: string }; + +type Detail = + | { status: "none" } + | { status: "loading"; id: number } + | { status: "item"; id: number; item: Item } + | { status: "missing"; id: number } + | { status: "error"; id: number; message: string }; + +const messageOf = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [query, setQuery] = useState(""); + const [searchState, setSearchState] = useState<SearchState>({ status: "idle" }); + const [shown, setShown] = useState<Shown | null>(null); + const [selectedId, setSelectedId] = useState<number | null>(null); + const [detail, setDetail] = useState<Detail>({ status: "none" }); + const searchSeq = useRef(0); + const detailSeq = useRef(0); + + function onQuery(q: string) { + setQuery(q); + const seq = ++searchSeq.current; + if (q.length < 2) { + setSearchState({ status: "idle" }); + setShown(null); + return; + } + setSearchState({ status: "loading", q }); + search(q).then( + hits => { + if (seq !== searchSeq.current) return; + setShown({ q, hits }); + setSearchState({ status: "done", q }); + }, + (e: unknown) => { + if (seq !== searchSeq.current) return; + setShown(null); + setSearchState({ status: "error", q, message: messageOf(e) }); + }, + ); + } + + function load(id: number) { + const seq = ++detailSeq.current; + setDetail({ status: "loading", id }); + fetchItem(id).then( + item => { + if (seq !== detailSeq.current) return; + setDetail(item ? { status: "item", id, item } : { status: "missing", id }); + }, + (e: unknown) => { + if (seq !== detailSeq.current) return; + setDetail({ status: "error", id, message: messageOf(e) }); + }, + ); + } + + function select(id: number) { + if (id === selectedId) return; + setSelectedId(id); + load(id); + } + + let results: ReactNode = null; + if (query.length === 1) { + results = <p>too short: {query}</p>; + } else if (query.length >= 2) { + results = ( + <> + {searchState.status === "loading" && <p>searching…</p>} + {searchState.status === "error" && <p>search failed: {searchState.message}</p>} + {shown && ( + <> + <p> + {shown.hits.length} results for {shown.q} + </p> + <ul> + {shown.hits.map(hit => ( + <li + key={hit.id} + data-testid="hit" + onClick={() => select(hit.id)} + style={{ cursor: "pointer" }} + > + {hit.id === selectedId ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + )} + </> + ); + } + + let detailBody: ReactNode; + switch (detail.status) { + case "none": + detailBody = <p>pick a result</p>; + break; + case "loading": + detailBody = <p>loading…</p>; + break; + case "item": + detailBody = ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + ); + break; + case "missing": + detailBody = <p>item {detail.id} not found</p>; + break; + case "error": { + const id = detail.id; + detailBody = ( + <> + <p>failed: {detail.message}</p> + <button type="button" data-testid="retry" onClick={() => load(id)}> + retry + </button> + </> + ); + break; + } + } + + return ( + <main> + <input + data-testid="query" + value={query} + onChange={e => onQuery(e.target.value)} + placeholder="search" + autoFocus + /> + <section data-testid="results">{results}</section> + <section data-testid="detail">{detailBody}</section> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-react-1/runlog.jsonl new file mode 100644 index 0000000..2ecd8d7 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:02:51.414Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:02:56.049Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/big/search-react-2/App.tsx new file mode 100644 index 0000000..30bb92f --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-2/App.tsx @@ -0,0 +1,136 @@ +import { useRef, useState } from "react"; +import { search, fetchItem } from "./api.js"; +import type { Hit, Item } from "./api.js"; + +type SearchState = + | { status: "idle" } + | { status: "searching"; shown: { q: string; hits: Hit[] } | null } + | { status: "done"; shown: { q: string; hits: Hit[] } } + | { status: "error"; message: string }; + +type DetailState = + | { status: "none" } + | { status: "loading"; id: number } + | { status: "item"; id: number; item: Item } + | { status: "missing"; id: number } + | { status: "error"; id: number; message: string }; + +const errMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [query, setQuery] = useState(""); + const [results, setResults] = useState<SearchState>({ status: "idle" }); + const [selected, setSelected] = useState<number | null>(null); + const [detail, setDetail] = useState<DetailState>({ status: "none" }); + const searchToken = useRef(0); + const detailToken = useRef(0); + + function onQuery(q: string) { + setQuery(q); + const token = ++searchToken.current; + if (q.length < 2) { + setResults({ status: "idle" }); + return; + } + setResults(prev => ({ + status: "searching", + shown: prev.status === "done" ? prev.shown : prev.status === "searching" ? prev.shown : null, + })); + search(q).then( + hits => { + if (token === searchToken.current) setResults({ status: "done", shown: { q, hits } }); + }, + (e: unknown) => { + if (token === searchToken.current) setResults({ status: "error", message: errMessage(e) }); + }, + ); + } + + function load(id: number) { + const token = ++detailToken.current; + setDetail({ status: "loading", id }); + fetchItem(id).then( + item => { + if (token !== detailToken.current) return; + setDetail(item ? { status: "item", id, item } : { status: "missing", id }); + }, + (e: unknown) => { + if (token === detailToken.current) setDetail({ status: "error", id, message: errMessage(e) }); + }, + ); + } + + function select(id: number) { + if (id === selected && detail.status !== "none") return; + setSelected(id); + load(id); + } + + let resultsBody: React.ReactNode = null; + if (query.length === 1) { + resultsBody = <p>too short: {query}</p>; + } else if (query.length >= 2) { + const shown = results.status === "done" || results.status === "searching" ? results.shown : null; + resultsBody = ( + <> + {results.status === "searching" && <p>searching…</p>} + {results.status === "error" && <p>search failed: {results.message}</p>} + {shown && ( + <> + <p> + {shown.hits.length} results for {shown.q} + </p> + <ul> + {shown.hits.map(hit => ( + <li key={hit.id} data-testid="hit" onClick={() => select(hit.id)} style={{ cursor: "pointer" }}> + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + )} + </> + ); + } + + let detailBody: React.ReactNode; + switch (detail.status) { + case "none": + detailBody = <p>pick a result</p>; + break; + case "loading": + detailBody = <p>loading…</p>; + break; + case "item": + detailBody = ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + ); + break; + case "missing": + detailBody = <p>item {detail.id} not found</p>; + break; + case "error": { + const id = detail.id; + detailBody = ( + <> + <p>failed: {detail.message}</p> + <button data-testid="retry" onClick={() => load(id)}> + retry + </button> + </> + ); + break; + } + } + + return ( + <main> + <input data-testid="query" value={query} onChange={e => onQuery(e.target.value)} placeholder="search" /> + <section data-testid="results">{resultsBody}</section> + <section data-testid="detail">{detailBody}</section> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-react-2/runlog.jsonl new file mode 100644 index 0000000..ae961f1 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:03:38.791Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:47.691Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/big/search-react-3/App.tsx new file mode 100644 index 0000000..5e4289c --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-3/App.tsx @@ -0,0 +1,151 @@ +import { useRef, useState } from "react"; +import type { ChangeEvent } from "react"; +import { search, fetchItem } from "./api.js"; +import type { Hit, Item } from "./api.js"; + +type Results = + | { kind: "idle" } + | { kind: "done"; q: string; hits: Hit[] } + | { kind: "error"; message: string }; + +type Detail = + | { kind: "none" } + | { kind: "loading"; id: number } + | { kind: "item"; item: Item } + | { kind: "missing"; id: number } + | { kind: "error"; id: number; message: string }; + +const messageOf = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [query, setQuery] = useState(""); + const [searching, setSearching] = useState(false); + const [results, setResults] = useState<Results>({ kind: "idle" }); + const [selected, setSelected] = useState<number | null>(null); + const [detail, setDetail] = useState<Detail>({ kind: "none" }); + const searchSeq = useRef(0); + const detailSeq = useRef(0); + + function onChange(e: ChangeEvent<HTMLInputElement>) { + const q = e.target.value; + setQuery(q); + const token = ++searchSeq.current; + if (q.length < 2) { + setSearching(false); + setResults({ kind: "idle" }); + return; + } + setSearching(true); + search(q).then( + hits => { + if (token !== searchSeq.current) return; + setSearching(false); + setResults({ kind: "done", q, hits }); + }, + (err: unknown) => { + if (token !== searchSeq.current) return; + setSearching(false); + setResults({ kind: "error", message: messageOf(err) }); + }, + ); + } + + function load(id: number) { + const token = ++detailSeq.current; + setDetail({ kind: "loading", id }); + fetchItem(id).then( + item => { + if (token !== detailSeq.current) return; + setDetail(item ? { kind: "item", item } : { kind: "missing", id }); + }, + (err: unknown) => { + if (token !== detailSeq.current) return; + setDetail({ kind: "error", id, message: messageOf(err) }); + }, + ); + } + + function select(id: number) { + if (id === selected) return; + setSelected(id); + load(id); + } + + let resultsBody: React.ReactNode = null; + if (query.length === 1) { + resultsBody = <p>too short: {query}</p>; + } else if (query.length >= 2) { + resultsBody = ( + <> + {searching && <p>searching…</p>} + {results.kind === "error" && !searching && <p>search failed: {results.message}</p>} + {results.kind === "done" && ( + <> + <p> + {results.hits.length} results for {results.q} + </p> + <ul> + {results.hits.map(hit => ( + <li + key={hit.id} + data-testid="hit" + onClick={() => select(hit.id)} + style={{ cursor: "pointer" }} + > + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + )} + </> + ); + } + + let detailBody: React.ReactNode; + switch (detail.kind) { + case "none": + detailBody = <p>pick a result</p>; + break; + case "loading": + detailBody = <p>loading…</p>; + break; + case "item": + detailBody = ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + ); + break; + case "missing": + detailBody = <p>item {detail.id} not found</p>; + break; + case "error": { + const id = detail.id; + detailBody = ( + <> + <p>failed: {detail.message}</p> + <button data-testid="retry" onClick={() => load(id)}> + retry + </button> + </> + ); + break; + } + } + + return ( + <main> + <input + data-testid="query" + value={query} + onChange={onChange} + placeholder="search" + autoFocus + /> + <section data-testid="results">{resultsBody}</section> + <section data-testid="detail">{detailBody}</section> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-react-3/runlog.jsonl new file mode 100644 index 0000000..5e31218 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:36.526Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:04:44.656Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/big/search-rx-1/main.rx new file mode 100644 index 0000000..078042c --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-1/main.rx @@ -0,0 +1,55 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +error NotFound(id: number) => `item ${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source query: string := "" + source selected: number := 0 + + res = query == "" ? { kind: "none", q: query, hits: [], msg: "" } + : len(query) < 2 ? { kind: "short", q: query, hits: [], msg: "" } + : { kind: "ok", q: query, hits: await search(query), msg: "" } + catch (e) => { kind: "err", q: query, hits: [], msg: e.message } + + <main> + <input data-testid="query" onInput={(e) => { query := e.currentTarget.value }} /> + <div data-testid="results"> + {loading fallback { <p>searching…</p> } { + {if pending? res { <p>searching…</p> }} + {if res.kind == "short" { <p>too short: {res.q}</p> }} + {if res.kind == "err" { <p>search failed: {res.msg}</p> }} + {if res.kind == "ok" { + <p>{len(res.hits)} results for {res.q}</p> + <ul> + {for h in res.hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{selected == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + </div> + <div data-testid="detail"> + {if selected == 0 { + <p>pick a result</p> + } else { + <Detail id={selected} /> + }} + </div> + </main> +} + +component Detail(props: { id: number }) { + item = found(await fetchItem(props.id), props.id) + action retry() { refresh item } + loading on(props.id) fallback { <p>loading…</p> } { + errored fallback(e) { <p>failed: {e.message}</p> <button data-testid="retry" onClick={() => { retry() }}>retry</button> } { + errored fallback(e: NotFound) { <p>{e.message}</p> } { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + } + } + } +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-rx-1/runlog.jsonl new file mode 100644 index 0000000..c4a59fa --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:02:36.213Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:02:43.839Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:03:04.937Z","cmd":"test","ok":true,"build":true,"passed":5,"total":5} +{"at":"2026-10-27T19:03:21.649Z","cmd":"test","ok":true,"build":true,"passed":6,"total":6} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/big/search-rx-2/main.rx new file mode 100644 index 0000000..92e71f1 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-2/main.rx @@ -0,0 +1,61 @@ +import pure { search, fetchItem } from "./api" +import { Hit, Item } from "./api" +import pure { len } from "./prelude" + +error NotFound(id: number) => `item ${id} not found` + +fn found(it: Item | null, id: number): Item => it == null ? throw NotFound(id) : it + +type Outcome = { ok: boolean; hits: Hit[]; message: string } + +export default component App() { + source query: string := "" + source selected: Hit | null := null + + outcome: Outcome = (len(query) < 2 ? { ok: true, hits: [], message: "" } : { ok: true, hits: await search(query), message: "" }) catch (e) => ({ ok: false, hits: [], message: e.message }) + typed = latest query + + <main> + <input data-testid="query" value={typed} onInput={(ev) => { query := ev.target.value }} /> + <div data-testid="results"> + {if len(typed) == 1 { + <p>too short: {typed}</p> + } else { + {if len(typed) > 1 { + {loading fallback { <p>searching…</p> } { + {if pending? outcome { <p>searching…</p> }} + {if outcome.ok { + <p>{outcome.hits.length} results for {query}</p> + <ul> + {for h in outcome.hits { + <li data-testid="hit" onClick={() => { selected := h }}>{selected?.id == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + } else { + <p>search failed: {outcome.message}</p> + }} + }} + }} + }} + </div> + <div data-testid="detail"> + {if selected { + item = found(await fetchItem(selected.id), selected.id) + action retry() { refresh item } + {loading on(selected) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { retry() }}>retry</button> + } { + {errored fallback(e: NotFound) { <p>{e.message}</p> } { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + } else { + <p>pick a result</p> + }} + </div> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-rx-2/runlog.jsonl new file mode 100644 index 0000000..6efd61d --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:18.252Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:04:26.137Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/big/search-rx-3/main.rx new file mode 100644 index 0000000..e63f164 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-3/main.rx @@ -0,0 +1,57 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +export default component App() { + source q: string := "" + source selected: number | null := null + + item = selected == null ? null : await fetchItem(selected) + + <main> + <input data-testid="query" value={q} onInput={(e) => { q := e.target.value }} /> + <section data-testid="results"> + {if q == "" { + } else { + {if len(q) < 2 { + <p>too short: {q}</p> + } else { + found = { q: q, err: "", hits: await search(q) } catch (e) => ({ q: q, err: e.message, hits: [] }) + {loading fallback { <p>searching…</p> } { + {if pending? found { + <p>searching…</p> + }} + {if found.err != "" { + <p>search failed: {found.err}</p> + } else { + <p>{len(found.hits)} results for {found.q}</p> + <ul> + {for h in found.hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{h.id == selected ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + </section> + <section data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + {loading on(selected) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {if item { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + } else { + <p>item {selected} not found</p> + }} + }} + }} + }} + </section> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/search-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/search-rx-3/runlog.jsonl new file mode 100644 index 0000000..452c5f8 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/search-rx-3/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:09:16.566Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:09:22.237Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:09:41.939Z","cmd":"test","ok":false,"build":true,"passed":6,"total":7} +{"at":"2026-10-27T19:09:57.187Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} +{"at":"2026-10-27T19:10:03.228Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/big/settings-react-1/App.tsx new file mode 100644 index 0000000..fceb7dc --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-1/App.tsx @@ -0,0 +1,140 @@ +import { useEffect, useRef, useState } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings, Density } from "./api.js"; + +type LoadState = + | { status: "loading" } + | { status: "error"; message: string } + | { status: "loaded" }; + +const MAX = 20; + +function errMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function line(s: Settings): string { + return `Saved: ${s.name} · ${s.theme} · ${s.density}`; +} + +export default function App() { + const [load, setLoad] = useState<LoadState>({ status: "loading" }); + const [confirmed, setConfirmed] = useState<{ seq: number; settings: Settings } | null>(null); + const [pending, setPending] = useState<{ seq: number; settings: Settings } | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + const [draft, setDraft] = useState<Settings | null>(null); + const started = useRef(false); + const loadSeq = useRef(0); + const saveSeq = useRef(0); + + function retryLoad() { + setLoad({ status: "loading" }); + fetchSettings(); + } + + function fetchSettings() { + const my = ++loadSeq.current; + loadSettings().then( + s => { + if (my !== loadSeq.current) return; + setLoad({ status: "loaded" }); + setConfirmed({ seq: 0, settings: s }); + setDraft({ name: s.name, theme: s.theme, density: s.density }); + }, + (e: unknown) => { + if (my !== loadSeq.current) return; + setLoad({ status: "error", message: errMessage(e) }); + }, + ); + } + + useEffect(() => { + if (started.current) return; + started.current = true; + fetchSettings(); + }, []); + + function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX) return `Display name must be at most ${MAX} characters, not ${name.length}`; + return null; + } + + function save() { + if (!draft || nameError(draft.name)) return; + const settings: Settings = { ...draft }; + const seq = ++saveSeq.current; + setPending({ seq, settings }); + setSaveError(null); + saveSettings(settings).then( + saved => { + setConfirmed(c => (c && c.seq > seq ? c : { seq, settings: saved })); + setPending(p => (p && p.seq === seq ? null : p)); + }, + (e: unknown) => { + setPending(p => (p && p.seq === seq ? null : p)); + if (seq === saveSeq.current) setSaveError(`Save failed: ${errMessage(e)}`); + }, + ); + } + + if (load.status !== "loaded" || !draft) { + return ( + <main> + <h1>Settings</h1> + {load.status === "error" ? ( + <> + <p data-testid="load">Could not load settings: {load.message}</p> + <button data-testid="retry-load" onClick={retryLoad}>Retry</button> + </> + ) : ( + <p data-testid="load">Loading settings…</p> + )} + </main> + ); + } + + const error = nameError(draft.name); + const savedText = pending + ? `${line(pending.settings)} (saving…)` + : confirmed + ? line(confirmed.settings) + : ""; + const nextDensity: Density = draft.density === "comfortable" ? "compact" : "comfortable"; + + return ( + <main> + <h1>Settings</h1> + <p data-testid="saved">{savedText}</p> + <form onSubmit={e => { e.preventDefault(); save(); }}> + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={e => { const name = e.target.value; setDraft(d => (d ? { ...d, name } : d)); }} + /> + </label>{" "} + <span data-testid="name-count">{draft.name.length}/{MAX}</span> + {error && <p data-testid="name-error" role="alert">{error}</p>} + <div> + Theme:{" "} + <button type="button" data-testid="theme-light" aria-pressed={draft.theme === "light"} + onClick={() => setDraft(d => (d ? { ...d, theme: "light" } : d))}>Light</button>{" "} + <button type="button" data-testid="theme-dark" aria-pressed={draft.theme === "dark"} + onClick={() => setDraft(d => (d ? { ...d, theme: "dark" } : d))}>Dark</button> + </div> + <div> + Density:{" "} + <button type="button" data-testid="density-toggle" + onClick={() => setDraft(d => (d ? { ...d, density: d.density === "comfortable" ? "compact" : "comfortable" } : d))}> + {draft.density} (switch to {nextDensity}) + </button> + </div> + <p data-testid="preview">Preview: {draft.theme} / {draft.density}</p> + <button type="submit" data-testid="save">Save</button> + {saveError && <p data-testid="save-error" role="alert">{saveError}</p>} + </form> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-react-1/runlog.jsonl new file mode 100644 index 0000000..798c82e --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:02:24.163Z","cmd":"check","ok":true,"out":"/home/claude/runs/plain-big/settings-react-1/src/App.tsx\n 51:5 warning Unused eslint-disable directive (no problems were reported from 'react-hooks/exhaustive-deps')\n\n✖ 1 problem (0 errors, 1 warning)\n 0 errors and 1 warning potentially fixable with the `--fix` option."} +{"at":"2026-10-27T19:02:30.722Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-big/settings-react-1/src/App.tsx\n 50:5 error Error: Calling setState synchronously within an effect can trigger cascading renders\n\nEffects are intended to synchronize state between React and external systems such as manually updating the DOM, state management libraries, or other platform APIs. In general, the body of an effect should do one or both of the following:\n* Update external systems with the latest state from React.\n* Subscribe for updates from some external system, calling setState in a callback function when external state changes.\n\nCalling setState synchronously within an effect body causes cascading renders that can hurt performance, and is not recommended. (https://react.dev/learn/you-might-not-need-an-effect).\n\n/home/claude/runs/plain-big/settings-react-1/src/App.tsx:50:5\n 48 | if (started.current) return;\n 49 | started.current = true;\n> 50 | startLoad();\n | ^^^^^^^^^ Avoid calling setState() directly within an effect\n 51 | }, []);\n 52 |\n 53 | function nameError(name: string): string | null { react-hooks/set-state-in-effect\n\n✖ 1 problem (1 error, 0 warnings)"} +{"at":"2026-10-27T19:02:37.764Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:02:44.208Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/big/settings-react-2/App.tsx new file mode 100644 index 0000000..09b8c8f --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-2/App.tsx @@ -0,0 +1,186 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings } from "./api.js"; + +type LoadState = + | { status: "loading" } + | { status: "error"; message: string } + | { status: "loaded" }; + +type Pending = { seq: number; settings: Settings }; + +const MAX_NAME = 20; + +function messageOf(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX_NAME) return `Display name must be at most ${MAX_NAME} characters, not ${name.length}`; + return null; +} + +function line(s: Settings): string { + return `Saved: ${s.name} · ${s.theme} · ${s.density}`; +} + +export default function App() { + const [load, setLoad] = useState<LoadState>({ status: "loading" }); + const [saved, setSaved] = useState<Settings | null>(null); + const [draft, setDraft] = useState<Settings | null>(null); + const [pending, setPending] = useState<Pending | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + + const loadSeq = useRef(0); + const started = useRef(false); + const saveSeq = useRef(0); + const confirmedSeq = useRef(0); + + const fetchSettings = useCallback(() => { + const seq = ++loadSeq.current; + loadSettings().then( + (s) => { + if (seq !== loadSeq.current) return; + setSaved(s); + setDraft((d) => d ?? s); + setLoad({ status: "loaded" }); + }, + (e: unknown) => { + if (seq !== loadSeq.current) return; + setLoad({ status: "error", message: messageOf(e) }); + }, + ); + }, []); + + useEffect(() => { + if (started.current) return; + started.current = true; + fetchSettings(); + }, [fetchSettings]); + + const retryLoad = () => { + setLoad({ status: "loading" }); + fetchSettings(); + }; + + const save = () => { + if (!draft || nameError(draft.name) !== null) return; + const toSave: Settings = { ...draft }; + const seq = ++saveSeq.current; + setPending({ seq, settings: toSave }); + setSaveError(null); + saveSettings(toSave).then( + (result) => { + if (seq > confirmedSeq.current) { + confirmedSeq.current = seq; + setSaved(result); + } + if (seq === saveSeq.current) setPending(null); + }, + (e: unknown) => { + if (seq === saveSeq.current) { + setPending(null); + setSaveError(`Save failed: ${messageOf(e)}`); + } + }, + ); + }; + + if (load.status !== "loaded" || !saved || !draft) { + return ( + <main> + <h1>Settings</h1> + {load.status === "error" ? ( + <> + <p data-testid="load" role="alert"> + Could not load settings: {load.message} + </p> + <button type="button" data-testid="retry-load" onClick={retryLoad}> + Retry + </button> + </> + ) : ( + <p data-testid="load">Loading settings…</p> + )} + </main> + ); + } + + const error = nameError(draft.name); + const savedText = pending ? `${line(pending.settings)} (saving…)` : line(saved); + + return ( + <main> + <h1>Settings</h1> + <p data-testid="saved">{savedText}</p> + <form + onSubmit={(e) => { + e.preventDefault(); + save(); + }} + > + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + aria-invalid={error !== null} + onChange={(e) => { + const name = e.target.value; + setDraft((d) => (d ? { ...d, name } : d)); + }} + /> + </label> + <span data-testid="name-count"> + {draft.name.length}/{MAX_NAME} + </span> + {error !== null && ( + <p data-testid="name-error" role="alert"> + {error} + </p> + )} + <div> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => setDraft((d) => (d ? { ...d, theme: "light" } : d))} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => setDraft((d) => (d ? { ...d, theme: "dark" } : d))} + > + Dark + </button> + <button + type="button" + data-testid="density-toggle" + onClick={() => + setDraft((d) => + d ? { ...d, density: d.density === "compact" ? "comfortable" : "compact" } : d, + ) + } + > + Density: {draft.density} + </button> + </div> + <p data-testid="preview"> + Preview: {draft.theme} / {draft.density} + </p> + <button type="submit" data-testid="save"> + Save + </button> + {saveError !== null && ( + <p data-testid="save-error" role="alert"> + {saveError} + </p> + )} + </form> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-react-2/runlog.jsonl new file mode 100644 index 0000000..cc8a32d --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-2/runlog.jsonl @@ -0,0 +1,3 @@ +{"at":"2026-10-27T19:03:36.057Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-big/settings-react-2/src/App.tsx\n 60:5 error Error: Calling setState synchronously within an effect can trigger cascading renders\n\nEffects are intended to synchronize state between React and external systems such as manually updating the DOM, state management libraries, or other platform APIs. In general, the body of an effect should do one or both of the following:\n* Update external systems with the latest state from React.\n* Subscribe for updates from some external system, calling setState in a callback function when external state changes.\n\nCalling setState synchronously within an effect body causes cascading renders that can hurt performance, and is not recommended. (https://react.dev/learn/you-might-not-need-an-effect).\n\n/home/claude/runs/plain-big/settings-react-2/src/App.tsx:60:5\n 58 | if (started.current) return;\n 59 | started.current = true;\n> 60 | runLoad();\n | ^^^^^^^ Avoid calling setState() directly within an effect\n 61 | }, [runLoad]);\n 62 |\n 63 | const save = () => { react-hooks/set-state-in-effect\n\n✖ 1 problem (1 error, 0 warnings)"} +{"at":"2026-10-27T19:03:45.298Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:51.312Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/big/settings-react-3/App.tsx new file mode 100644 index 0000000..14612b4 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-3/App.tsx @@ -0,0 +1,176 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings, Theme, Density } from "./api.js"; + +const MAX_NAME = 20; + +function makeClient() { + return new QueryClient({ + defaultOptions: { + queries: { + retry: false, + staleTime: Infinity, + gcTime: Infinity, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + refetchOnMount: false, + }, + }, + }); +} + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX_NAME) return `Display name must be at most ${MAX_NAME} characters, not ${name.length}`; + return null; +} + +function line(s: Settings): string { + return `Saved: ${s.name} · ${s.theme} · ${s.density}`; +} + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +type SaveState = { + confirmed: Settings; + confirmedSeq: number; + issued: number; + pending: { seq: number; values: Settings }[]; + error: string | null; +}; + +function Editor({ initial }: { initial: Settings }) { + const [draft, setDraft] = useState<Settings>(() => ({ ...initial })); + const [save, setSave] = useState<SaveState>(() => ({ + confirmed: initial, + confirmedSeq: 0, + issued: 0, + pending: [], + error: null, + })); + + const error = nameError(draft.name); + + function onSave() { + if (error) return; + const values: Settings = { name: draft.name, theme: draft.theme, density: draft.density }; + const seq = save.issued + 1; + setSave(s => ({ ...s, issued: seq, pending: [...s.pending, { seq, values }], error: null })); + saveSettings(values).then( + saved => { + setSave(s => { + const pending = s.pending.filter(p => p.seq !== seq); + if (seq > s.confirmedSeq) return { ...s, pending, confirmed: saved, confirmedSeq: seq }; + return { ...s, pending }; + }); + }, + (e: unknown) => { + setSave(s => { + const pending = s.pending.filter(p => p.seq !== seq); + const latest = seq === s.issued; + return { ...s, pending, error: latest ? `Save failed: ${errorMessage(e)}` : s.error }; + }); + }, + ); + } + + const newestPending = save.pending.reduce<{ seq: number; values: Settings } | null>( + (a, p) => (a === null || p.seq > a.seq ? p : a), + null, + ); + const savedText = + newestPending && newestPending.seq > save.confirmedSeq + ? `${line(newestPending.values)} (saving…)` + : line(save.confirmed); + + const setTheme = (theme: Theme) => setDraft(d => ({ ...d, theme })); + const toggleDensity = () => + setDraft(d => ({ ...d, density: (d.density === "comfortable" ? "compact" : "comfortable") as Density })); + + return ( + <div> + <p data-testid="saved">{savedText}</p> + <form + onSubmit={e => { + e.preventDefault(); + onSave(); + }} + > + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={e => { + const name = e.target.value; + setDraft(d => ({ ...d, name })); + }} + aria-invalid={error ? true : undefined} + /> + </label> + <span data-testid="name-count"> + {draft.name.length}/{MAX_NAME} + </span> + {error && ( + <p data-testid="name-error" role="alert"> + {error} + </p> + )} + <div> + <button type="button" data-testid="theme-light" aria-pressed={draft.theme === "light"} onClick={() => setTheme("light")}> + Light + </button> + <button type="button" data-testid="theme-dark" aria-pressed={draft.theme === "dark"} onClick={() => setTheme("dark")}> + Dark + </button> + <button type="button" data-testid="density-toggle" onClick={toggleDensity}> + Density: {draft.density} + </button> + </div> + <p data-testid="preview"> + Preview: {draft.theme} / {draft.density} + </p> + <button type="submit" data-testid="save"> + Save + </button> + {save.error && ( + <p data-testid="save-error" role="alert"> + {save.error} + </p> + )} + </form> + </div> + ); +} + +function SettingsPage() { + const query = useQuery({ queryKey: ["settings"], queryFn: () => loadSettings() }); + + if (query.data) return <Editor initial={query.data} />; + + if (query.isError && !query.isFetching) { + return ( + <div> + <p data-testid="load">Could not load settings: {errorMessage(query.error)}</p> + <button type="button" data-testid="retry-load" onClick={() => void query.refetch()}> + Retry + </button> + </div> + ); + } + + return <p data-testid="load">Loading settings…</p>; +} + +export default function App() { + const [client] = useState(makeClient); + return ( + <QueryClientProvider client={client}> + <h1>Settings</h1> + <SettingsPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-react-3/runlog.jsonl new file mode 100644 index 0000000..fc2db57 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:53.056Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:04:58.782Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/big/settings-rx-1/main.rx new file mode 100644 index 0000000..d9916f8 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-1/main.rx @@ -0,0 +1,64 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +type Settings = { name: string; theme: string; density: string } + +fn nameError(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +export default component App() { + source settings: Settings = await loadSettings() + source saveError: string = "" + + action save(next: Settings) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } + } + + <main> + {errored fallback(e) { + <p data-testid="load">Could not load settings: {e.message}</p> + <button data-testid="retry-load" onClick={() => { refresh settings }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + source name := peek(settings.name) + source theme := peek(settings.theme) + source density := peek(settings.density) + err = nameError(name) + + <p data-testid="saved">Saved: {settings.name} · {settings.theme} · {settings.density}{pending? settings ? " (saving…)" : ""}</p> + <form onSubmit={(ev) => { ev.preventDefault() }}> + <label>Display name{" "} + <input data-testid="name" value={name} onInput={(ev) => { name := ev.target.value }} /> + </label> + <span data-testid="name-count">{len(name)}/20</span> + {if err != "" { + <p data-testid="name-error">{err}</p> + }} + <div> + <button type="button" data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button type="button" data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button type="button" data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">Preview: {theme} / {density}</p> + <button type="button" data-testid="save" onClick={() => { + if (nameError(name) == "") { save({ name: name, theme: theme, density: density }) } + }}>Save</button> + {if saveError != "" { + <p data-testid="save-error">Save failed: {saveError}</p> + }} + </form> + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-rx-1/runlog.jsonl new file mode 100644 index 0000000..9727b13 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:02:38.350Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:02:46.828Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:03:19.734Z","cmd":"test","ok":false,"build":true,"passed":6,"total":7} +{"at":"2026-10-27T19:03:34.851Z","cmd":"test","ok":false,"build":true,"passed":6,"total":7} +{"at":"2026-10-27T19:03:49.167Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/big/settings-rx-2/main.rx new file mode 100644 index 0000000..74a30e4 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-2/main.rx @@ -0,0 +1,70 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import { Settings } from "./api" +import pure { len } from "./prelude" + +source settings: Settings = await loadSettings() +source saveError: string = "" + +action save(next: Settings) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } +} + +fn nameError(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +component Form(props: { initial: Settings }) { + source name: string := peek(props.initial.name) + source theme: string := peek(props.initial.theme) + source density: string := peek(props.initial.density) + err = nameError(name) + <form onSubmit={e => { e.preventDefault() }}> + <label> + Display name{" "} + <input data-testid="name" value={name} onInput={e => { name := e.currentTarget.value }} /> + </label> + <span data-testid="name-count">{`${len(name)}/20`}</span> + {if err != "" { + <p data-testid="name-error">{err}</p> + }} + <div> + <button type="button" data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button type="button" data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button type="button" data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + <button type="button" data-testid="save" onClick={() => { + if (nameError(name) == "") { + save({ name: name, theme: theme, density: density }) + } + }}>Save</button> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + </form> +} + +export default component App() { + <main> + <h1>Settings</h1> + {errored fallback(e) { + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { action { refresh settings } }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{`Saved: ${settings.name} · ${settings.theme} · ${settings.density}${pending? settings ? " (saving…)" : ""}`}</p> + <Form initial={settings} /> + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-rx-2/runlog.jsonl new file mode 100644 index 0000000..c372265 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-2/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:04:46.727Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-big/settings-rx-2/src/main.rx:9: error OPTIMISTIC_NO_QUESTION [AC-3] saveError is declared with := — a guess needs a question to confirm it (source saveError = …, ST-5c)"} +{"at":"2026-10-27T19:04:49.240Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:04:56.534Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:05:15.325Z","cmd":"test","ok":false,"build":true,"passed":6,"total":7} +{"at":"2026-10-27T19:05:26.599Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/big/settings-rx-3/main.rx new file mode 100644 index 0000000..3dfd425 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-3/main.rx @@ -0,0 +1,70 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +source settings = await loadSettings() +source saveError: string = "" + +action save(next) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } +} + +fn nameErrorOf(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +component Form(props: { initial: { name: string; theme: string; density: string } }) { + source name: string := peek(props.initial.name) + source theme: string := peek(props.initial.theme) + source density: string := peek(props.initial.density) + nameError = nameErrorOf(name) + + <form onSubmit={(e) => { e.preventDefault() }}> + <label> + Display name{" "} + <input data-testid="name" value={name} onInput={(e) => { name := e.target.value }} /> + </label> + <span data-testid="name-count">{`${len(name)}/20`}</span> + {if nameError != "" { + <p data-testid="name-error">{nameError}</p> + }} + <div> + <button type="button" data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button type="button" data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button type="button" data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + <button type="button" data-testid="save" onClick={() => { + if (nameErrorOf(name) == "") { + save({ name: name, theme: theme, density: density }) + } + }}>Save</button> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + </form> +} + +export default component App() { + <main> + <h1>Settings</h1> + {errored fallback(e) { + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { refresh settings }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{`Saved: ${settings.name} · ${settings.theme} · ${settings.density}${pending? settings ? " (saving…)" : ""}`}</p> + <Form initial={settings} /> + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/settings-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/settings-rx-3/runlog.jsonl new file mode 100644 index 0000000..ccfd308 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/settings-rx-3/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:09:05.105Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-big/settings-rx-3/src/main.rx:8: error OPTIMISTIC_NO_QUESTION [AC-3] saveError is declared with := — a guess needs a question to confirm it (source saveError = …, ST-5c)"} +{"at":"2026-10-27T19:09:07.533Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:09:13.850Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:09:26.244Z","cmd":"test","ok":false,"build":true,"passed":1,"total":2} +{"at":"2026-10-27T19:09:36.670Z","cmd":"test","ok":true,"build":true,"passed":1,"total":1} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/big/tracker-react-1/App.tsx new file mode 100644 index 0000000..732d72f --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-1/App.tsx @@ -0,0 +1,184 @@ +import { useEffect, useRef, useState } from "react"; +import type { ReactNode } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; + +type ListResult = + | { filter: Filter; ok: true; issues: Issue[] } + | { filter: Filter; ok: false; message: string }; + +type DetailResult = + | { key: string; ok: true; issue: IssueDetail | null } + | { key: string; ok: false; message: string }; + +type Pending = { seq: number; user: string }; +type AssignError = { id: number; seq: number; message: string }; + +const messageOf = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [list, setList] = useState<ListResult | null>(null); + // a fetch generation per filter selection, so re-selecting a filter reloads it + const [listGen, setListGen] = useState(0); + const [listGenOf, setListGenOf] = useState(0); + + const [selectedId, setSelectedId] = useState<number | null>(null); + const [attempt, setAttempt] = useState(0); + const [detail, setDetail] = useState<DetailResult | null>(null); + + const [pending, setPending] = useState<Map<number, Pending>>(() => new Map()); + const [confirmed, setConfirmed] = useState<Map<number, string>>(() => new Map()); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seqRef = useRef(0); + + // ---- list ---- + useEffect(() => { + let live = true; + const gen = listGen; + fetchIssues(filter).then( + issues => { if (live) { setList({ filter, ok: true, issues }); setListGenOf(gen); } }, + (e: unknown) => { if (live) { setList({ filter, ok: false, message: messageOf(e) }); setListGenOf(gen); } }, + ); + return () => { live = false; }; + }, [filter, listGen]); + + const listReady = list !== null && list.filter === filter && listGenOf === listGen; + + const chooseFilter = (f: Filter) => { + if (f === filter) return; + setFilter(f); + setListGen(g => g + 1); + }; + + // ---- detail ---- + const detailKey = selectedId === null ? null : `${selectedId}:${attempt}`; + useEffect(() => { + if (selectedId === null) return; + let live = true; + const key = `${selectedId}:${attempt}`; + fetchIssue(selectedId).then( + issue => { if (live) setDetail({ key, ok: true, issue }); }, + (e: unknown) => { if (live) setDetail({ key, ok: false, message: messageOf(e) }); }, + ); + return () => { live = false; }; + }, [selectedId, attempt]); + + const select = (id: number) => { + if (id === selectedId) return; + setSelectedId(id); + setAttempt(a => a + 1); + setAssignError(null); + }; + + const retry = () => setAttempt(a => a + 1); + + // ---- assign ---- + const doAssign = () => { + if (selectedId === null) return; + const id = selectedId; + const who = user; + const seq = ++seqRef.current; + setAssignError(null); + setPending(prev => new Map(prev).set(id, { seq, user: who })); + assign(id, who).then( + res => { + setConfirmed(prev => new Map(prev).set(res.id, res.assignee)); + setPending(prev => { + if (prev.get(id)?.seq !== seq) return prev; + const next = new Map(prev); + next.delete(id); + return next; + }); + }, + (e: unknown) => { + setPending(prev => { + if (prev.get(id)?.seq !== seq) return prev; + const next = new Map(prev); + next.delete(id); + return next; + }); + // only the latest assignment's failure is reported + if (seqRef.current === seq) setAssignError({ id, seq, message: messageOf(e) }); + }, + ); + }; + + const assigneeOf = (issue: Issue): string | null => { + const p = pending.get(issue.id); + if (p) return p.user; + if (confirmed.has(issue.id)) return confirmed.get(issue.id) ?? null; + return issue.assignee; + }; + + // ---- render ---- + let listBody: ReactNode; + if (!listReady || list === null) { + listBody = <p>loading {filter} issues</p>; + } else if (!list.ok) { + listBody = <p>list error: {list.message}</p>; + } else { + listBody = ( + <ul> + {list.issues.map(issue => ( + <li + key={issue.id} + data-testid="issue" + onClick={() => select(issue.id)} + style={{ cursor: "pointer", fontWeight: issue.id === selectedId ? "bold" : undefined }} + > + {`#${issue.id} ${issue.title} (${assigneeOf(issue) ?? "unassigned"})`} + </li> + ))} + </ul> + ); + } + + let detailBody: ReactNode; + if (selectedId === null) { + detailBody = <p>select an issue</p>; + } else { + const d = detail !== null && detail.key === detailKey ? detail : null; + let content: ReactNode; + if (d === null) content = <p>loading #{selectedId}</p>; + else if (!d.ok) content = <p>detail error: {d.message}</p>; + else if (d.issue === null) content = <p>detail error: issue #{selectedId} not found</p>; + else + content = ( + <> + <h2>{`#${d.issue.id} ${d.issue.title}`}</h2> + <p>{d.issue.body}</p> + </> + ); + detailBody = ( + <> + {content} + <button data-testid="retry" onClick={retry}>retry</button> + <button data-testid="assign" onClick={doAssign}>{`assign to ${user}`}</button> + {assignError !== null && assignError.id === selectedId && ( + <p data-testid="assign-error">assign error: {assignError.message}</p> + )} + </> + ); + } + + return ( + <main> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button data-testid="filter-open" onClick={() => chooseFilter("open")}>open</button> + <button data-testid="filter-closed" onClick={() => chooseFilter("closed")}>closed</button> + </div> + <div style={{ display: "flex", gap: "2rem" }}> + <section data-testid="list">{listBody}</section> + <section data-testid="detail">{detailBody}</section> + </div> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-react-1/runlog.jsonl new file mode 100644 index 0000000..e6dcec8 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:03:00.142Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:09.847Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/big/tracker-react-2/App.tsx new file mode 100644 index 0000000..cf0781a --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-2/App.tsx @@ -0,0 +1,208 @@ +import { useRef, useState } from "react"; +import { + QueryClient, + QueryClientProvider, + notifyManager, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues, type Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type PendingAssign = { token: number; id: number; user: string }; +type AssignError = { id: number; message: string }; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + staleTime: Infinity, + gcTime: 0, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }, + }, +}); + +function messageOf(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function IssueList({ + filter, + pending, + onSelect, +}: { + filter: Filter; + pending: PendingAssign[]; + onSelect: (id: number) => void; +}) { + const q = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + let content; + if (q.isPending) { + content = <p>loading {filter} issues</p>; + } else if (q.isError) { + content = <p>list error: {messageOf(q.error)}</p>; + } else { + content = ( + <ul> + {q.data.map(issue => { + let assignee = issue.assignee; + for (const p of pending) if (p.id === issue.id) assignee = p.user; + return ( + <li + key={issue.id} + data-testid="issue" + style={{ cursor: "pointer" }} + onClick={() => onSelect(issue.id)} + > + {`#${issue.id} ${issue.title} (${assignee ?? "unassigned"})`} + </li> + ); + })} + </ul> + ); + } + return <section data-testid="list">{content}</section>; +} + +function Detail({ + id, + user, + assignError, + onAssign, +}: { + id: number; + user: string; + assignError: string | null; + onAssign: () => void; +}) { + const q = useQuery({ + queryKey: ["issue", id], + queryFn: () => fetchIssue(id), + }); + + let content; + if (q.isFetching || q.isPending) { + content = <p>loading #{id}</p>; + } else if (q.isError) { + content = <p>detail error: {messageOf(q.error)}</p>; + } else if (q.data === null) { + content = <p>detail error: issue #{id} not found</p>; + } else { + content = ( + <> + <h2>{`#${q.data.id} ${q.data.title}`}</h2> + <p>{q.data.body}</p> + </> + ); + } + + return ( + <> + {content} + <button data-testid="retry" onClick={() => void q.refetch()}> + retry + </button> + <button data-testid="assign" onClick={onAssign}> + {`assign to ${user}`} + </button> + {assignError !== null && <p>{assignError}</p>} + </> + ); +} + +function Tracker() { + const client = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + const [pending, setPending] = useState<PendingAssign[]>([]); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seq = useRef(0); + const applied = useRef(new Map<number, number>()); + + function select(id: number) { + if (id === selected) return; + setSelected(id); + setAssignError(null); + } + + async function startAssign() { + if (selected === null) return; + const id = selected; + const who = user; + const token = ++seq.current; + setAssignError(null); + setPending(p => [...p, { token, id, user: who }]); + const done = () => setPending(p => p.filter(x => x.token !== token)); + try { + const res = await assign(id, who); + notifyManager.batch(() => { + if ((applied.current.get(id) ?? -1) < token) { + applied.current.set(id, token); + client.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === res.id ? { ...i, assignee: res.assignee } : i)), + ); + } + notifyManager.schedule(done); + }); + } catch (e) { + done(); + setAssignError({ id, message: messageOf(e) }); + } + } + + return ( + <main style={{ display: "flex", gap: "2rem", fontFamily: "sans-serif" }}> + <div> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <IssueList filter={filter} pending={pending} onSelect={select} /> + </div> + <section data-testid="detail"> + {selected === null ? ( + <p>select an issue</p> + ) : ( + <Detail + key={selected} + id={selected} + user={user} + assignError={assignError !== null && assignError.id === selected ? assignError.message : null} + onAssign={() => void startAssign()} + /> + )} + </section> + </main> + ); +} + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-react-2/runlog.jsonl new file mode 100644 index 0000000..f9d25ed --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:43.770Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:04:52.432Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/big/tracker-react-3/App.tsx new file mode 100644 index 0000000..715d17e --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-3/App.tsx @@ -0,0 +1,187 @@ +import { useEffect, useState } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; +type ListQuery = { filter: Filter; n: number }; +type DetailQuery = { id: number; n: number }; + +type ListResult = + | { query: ListQuery; ok: true; issues: Issue[] } + | { query: ListQuery; ok: false; message: string }; +type DetailResult = + | { query: DetailQuery; ok: true; issue: IssueDetail | null } + | { query: DetailQuery; ok: false; message: string }; + +type PendingAssign = { token: number; id: number; assignee: string }; + +const errMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +let tokenSeq = 0; + +export default function App() { + const [user, setUser] = useState("guest"); + const [listQuery, setListQuery] = useState<ListQuery>({ filter: "open", n: 0 }); + const [listResult, setListResult] = useState<ListResult | null>(null); + const [detailQuery, setDetailQuery] = useState<DetailQuery | null>(null); + const [detailResult, setDetailResult] = useState<DetailResult | null>(null); + // assignments still waiting for the server, oldest first + const [pending, setPending] = useState<PendingAssign[]>([]); + // assignments the server accepted, applied over the loaded list + const [confirmed, setConfirmed] = useState<Record<number, string>>({}); + const [assignError, setAssignError] = useState<{ id: number; message: string } | null>(null); + + useEffect(() => { + let live = true; + fetchIssues(listQuery.filter).then( + issues => { if (live) setListResult({ query: listQuery, ok: true, issues }); }, + (e: unknown) => { if (live) setListResult({ query: listQuery, ok: false, message: errMessage(e) }); }, + ); + return () => { live = false; }; + }, [listQuery]); + + useEffect(() => { + if (!detailQuery) return; + let live = true; + fetchIssue(detailQuery.id).then( + issue => { if (live) setDetailResult({ query: detailQuery, ok: true, issue }); }, + (e: unknown) => { if (live) setDetailResult({ query: detailQuery, ok: false, message: errMessage(e) }); }, + ); + return () => { live = false; }; + }, [detailQuery]); + + function chooseFilter(filter: Filter) { + if (filter === listQuery.filter) return; + setListQuery({ filter, n: listQuery.n + 1 }); + // a fresh fetch reflects every assignment the server already accepted + setConfirmed({}); + } + + function select(id: number) { + if (detailQuery?.id === id) return; + setDetailQuery({ id, n: (detailQuery?.n ?? 0) + 1 }); + setAssignError(null); + } + + function retry() { + if (!detailQuery) return; + setDetailQuery({ id: detailQuery.id, n: detailQuery.n + 1 }); + } + + function doAssign() { + if (!detailQuery) return; + const id = detailQuery.id; + const who = user; + const token = ++tokenSeq; + setAssignError(null); + setPending(p => [...p, { token, id, assignee: who }]); + assign(id, who).then( + res => { + setPending(p => p.filter(a => a.token !== token)); + setConfirmed(c => ({ ...c, [res.id]: res.assignee })); + }, + (e: unknown) => { + setPending(p => p.filter(a => a.token !== token)); + setAssignError({ id, message: errMessage(e) }); + }, + ); + } + + function assigneeOf(issue: Issue): string | null { + for (let i = pending.length - 1; i >= 0; i--) { + if (pending[i].id === issue.id) return pending[i].assignee; + } + if (issue.id in confirmed) return confirmed[issue.id]; + return issue.assignee; + } + + const listCurrent = listResult && listResult.query === listQuery ? listResult : null; + let list; + if (!listCurrent) { + list = <p>loading {listQuery.filter} issues</p>; + } else if (!listCurrent.ok) { + list = <p>list error: {listCurrent.message}</p>; + } else { + list = ( + <ul> + {listCurrent.issues.map(issue => { + const who = assigneeOf(issue); + return ( + <li + key={issue.id} + data-testid="issue" + onClick={() => select(issue.id)} + style={{ + cursor: "pointer", + fontWeight: detailQuery?.id === issue.id ? "bold" : undefined, + }} + > + {`#${issue.id} ${issue.title} (${who ?? "unassigned"})`} + </li> + ); + })} + </ul> + ); + } + + let detail; + if (!detailQuery) { + detail = <p>select an issue</p>; + } else { + const current = detailResult && detailResult.query === detailQuery ? detailResult : null; + let body; + if (!current) { + body = <p>loading #{detailQuery.id}</p>; + } else if (!current.ok) { + body = <p>detail error: {current.message}</p>; + } else if (current.issue === null) { + body = <p>detail error: issue #{detailQuery.id} not found</p>; + } else { + body = ( + <> + <h2>{`#${current.issue.id} ${current.issue.title}`}</h2> + <p>{current.issue.body}</p> + </> + ); + } + detail = ( + <> + {body} + <button data-testid="retry" onClick={retry}>retry</button> + <button data-testid="assign" onClick={doAssign}>{`assign to ${user}`}</button> + {assignError && assignError.id === detailQuery.id && ( + <p data-testid="assign-error">{assignError.message}</p> + )} + </> + ); + } + + return ( + <div> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + data-testid="filter-open" + aria-pressed={listQuery.filter === "open"} + onClick={() => chooseFilter("open")} + > + open + </button> + <button + data-testid="filter-closed" + aria-pressed={listQuery.filter === "closed"} + onClick={() => chooseFilter("closed")} + > + closed + </button> + </div> + <div style={{ display: "flex", gap: "2rem" }}> + <section data-testid="list">{list}</section> + <section data-testid="detail">{detail}</section> + </div> + </div> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-react-3/runlog.jsonl new file mode 100644 index 0000000..c058424 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:09:27.764Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:09:32.980Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/main.rx new file mode 100644 index 0000000..77360d3 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/main.rx @@ -0,0 +1,67 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { map } from "./prelude" + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: { id: number; msg: string } | null = null + + action assignTo(id: number, who: string) { + optimistic issues := xs => map(xs, i => i.id == id ? { ...i, assignee: who } : i) + optimistic assignError := null + try { + let r = await assign(id, who) + issues := xs => map(xs, i => i.id == r.id ? { ...i, assignee: r.assignee } : i) + } catch (e) { + assignError := { id: id, msg: e.message } + } + } + + <div> + <label>user <input data-testid="user" value={user} onInput={(e) => { user := e.currentTarget.value }} /></label> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <ul data-testid="list"> + {loading on(filter) fallback { <li>loading {filter} issues</li> } { + {errored fallback(e) { <li>list error: {e.message}</li> } { + {for it in issues { + <li data-testid="issue" onClick={() => { selected := it.id; onFlush(() => { assignError := null }) }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + }} + }} + </ul> + <div data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + <Detail id={selected} /> + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>assign to {user}</button> + {if assignError != null && assignError.id == selected { + <p>{assignError.msg}</p> + }} + }} + </div> + </div> +} + +component Detail(props: { id: number }) { + detail = found(await fetchIssue(props.id), props.id) + <div> + {loading on(props.id) fallback { <p>loading #{props.id}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + </div> +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl new file mode 100644 index 0000000..b12701d --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:02:46.133Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:02:54.009Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/main.rx new file mode 100644 index 0000000..fa5aa78 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/main.rx @@ -0,0 +1,70 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { map } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: string = "" + detail = selected == null ? null : found(await fetchIssue(selected), selected) + + action assignIssue(id: number, who: string) { + optimistic issues := xs => map(xs, i => i.id == id ? { ...i, assignee: who } : i) + optimistic assignError := "" + try { + let answer = await assign(id, who) + issues := xs => map(xs, i => i.id == answer.id ? { ...i, assignee: answer.assignee } : i) + } catch (e) { + assignError := e.message + } + } + + action retry() { + refresh detail + } + + <main> + <label>user <input data-testid="user" value={user} onInput={(ev) => { user := ev.target.value }} /></label> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <section data-testid="list"> + {loading on(filter) fallback { <p>loading {latest filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for issue in issues { + <li data-testid="issue" onClick={() => { + selected := issue.id + onFlush(() => { assignError := "" }) + }}>#{issue.id} {issue.title} ({issue.assignee ?? "unassigned"})</li> + }} + </ul> + }} + }} + </section> + <section data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + {loading on(selected) fallback { <p>loading #{latest selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + <button data-testid="retry" onClick={() => { retry() }}>retry</button> + <button data-testid="assign" onClick={() => { assignIssue(selected, user) }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + }} + </section> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl new file mode 100644 index 0000000..5b535a1 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:14.957Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:04:22.756Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/main.rx new file mode 100644 index 0000000..841e1f9 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/main.rx @@ -0,0 +1,61 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { map } from "./prelude" + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id) => it == null ? throw NotFound(id) : it + +fn label(i) => `#${i.id} ${i.title} (${i.assignee ?? "unassigned"})` + +export default component App() { + source user: string := "guest" + source filt: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filt) + source assignError: string = "" + + action assignTo(id: number, who: string) { + optimistic issues := xs => map(xs, i => i.id == id ? { ...i, assignee: who } : i) + optimistic assignError := "" + try { + let r = await assign(id, who) + issues := xs => map(xs, i => i.id == r.id ? { ...i, assignee: r.assignee } : i) + } catch (e) { + assignError := e.message + } + } + + <div> + <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /> + <button data-testid="filter-open" onClick={() => { filt := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filt := "closed" }}>closed</button> + <ul data-testid="list"> + {loading on(filt) fallback { <li>{`loading ${filt} issues`}</li> } { + {errored fallback(e) { <li>{`list error: ${e.message}`}</li> } { + {for i in issues { + <li data-testid="issue" onClick={() => { selected := i.id; assignError := "" }}>{label(i)}</li> + }} + }} + }} + </ul> + <div data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + detail = found(await fetchIssue(selected), selected) + {loading on(selected) fallback { <p>{`loading #${selected}`}</p> } { + {errored fallback(e) { <p>{`detail error: ${e.message}`}</p> } { + <h2>{`#${detail.id} ${detail.title}`}</h2> + <p>{detail.body}</p> + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>{`assign to ${user}`}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + }} + </div> + </div> +} diff --git a/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl new file mode 100644 index 0000000..7cb33ba --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl @@ -0,0 +1,3 @@ +{"at":"2026-10-27T19:09:17.838Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:09:25.675Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:09:48.424Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/small/search-react-1/App.tsx new file mode 100644 index 0000000..4155ede --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-1/App.tsx @@ -0,0 +1,102 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, keepPreviousData, useQuery } from "@tanstack/react-query"; +import { search, fetchItem, type Hit, type Item } from "./api.js"; + +const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +type Found = { q: string; hits: Hit[] }; + +function Results({ q, selected, onPick }: { q: string; selected: number | null; onPick: (id: number) => void }) { + const found = useQuery<Found, Error>({ + queryKey: ["search", q], + queryFn: async () => ({ q, hits: await search(q) }), + enabled: q.length >= 2, + retry: false, + placeholderData: keepPreviousData, + }); + + if (q.length === 0) return null; + if (q.length === 1) return <p>{`too short: ${q}`}</p>; + + const loading = found.isPending || found.isFetching; + if (!loading && found.isError) return <p>{`search failed: ${errorMessage(found.error)}`}</p>; + + return ( + <> + {loading && <p>searching…</p>} + {found.data && ( + <> + <p>{`${found.data.hits.length} results for ${found.data.q}`}</p> + <ul> + {found.data.hits.map(hit => ( + <li key={hit.id} data-testid="hit" onClick={() => onPick(hit.id)}> + {hit.id === selected ? "> " : ""} + {hit.name} + </li> + ))} + </ul> + </> + )} + </> + ); +} + +function Detail({ id }: { id: number }) { + const item = useQuery<Item | null, Error>({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + retry: false, + staleTime: Infinity, + }); + + if (item.isFetching) return <p>loading…</p>; + if (item.isError) { + return ( + <> + <p>{`failed: ${errorMessage(item.error)}`}</p> + <button type="button" data-testid="retry" onClick={() => void item.refetch()}> + retry + </button> + </> + ); + } + if (item.data === null || item.data === undefined) return <p>{`item ${id} not found`}</p>; + return ( + <> + <h2>{item.data.name}</h2> + <p>{item.data.blurb}</p> + </> + ); +} + +function Page() { + const [query, setQuery] = useState(""); + const [selected, setSelected] = useState<number | null>(null); + + return ( + <main> + <input + type="text" + data-testid="query" + aria-label="query" + value={query} + onChange={e => setQuery(e.target.value)} + /> + <div data-testid="results"> + <Results q={query} selected={selected} onPick={setSelected} /> + </div> + <section data-testid="detail"> + {selected === null ? <p>pick a result</p> : <Detail key={selected} id={selected} />} + </section> + </main> + ); +} + +export default function App() { + const [client] = useState(() => new QueryClient()); + return ( + <QueryClientProvider client={client}> + <Page /> + </QueryClientProvider> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-react-1/runlog.jsonl new file mode 100644 index 0000000..d5407a1 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:03:06.788Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:15.379Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:20.937Z","cmd":"test","ok":false,"build":true,"passed":1,"total":4} +{"at":"2026-10-27T19:03:31.515Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:37.372Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/small/search-react-2/App.tsx new file mode 100644 index 0000000..eed1666 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-2/App.tsx @@ -0,0 +1,134 @@ +import { useState, type ReactNode } from "react"; +import { + QueryClient, + QueryClientProvider, + keepPreviousData, + useQuery, +} from "@tanstack/react-query"; +import { fetchItem, search, type Hit } from "./api.js"; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, refetchOnWindowFocus: false }, + }, +}); + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +type Found = { q: string; hits: Hit[] }; + +function Results({ + q, + selectedId, + onSelect, +}: { + q: string; + selectedId: number | null; + onSelect: (id: number) => void; +}) { + const enabled = q.length >= 2; + const result = useQuery({ + queryKey: ["search", q], + queryFn: async (): Promise<Found> => ({ q, hits: await search(q) }), + enabled, + placeholderData: keepPreviousData, + }); + + let body: ReactNode = null; + if (q.length === 1) { + body = <p>too short: {q}</p>; + } else if (enabled) { + if (result.isError) { + body = <p>search failed: {errorMessage(result.error)}</p>; + } else { + const shown = result.data; + body = ( + <> + {result.isFetching && <p>searching…</p>} + {shown && ( + <> + <p> + {shown.hits.length} results for {shown.q} + </p> + <ul> + {shown.hits.map((hit) => ( + <li key={hit.id} data-testid="hit" onClick={() => onSelect(hit.id)}> + <button type="button"> + {hit.id === selectedId ? "> " : ""} + {hit.name} + </button> + </li> + ))} + </ul> + </> + )} + </> + ); + } + } + + return <div data-testid="results">{body}</div>; +} + +function ItemView({ id }: { id: number }) { + const item = useQuery({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + }); + + if (item.isPending) return <p>loading…</p>; + if (item.isError) { + return ( + <> + <p>failed: {errorMessage(item.error)}</p> + <button type="button" data-testid="retry" onClick={() => void item.refetch()}> + retry + </button> + </> + ); + } + if (item.data === null) return <p>item {id} not found</p>; + return ( + <> + <h2>{item.data.name}</h2> + <p>{item.data.blurb}</p> + </> + ); +} + +function Detail({ id }: { id: number | null }) { + return ( + <div data-testid="detail"> + {id === null ? <p>pick a result</p> : <ItemView id={id} />} + </div> + ); +} + +function Page() { + const [q, setQ] = useState(""); + const [selectedId, setSelectedId] = useState<number | null>(null); + + return ( + <main> + <input + data-testid="query" + type="text" + aria-label="search" + value={q} + onChange={(e) => setQ(e.target.value)} + /> + <Results q={q} selectedId={selectedId} onSelect={setSelectedId} /> + <Detail id={selectedId} /> + </main> + ); +} + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Page /> + </QueryClientProvider> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-react-2/runlog.jsonl new file mode 100644 index 0000000..40660fe --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:05:01.068Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:05:08.231Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/small/search-react-3/App.tsx new file mode 100644 index 0000000..3a7c738 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-3/App.tsx @@ -0,0 +1,114 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, keepPreviousData, useQuery } from "@tanstack/react-query"; +import { search, fetchItem } from "./api.js"; + +const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +export default function App() { + // Each search and item request is looked up fresh (gcTime 0) and never retried, + // so the page shows exactly what the backend answers, when it answers. + const [client] = useState( + () => + new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false }, + }, + }), + ); + return ( + <QueryClientProvider client={client}> + <Catalogue /> + </QueryClientProvider> + ); +} + +function Catalogue() { + const [query, setQuery] = useState(""); + const [selectedId, setSelectedId] = useState<number | null>(null); + + const hits = useQuery({ + queryKey: ["search", query], + queryFn: () => search(query), + enabled: query.length >= 2, + // Keep the previous hits on screen while the new search is in the air. + placeholderData: keepPreviousData, + }); + + const q = query; + const searching = q.length >= 2 && !hits.isError && (hits.isPending || hits.isPlaceholderData); + const showHits = q.length >= 2 && !hits.isError && hits.data !== undefined; + + return ( + <main> + <input + data-testid="query" + type="text" + aria-label="query" + value={query} + onChange={e => setQuery(e.target.value)} + /> + + <section data-testid="results"> + {q.length === 1 && <p>{`too short: ${q}`}</p>} + {q.length >= 2 && hits.isError && <p>{`search failed: ${messageOf(hits.error)}`}</p>} + {searching && <p>searching…</p>} + {showHits && !hits.isPlaceholderData && ( + <p>{`${hits.data?.length ?? 0} results for ${q}`}</p> + )} + {showHits && ( + <ul> + {hits.data?.map((hit, i) => { + const selected = hit.id === selectedId; + return ( + <li + key={`${i}-${hit.id}`} + data-testid="hit" + role="button" + tabIndex={0} + onClick={() => setSelectedId(hit.id)} + onKeyDown={e => { + if (e.key === "Enter" || e.key === " ") setSelectedId(hit.id); + }} + > + {selected ? "> " : ""} + {hit.name} + </li> + ); + })} + </ul> + )} + </section> + + <section data-testid="detail"> + {selectedId === null ? "pick a result" : <Detail key={selectedId} id={selectedId} />} + </section> + </main> + ); +} + +function Detail({ id }: { id: number }) { + const item = useQuery({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + }); + + if (item.isFetching) return <p>loading…</p>; + if (item.isError) { + return ( + <> + <p>{`failed: ${messageOf(item.error)}`}</p> + <button type="button" data-testid="retry" onClick={() => void item.refetch()}> + retry + </button> + </> + ); + } + if (item.data === null) return <p>{`item ${id} not found`}</p>; + if (item.data === undefined) return <p>loading…</p>; + return ( + <> + <h2>{item.data.name}</h2> + <p>{item.data.blurb}</p> + </> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-react-3/runlog.jsonl new file mode 100644 index 0000000..1300944 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:09:30.507Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:09:38.598Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/small/search-rx-1/main.rx new file mode 100644 index 0000000..c1c72bf --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-1/main.rx @@ -0,0 +1,65 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +source query := "" +source selected: number | null := null + +// The hits for the query, with the query they answer. Empty and one-character +// queries ask nothing. While a new answer is on its way the last one stays put. +answer = len(query) < 2 ? null : { q: query, hits: await search(query) } + +// The chosen hit's full item, or null when there is no such item. +item = selected == null ? null : await fetchItem(selected) + +export default component App() { + <main> + <input data-testid="query" type="text" onInput={(e) => { query := e.target.value }} /> + + <section data-testid="results"> + {if len(query) == 1 { + <p>too short: {query}</p> + }} + {if len(query) > 1 { + {loading fallback { <p>searching…</p> } { + {if pending? answer { + <p>searching…</p> + }} + {errored fallback(e) { + <p>search failed: {e.message}</p> + } { + {if answer { + <div> + <p>{len(answer.hits)} results for {answer.q}</p> + <ul> + {for h in answer.hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{selected == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + </div> + }} + }} + }} + }} + </section> + + <section data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + {loading on(selected) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {if item == null { + <p>item {selected} not found</p> + } else { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + }} + </section> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-rx-1/runlog.jsonl new file mode 100644 index 0000000..6cb59a3 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:03:24.928Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:03:33.118Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:03:57.916Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:04:12.021Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/small/search-rx-2/main.rx new file mode 100644 index 0000000..6e77aae --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-2/main.rx @@ -0,0 +1,66 @@ +import pure { search, fetchItem } from "./api" + +// One item, asked again whenever `id` changes. `loading on(props.id)` shows +// "loading…" on each new selection; `refresh item` retries after a failure. +component Detail(props: { id: number }) { + item = await fetchItem(props.id) + + <div> + {loading on(props.id) fallback { <p>loading…</p> } { + errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + if item == null { + <p>item {props.id} not found</p> + } else { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + } + } + }} + </div> +} + +export default component App() { + source query := "" + source selected: number | null := null + + // Empty and one-character queries ask nothing; the held value is the last + // answer, and `q` says which query that answer is for. + answered = query.length < 2 ? null : { q: query, hits: await search(query) } + busy = pending? answered + + <main> + <input data-testid="query" value={query} onInput={(e) => { query := e.target.value }} /> + <section data-testid="results"> + {if query.length == 1 { + <p>too short: {query}</p> + }} + {if busy { + <p>searching…</p> + }} + {if query.length >= 2 { + errored fallback(e) { + <p>search failed: {e.message}</p> + } { + if answered { + <p>{answered.hits.length} results for {answered.q}</p> + <ul> + {for hit in answered.hits { + <li data-testid="hit" onClick={() => { selected := hit.id }}>{selected == hit.id ? "> " : ""}{hit.name}</li> + }} + </ul> + } + } + }} + </section> + <aside data-testid="detail"> + {if selected != null { + <Detail id={selected} /> + } else { + <p>pick a result</p> + }} + </aside> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-rx-2/runlog.jsonl new file mode 100644 index 0000000..287e145 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-2/runlog.jsonl @@ -0,0 +1,6 @@ +{"at":"2026-10-27T19:05:33.484Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:05:39.583Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:05:44.684Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:05:48.597Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:05:55.603Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:05:59.931Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/small/search-rx-3/main.rx new file mode 100644 index 0000000..37001c0 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-3/main.rx @@ -0,0 +1,61 @@ +import pure { search, fetchItem } from "./api" + +type Hit = { id: number; name: string } +type Item = { id: number; name: string; blurb: string } + +error NotFound(id: number) => `item ${id} not found` + +fn found(it: Item | null, id: number): Item => it == null ? throw NotFound(id) : it + +component Detail(props: { id: number }) { + item = found(await fetchItem(props.id), props.id) + <div> + {loading on(props.id) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {errored fallback(e: NotFound) { <p>{e.message}</p> } { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + </div> +} + +export default component App() { + source query := "" + source selected: number | null := null + + hits: Hit[] = query.length < 2 ? [] : await search(query) + + <main> + <input data-testid="query" value={query} onInput={(e) => { query := e.target.value }} /> + <div data-testid="results"> + {if query.length == 1 { + <p>too short: {query}</p> + }} + {if query.length > 1 { + {loading fallback { <p>searching…</p> } { + {errored fallback(e) { <p>search failed: {e.message}</p> } { + {if pending? hits { <p>searching…</p> }} + <p>{hits.length} results for {query}</p> + <ul> + {for h in hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{selected == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + </div> + <section data-testid="detail"> + {if selected != null { + <Detail id={selected} /> + } else { + <p>pick a result</p> + }} + </section> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/search-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/search-rx-3/runlog.jsonl new file mode 100644 index 0000000..dd23ae3 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/search-rx-3/runlog.jsonl @@ -0,0 +1,3 @@ +{"at":"2026-10-27T19:09:46.074Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:09:53.716Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:10:23.723Z","cmd":"test","ok":true,"build":true,"passed":8,"total":8} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/small/settings-react-1/App.tsx new file mode 100644 index 0000000..4f72016 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-1/App.tsx @@ -0,0 +1,154 @@ +import { useState } from "react"; +import { + QueryClient, + QueryClientProvider, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings } from "./api.js"; + +const SETTINGS_KEY = ["settings"] as const; +const MAX_NAME = 20; + +function nameError(name: string): string | null { + if (name.trim() === "") return "Display name is required"; + if (name.length > MAX_NAME) { + return `Display name must be at most ${MAX_NAME} characters, not ${name.length}`; + } + return null; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function SettingsForm({ saved }: { saved: Settings }) { + const queryClient = useQueryClient(); + // The draft is seeded once from the first loaded settings and then belongs to the user. + const [draft, setDraft] = useState<Settings>(saved); + + const save = useMutation({ + mutationFn: (settings: Settings) => saveSettings(settings), + onSuccess: (result) => { + queryClient.setQueryData(SETTINGS_KEY, result); + }, + }); + + const error = nameError(draft.name); + const pending = save.isPending ? save.variables : undefined; + const shown = pending ?? saved; + const savedText = + `Saved: ${shown.name} · ${shown.theme} · ${shown.density}` + + (pending ? " (saving…)" : ""); + + const onSave = () => { + if (nameError(draft.name) !== null) return; + save.mutate(draft); + }; + + return ( + <div> + <p data-testid="saved">{savedText}</p> + + <form + onSubmit={(event) => { + event.preventDefault(); + onSave(); + }} + > + <label> + Display name + <input + data-testid="name" + value={draft.name} + onChange={(event) => { + const name = event.target.value; + setDraft((d) => ({ ...d, name })); + }} + /> + </label> + <span data-testid="name-count">{`${draft.name.length}/${MAX_NAME}`}</span> + {error !== null && <p data-testid="name-error">{error}</p>} + + <fieldset> + <legend>Theme</legend> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => setDraft((d) => ({ ...d, theme: "light" }))} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => setDraft((d) => ({ ...d, theme: "dark" }))} + > + Dark + </button> + </fieldset> + + <button + type="button" + data-testid="density-toggle" + onClick={() => + setDraft((d) => ({ + ...d, + density: d.density === "compact" ? "comfortable" : "compact", + })) + } + > + {`Density: ${draft.density}`} + </button> + + <p data-testid="preview">{`Preview: ${draft.theme} / ${draft.density}`}</p> + + <button type="button" data-testid="save" onClick={onSave}> + Save + </button> + </form> + + {save.isError && ( + <p data-testid="save-error">{`Save failed: ${errorMessage(save.error)}`}</p> + )} + </div> + ); +} + +function SettingsPage() { + const query = useQuery({ + queryKey: SETTINGS_KEY, + queryFn: () => loadSettings(), + retry: false, + refetchOnWindowFocus: false, + }); + + if (query.data === undefined) { + if (query.isError) { + return ( + <div> + <p data-testid="load">{`Could not load settings: ${errorMessage(query.error)}`}</p> + <button type="button" data-testid="retry-load" onClick={() => void query.refetch()}> + Retry + </button> + </div> + ); + } + return <p data-testid="load">Loading settings…</p>; + } + + return <SettingsForm saved={query.data} />; +} + +export default function App() { + const [client] = useState(() => new QueryClient()); + return ( + <QueryClientProvider client={client}> + <SettingsPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-react-1/runlog.jsonl new file mode 100644 index 0000000..323e6a6 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:03:09.410Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:19.303Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:03:33.418Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:03:42.097Z","cmd":"test","ok":true,"build":true,"passed":1,"total":1} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/small/settings-react-2/App.tsx new file mode 100644 index 0000000..5e88ebd --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-2/App.tsx @@ -0,0 +1,172 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Density, Settings, Theme } from "./api.js"; + +type LoadState = + | { status: "loading" } + | { status: "error"; message: string } + | { status: "ready" }; + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > 20) { + return `Display name must be at most 20 characters, not ${name.length}`; + } + return null; +} + +function describe(s: Settings): string { + return `${s.name} · ${s.theme} · ${s.density}`; +} + +export default function App() { + const [load, setLoad] = useState<LoadState>({ status: "loading" }); + const [saved, setSaved] = useState<Settings | null>(null); + const [draft, setDraft] = useState<Settings | null>(null); + const [saving, setSaving] = useState<Settings | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + + const started = useRef(false); + const loadSeq = useRef(0); + const saveSeq = useRef(0); + + const fetchSettings = useCallback(() => { + const id = ++loadSeq.current; + loadSettings().then( + (s) => { + if (id !== loadSeq.current) return; + setSaved(s); + setDraft({ name: s.name, theme: s.theme, density: s.density }); + setLoad({ status: "ready" }); + }, + (e: unknown) => { + if (id !== loadSeq.current) return; + setLoad({ status: "error", message: errorMessage(e) }); + }, + ); + }, []); + + useEffect(() => { + if (started.current) return; + started.current = true; + fetchSettings(); + }, [fetchSettings]); + + function retryLoad() { + setLoad({ status: "loading" }); + fetchSettings(); + } + + function save(d: Settings) { + if (nameError(d.name) !== null) return; + const toSave: Settings = { name: d.name, theme: d.theme, density: d.density }; + const id = ++saveSeq.current; + setSaving(toSave); + setSaveError(null); + saveSettings(toSave).then( + (result) => { + if (id !== saveSeq.current) return; + setSaved(result); + setSaving(null); + }, + (e: unknown) => { + if (id !== saveSeq.current) return; + setSaving(null); + setSaveError(`Save failed: ${errorMessage(e)}`); + }, + ); + } + + function setName(name: string) { + setDraft((d) => (d ? { ...d, name } : d)); + } + + function setTheme(theme: Theme) { + setDraft((d) => (d ? { ...d, theme } : d)); + } + + function toggleDensity() { + setDraft((d) => { + if (!d) return d; + const density: Density = d.density === "compact" ? "comfortable" : "compact"; + return { ...d, density }; + }); + } + + const error = draft ? nameError(draft.name) : null; + const savedLine = saved + ? `Saved: ${describe(saving ?? saved)}${saving ? " (saving…)" : ""}` + : null; + + return ( + <main style={{ fontFamily: "system-ui, sans-serif", maxWidth: 480, margin: "2rem auto", padding: "0 16px" }}> + <h1>Settings</h1> + + {load.status === "loading" && <p data-testid="load">Loading settings…</p>} + + {load.status === "error" && ( + <div> + <p data-testid="load">{`Could not load settings: ${load.message}`}</p> + <button type="button" data-testid="retry-load" onClick={retryLoad}> + Retry + </button> + </div> + )} + + {savedLine !== null && <p data-testid="saved">{savedLine}</p>} + + {draft && ( + <div> + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + aria-invalid={error !== null} + onChange={(e) => setName(e.target.value)} + /> + </label> + <p data-testid="name-count">{`${draft.name.length}/20`}</p> + {error !== null && <p data-testid="name-error">{error}</p>} + + <div> + <span>Theme </span> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => setTheme("light")} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => setTheme("dark")} + > + Dark + </button> + </div> + + <div> + <button type="button" data-testid="density-toggle" onClick={toggleDensity}> + {`Density: ${draft.density}`} + </button> + </div> + + <p data-testid="preview">{`Preview: ${draft.theme} / ${draft.density}`}</p> + + <button type="button" data-testid="save" onClick={() => save(draft)}> + Save + </button> + {saveError !== null && <p data-testid="save-error">{saveError}</p>} + </div> + )} + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-react-2/runlog.jsonl new file mode 100644 index 0000000..ef2bbbf --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:04:40.041Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:04:48.144Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/small/settings-react-3/App.tsx new file mode 100644 index 0000000..f69f7b6 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-3/App.tsx @@ -0,0 +1,157 @@ +import { useCallback, useEffect, useRef, useState, type FormEvent } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Density, Settings, Theme } from "./api.js"; + +type Status = { kind: "loading" } | { kind: "error"; message: string } | { kind: "ready" }; + +const MAX_NAME = 20; + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX_NAME) return `Display name must be at most ${MAX_NAME} characters, not ${name.length}`; + return null; +} + +function describe(s: Settings): string { + return `${s.name} · ${s.theme} · ${s.density}`; +} + +export default function App() { + const [status, setStatus] = useState<Status>({ kind: "loading" }); + const [saved, setSaved] = useState<Settings | null>(null); + const [draft, setDraft] = useState<Settings | null>(null); + const [pending, setPending] = useState<Settings | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + const loadSeq = useRef(0); + const saveSeq = useRef(0); + const requested = useRef(false); + + const fetchSettings = useCallback(() => { + const id = ++loadSeq.current; + loadSettings().then( + (s) => { + if (id !== loadSeq.current) return; + setSaved(s); + setDraft(s); + setStatus({ kind: "ready" }); + }, + (e: unknown) => { + if (id !== loadSeq.current) return; + setStatus({ kind: "error", message: errorMessage(e) }); + }, + ); + }, []); + + useEffect(() => { + if (requested.current) return; + requested.current = true; + fetchSettings(); + }, [fetchSettings]); + + function retryLoad() { + setStatus({ kind: "loading" }); + fetchSettings(); + } + + function update(patch: Partial<Settings>) { + setDraft((d) => (d ? { ...d, ...patch } : d)); + } + + function onSave(e: FormEvent) { + e.preventDefault(); + if (!draft || nameError(draft.name) !== null) return; + const values: Settings = { name: draft.name, theme: draft.theme, density: draft.density }; + const id = ++saveSeq.current; + setSaveError(null); + setPending(values); + saveSettings({ ...values }).then( + (s) => { + if (id !== saveSeq.current) return; + setSaved(s); + setPending(null); + }, + (err: unknown) => { + if (id !== saveSeq.current) return; + setPending(null); + setSaveError(`Save failed: ${errorMessage(err)}`); + }, + ); + } + + const invalid = draft ? nameError(draft.name) : null; + + return ( + <main> + <h1>Settings</h1> + + {status.kind === "loading" && <p data-testid="load">Loading settings…</p>} + {status.kind === "error" && ( + <> + <p data-testid="load">{`Could not load settings: ${status.message}`}</p> + <button type="button" data-testid="retry-load" onClick={retryLoad}> + Retry + </button> + </> + )} + + {saved && ( + <p data-testid="saved"> + {pending ? `Saved: ${describe(pending)} (saving…)` : `Saved: ${describe(saved)}`} + </p> + )} + + {draft && ( + <form onSubmit={onSave} noValidate> + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={(e) => update({ name: e.target.value })} + /> + </label> + <span data-testid="name-count">{`${draft.name.length}/${MAX_NAME}`}</span> + {invalid && <p data-testid="name-error">{invalid}</p>} + + <div> + <span>Theme </span> + {(["light", "dark"] as Theme[]).map((t) => ( + <button + key={t} + type="button" + data-testid={`theme-${t}`} + aria-pressed={draft.theme === t} + onClick={() => update({ theme: t })} + > + {t} + </button> + ))} + </div> + + <div> + <button + type="button" + data-testid="density-toggle" + onClick={() => + update({ density: (draft.density === "compact" ? "comfortable" : "compact") as Density }) + } + > + Toggle density + </button> + </div> + + <p data-testid="preview">{`Preview: ${draft.theme} / ${draft.density}`}</p> + + <button type="submit" data-testid="save"> + Save + </button> + {saveError && <p data-testid="save-error">{saveError}</p>} + </form> + )} + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-react-3/runlog.jsonl new file mode 100644 index 0000000..ccdbfa2 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:09:44.278Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:09:51.665Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/small/settings-rx-1/main.rx new file mode 100644 index 0000000..a43b5e5 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-1/main.rx @@ -0,0 +1,78 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" + +type Settings = { name: string; theme: string; density: string } + +// The loaded (and then saved) settings: a source with a standing question. The save action may guess at it. +source saved: Settings = await loadSettings() +// The last save's failure, if any. Cleared as soon as a new save starts. +source saveError: string = "" + +action save(next: Settings) { + optimistic saved := next + optimistic saveError := "" + affects saved + try { + let answer = await saveSettings(next) + saved := answer + } catch (e) { + saveError := e.message + } +} + +// The draft is seeded once from the loaded settings and belongs to the user from then on. +component Form(props: { initial: Settings }) { + source draftName := peek(props.initial.name) + source draftTheme := peek(props.initial.theme) + source draftDensity := peek(props.initial.density) + nameError = draftName.length == 0 ? "Display name is required" : draftName.length > 20 ? `Display name must be at most 20 characters, not ${draftName.length}` : "" + + <div> + <label> + Display name + <input data-testid="name" value={draftName} onInput={(e) => { draftName := e.target.value }} /> + </label> + <span data-testid="name-count">{draftName.length}/20</span> + {if nameError != "" { + <p data-testid="name-error">{nameError}</p> + }} + <div> + <button data-testid="theme-light" onClick={() => { draftTheme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { draftTheme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { draftDensity := draftDensity == "compact" ? "comfortable" : "compact" }}>Density</button> + </div> + <p data-testid="preview">Preview: {draftTheme} / {draftDensity}</p> + <button data-testid="save" onClick={() => { if (nameError == "") { save({ name: draftName, theme: draftTheme, density: draftDensity }) } }}>Save</button> + </div> +} + +export default component App() { + source attempt := 0 + isSaving = pending? saved + + action retry() { + attempt := attempt + 1 + refresh saved + } + + <main> + {loading on(attempt) fallback { + <p data-testid="load">Loading settings…</p> + } { + {errored fallback(e) { + <div> + <p data-testid="load">Could not load settings: {e.message}</p> + <button data-testid="retry-load" onClick={() => { retry() }}>Retry</button> + </div> + } { + <div> + <p data-testid="saved">Saved: {saved.name} · {saved.theme} · {saved.density}{isSaving ? " (saving…)" : ""}</p> + {if saveError != "" { + <p data-testid="save-error">Save failed: {saveError}</p> + }} + <Form initial={saved} /> + </div> + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-rx-1/runlog.jsonl new file mode 100644 index 0000000..ac1db05 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:03:43.677Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-1/src/main.rx: line 60: expression near '<p data-testid=\"load'"} +{"at":"2026-10-27T19:03:50.678Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:03:59.545Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:04:13.533Z","cmd":"test","ok":false,"build":true,"passed":2,"total":3} +{"at":"2026-10-27T19:04:23.890Z","cmd":"test","ok":true,"build":true,"passed":3,"total":3} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/small/settings-rx-2/main.rx new file mode 100644 index 0000000..d9ab1c9 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-2/main.rx @@ -0,0 +1,71 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" + +type Settings = { name: string; theme: string; density: string } + +export default component App() { + source saved = await loadSettings() + source saveError: string = "" + + action save(next: Settings) { + optimistic saved := next + optimistic saveError := "" + affects saved + try { + let answer = await saveSettings(next) + saved := answer + } catch (e) { + saveError := e.message + } + } + + source attempt := 0 + + action retry() { + attempt := attempt + 1 + refresh saved + } + + <main> + {loading on(attempt) fallback { <p data-testid="load">Loading settings…</p> } { + {errored fallback(e) { + <div> + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { retry() }}>Retry</button> + </div> + } { + source draftName := peek(saved.name) + source draftTheme := peek(saved.theme) + source draftDensity := peek(saved.density) + nameLen = draftName.length + nameError = nameLen == 0 ? "Display name is required" : nameLen > 20 ? `Display name must be at most 20 characters, not ${nameLen}` : "" + + <p data-testid="saved">{`Saved: ${saved.name} · ${saved.theme} · ${saved.density}`}{pending? saved ? " (saving…)" : ""}</p> + <div class="draft"> + <label> + Display name + <input data-testid="name" value={draftName} onInput={(e) => { draftName := e.target.value }} /> + </label> + <span data-testid="name-count">{`${nameLen}/20`}</span> + {if nameError != "" { + <p data-testid="name-error">{nameError}</p> + }} + <div> + <button data-testid="theme-light" onClick={() => { draftTheme := "light" }}>Light theme</button> + <button data-testid="theme-dark" onClick={() => { draftTheme := "dark" }}>Dark theme</button> + </div> + <button data-testid="density-toggle" onClick={() => { draftDensity := draftDensity == "compact" ? "comfortable" : "compact" }}>Switch density</button> + <p data-testid="preview">{`Preview: ${draftTheme} / ${draftDensity}`}</p> + <button data-testid="save" onClick={() => { + if (nameError == "") { + save({ name: draftName, theme: draftTheme, density: draftDensity }) + } + }}>Save</button> + </div> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-rx-2/runlog.jsonl new file mode 100644 index 0000000..e6679af --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-2/runlog.jsonl @@ -0,0 +1,10 @@ +{"at":"2026-10-27T19:10:05.694Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-2/src/main.rx: line 8: expected '{' near 'source draftTheme :='"} +{"at":"2026-10-27T19:10:15.869Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-2/src/main.rx: line 7: declaration or markup near '{\n source draftName'"} +{"at":"2026-10-27T19:10:28.564Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-2/src/main.rx: line 7: declaration or markup near '{\n source draftName'"} +{"at":"2026-10-27T19:10:55.694Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:11:04.979Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:11:18.547Z","cmd":"test","ok":false,"build":true,"passed":1,"total":4} +{"at":"2026-10-27T19:13:43.731Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-2/src/main.rx: line 70: declaration or markup near '}}\n </main>\n}\n'"} +{"at":"2026-10-27T19:13:50.217Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:14:01.656Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:14:09.613Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/main.rx new file mode 100644 index 0000000..d22af8e --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/main.rx @@ -0,0 +1,47 @@ +import { saved, saveError, save, Settings } from "./store.rx" + +component Editor(props: { initial: Settings }) { + source draftName := peek(props.initial.name) + source draftTheme := peek(props.initial.theme) + source draftDensity := peek(props.initial.density) + + nameLen = draftName.length + nameError = nameLen == 0 ? "Display name is required" : nameLen > 20 ? `Display name must be at most 20 characters, not ${nameLen}` : "" + invalid = nameError != "" + + <div> + <label> + Name + <input data-testid="name" value={draftName} onInput={(e) => { draftName := e.target.value }} /> + </label> + <p data-testid="name-count">{`${nameLen}/20`}</p> + {if invalid { <p data-testid="name-error">{nameError}</p> }} + <div> + <button data-testid="theme-light" onClick={() => { draftTheme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { draftTheme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { draftDensity := draftDensity == "comfortable" ? "compact" : "comfortable" }}>Density</button> + </div> + <p data-testid="preview">{`Preview: ${draftTheme} / ${draftDensity}`}</p> + <button data-testid="save" onClick={() => { if (!invalid) { save({ name: draftName, theme: draftTheme, density: draftDensity }) } }}>Save</button> + </div> +} + +export default component App() { + savingNow = pending? saved + savedLine = `Saved: ${saved.name} · ${saved.theme} · ${saved.density}${savingNow ? " (saving…)" : ""}` + + <main> + {errored fallback(e) { + <div> + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { refresh saved }}>Retry</button> + </div> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{savedLine}</p> + {if saveError != "" { <p data-testid="save-error">{`Save failed: ${saveError}`}</p> }} + <Editor initial={saved} /> + }} + }} + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/runlog.jsonl new file mode 100644 index 0000000..3285469 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:10:39.892Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/plain-small/settings-rx-3/src/main.rx: line 35: expression near '<div>\n <p dat'"} +{"at":"2026-10-27T19:10:45.439Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:10:53.381Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:11:15.986Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/plain-2026-10-27/runs/small/settings-rx-3/store.rx b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/store.rx new file mode 100644 index 0000000..a50be64 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/settings-rx-3/store.rx @@ -0,0 +1,19 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" + +export type Settings = { name: string; theme: string; density: string } + +export source saved: Settings = await loadSettings() +export source saveError: string = "" + +export action save(next: Settings) { + optimistic saved := next + optimistic saveError := "" + affects saved + try { + let answer = await saveSettings(next) + saved := answer + } catch (e) { + saveError := e.message + } +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-1/App.tsx b/eval/results/plain-2026-10-27/runs/small/tracker-react-1/App.tsx new file mode 100644 index 0000000..7ef2eca --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-1/App.tsx @@ -0,0 +1,191 @@ +import { useEffect, useState } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; + +type ListState = + | { filter: Filter; ok: true; issues: Issue[] } + | { filter: Filter; ok: false; message: string }; + +type DetailResult = { value: IssueDetail | null } | { message: string }; + +type DetailState = { id: number; attempt: number; result: DetailResult }; + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +export default function App() { + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [list, setList] = useState<ListState | null>(null); + const [selected, setSelected] = useState<number | null>(null); + const [attempt, setAttempt] = useState(0); + const [detail, setDetail] = useState<DetailState | null>(null); + const [assignError, setAssignError] = useState<{ id: number; message: string } | null>(null); + + // Load the issues for the current filter. Results from a filter that is no longer + // current are dropped by the cleanup. + useEffect(() => { + let live = true; + fetchIssues(filter).then( + issues => { if (live) setList({ filter, ok: true, issues }); }, + e => { if (live) setList({ filter, ok: false, message: errorMessage(e) }); }, + ); + return () => { live = false; }; + }, [filter]); + + // Load the selected issue's detail, again whenever a retry is asked for. + useEffect(() => { + if (selected === null) return; + let live = true; + const id = selected; + const at = attempt; + const done = (result: DetailResult) => { if (live) setDetail({ id, attempt: at, result }); }; + fetchIssue(id).then( + value => done({ value }), + e => done({ message: errorMessage(e) }), + ); + return () => { live = false; }; + }, [selected, attempt]); + + const changeFilter = (next: Filter) => { + if (next === filter) return; + setList(null); + setFilter(next); + }; + + const select = (id: number) => { + setAssignError(null); + setDetail(null); + setSelected(id); + }; + + const retry = () => { + setDetail(null); + setAttempt(a => a + 1); + }; + + const setAssignee = (id: number, assignee: string | null) => + setList(l => (l && l.ok ? { ...l, issues: l.issues.map(i => (i.id === id ? { ...i, assignee } : i)) } : l)); + + const assignSelected = async (id: number) => { + const who = user; + const row = list?.ok ? list.issues.find(i => i.id === id) : undefined; + const before = row ? row.assignee : null; + setAssignError(null); + setAssignee(id, who); + try { + const updated = await assign(id, who); + setAssignee(id, updated.assignee); + } catch (e) { + setAssignee(id, before); + setAssignError({ id, message: errorMessage(e) }); + } + }; + + const listLoading = list === null || list.filter !== filter; + + let listContent; + if (listLoading) { + listContent = <p>{`loading ${filter} issues`}</p>; + } else if (!list.ok) { + listContent = <p>{`list error: ${list.message}`}</p>; + } else { + listContent = ( + <ul style={{ listStyle: "none", padding: 0, margin: 0 }}> + {list.issues.map(i => ( + <li key={i.id}> + <button + type="button" + data-testid="issue" + aria-current={i.id === selected ? "true" : undefined} + onClick={() => select(i.id)} + style={{ width: "100%", textAlign: "left", padding: "6px 8px", background: i.id === selected ? "#e8eefc" : "transparent" }} + > + {`#${i.id} ${i.title} (${i.assignee ?? "unassigned"})`} + </button> + </li> + ))} + </ul> + ); + } + + const detailCurrent = detail !== null && detail.id === selected && detail.attempt === attempt; + + let detailContent; + if (selected === null) { + detailContent = <p>select an issue</p>; + } else if (!detailCurrent) { + detailContent = <p>{`loading #${selected}`}</p>; + } else if ("message" in detail.result) { + detailContent = <p>{`detail error: ${detail.result.message}`}</p>; + } else if (detail.result.value === null) { + detailContent = <p>{`detail error: issue #${selected} not found`}</p>; + } else { + detailContent = ( + <> + <h2>{`#${detail.result.value.id} ${detail.result.value.title}`}</h2> + <p>{detail.result.value.body}</p> + </> + ); + } + const showRetry = detailCurrent && ("message" in detail.result || detail.result.value === null); + + return ( + <div style={{ fontFamily: "system-ui, sans-serif", padding: 16 }}> + <header style={{ marginBottom: 12 }}> + <label> + Current user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + </header> + <div style={{ display: "flex", gap: 16, alignItems: "flex-start" }}> + <section style={{ flex: "1 1 0", minWidth: 0 }}> + <div style={{ display: "flex", gap: 8, marginBottom: 8 }}> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => changeFilter("open")} + > + open + </button> + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => changeFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + <section data-testid="detail" style={{ flex: "1 1 0", minWidth: 0 }}> + {detailContent} + {showRetry && ( + <button type="button" data-testid="retry" onClick={retry}> + retry + </button> + )} + {selected !== null && ( + <div style={{ marginTop: 12 }}> + <button + type="button" + data-testid="assign" + onClick={() => assignSelected(selected)} + > + {`assign to ${user}`} + </button> + {assignError !== null && assignError.id === selected && ( + <p>{`detail error: ${assignError.message}`}</p> + )} + </div> + )} + </section> + </div> + </div> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-react-1/runlog.jsonl new file mode 100644 index 0000000..d726f8c --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:03:48.764Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:03:58.366Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-2/App.tsx b/eval/results/plain-2026-10-27/runs/small/tracker-react-2/App.tsx new file mode 100644 index 0000000..8e22827 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-2/App.tsx @@ -0,0 +1,215 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type AssignError = { id: number; message: string }; +type AssignVars = { id: number; user: string; before: string | null }; + +export default function App() { + const [queryClient] = useState( + () => new QueryClient({ defaultOptions: { queries: { retry: false, refetchOnWindowFocus: false } } }), + ); + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} + +function setAssignee(queryClient: QueryClient, id: number, assignee: string | null) { + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, list => + list?.map(issue => (issue.id === id ? { ...issue, assignee } : issue)), + ); +} + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selectedId, setSelectedId] = useState<number | null>(null); + const [assignError, setAssignError] = useState<AssignError | null>(null); + + const assignMutation = useMutation({ + mutationFn: ({ id, user }: AssignVars) => assign(id, user), + onSuccess: (result, vars) => setAssignee(queryClient, vars.id, result.assignee), + onError: (error, vars) => { + setAssignee(queryClient, vars.id, vars.before); + setAssignError({ id: vars.id, message: error.message }); + }, + }); + + const select = (id: number) => { + if (id !== selectedId) setAssignError(null); + setSelectedId(id); + }; + + const assignSelected = (id: number) => { + setAssignError(null); + // The assignee the row had before this assignment, taken from any cached list that shows it. + let before: string | null = null; + for (const [, list] of queryClient.getQueriesData<Issue[]>({ queryKey: ["issues"] })) { + const row = list?.find(issue => issue.id === id); + if (row) { before = row.assignee; break; } + } + setAssignee(queryClient, id, user); + assignMutation.mutate({ id, user, before }); + }; + + return ( + <div style={{ fontFamily: "system-ui, sans-serif", padding: 16, maxWidth: 1000, margin: "0 auto" }}> + <h1 style={{ fontSize: 22, margin: "0 0 12px" }}>Issue tracker</h1> + <label style={{ display: "block", marginBottom: 16 }}> + Current user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div style={{ display: "flex", gap: 24, flexWrap: "wrap", alignItems: "flex-start" }}> + <section style={{ flex: "1 1 320px", minWidth: 0 }}> + <div style={{ marginBottom: 8 }}> + <button + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button>{" "} + <button + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <IssueList filter={filter} selectedId={selectedId} onSelect={select} /> + </section> + <section style={{ flex: "1 1 320px", minWidth: 0 }}> + <Detail + selectedId={selectedId} + user={user} + assignError={assignError} + onAssign={assignSelected} + /> + </section> + </div> + </div> + ); +} + +function IssueList({ + filter, + selectedId, + onSelect, +}: { + filter: Filter; + selectedId: number | null; + onSelect: (id: number) => void; +}) { + const query = useQuery({ queryKey: ["issues", filter], queryFn: () => fetchIssues(filter) }); + + let content; + if (query.isFetching) { + content = <p>{`loading ${filter} issues`}</p>; + } else if (query.isError) { + content = <p>{`list error: ${query.error.message}`}</p>; + } else { + content = ( + <ul style={{ listStyle: "none", padding: 0, margin: 0 }}> + {(query.data ?? []).map(issue => { + const selected = issue.id === selectedId; + return ( + <li + key={issue.id} + data-testid="issue" + role="button" + tabIndex={0} + aria-pressed={selected} + onClick={() => onSelect(issue.id)} + onKeyDown={e => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onSelect(issue.id); + } + }} + style={{ + padding: "8px 10px", + borderBottom: "1px solid #ddd", + cursor: "pointer", + background: selected ? "#e8f0fe" : "transparent", + }} + > + {`#${issue.id} ${issue.title} (${issue.assignee ?? "unassigned"})`} + </li> + ); + })} + </ul> + ); + } + + return ( + <div data-testid="list"> + {content} + </div> + ); +} + +function Detail({ + selectedId, + user, + assignError, + onAssign, +}: { + selectedId: number | null; + user: string; + assignError: AssignError | null; + onAssign: (id: number) => void; +}) { + if (selectedId === null) { + return <div data-testid="detail"><p>select an issue</p></div>; + } + return ( + <div data-testid="detail"> + <IssueContent key={selectedId} id={selectedId} /> + <p> + <button data-testid="assign" onClick={() => onAssign(selectedId)}> + {`assign to ${user}`} + </button> + </p> + {assignError && assignError.id === selectedId && ( + <p>{`assign error: ${assignError.message}`}</p> + )} + </div> + ); +} + +function IssueContent({ id }: { id: number }) { + const query = useQuery({ queryKey: ["issue", id], queryFn: () => fetchIssue(id) }); + + let body; + if (query.isFetching) { + body = <p>{`loading #${id}`}</p>; + } else if (query.isError) { + body = <p>{`detail error: ${query.error.message}`}</p>; + } else if (query.data === null) { + body = <p>{`detail error: issue #${id} not found`}</p>; + } else if (query.data) { + body = ( + <> + <h2 style={{ fontSize: 18, margin: "0 0 8px" }}>{`#${query.data.id} ${query.data.title}`}</h2> + <p>{query.data.body}</p> + </> + ); + } + + return ( + <div> + {body} + <p> + <button data-testid="retry" onClick={() => void query.refetch()}> + retry + </button> + </p> + </div> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-react-2/runlog.jsonl new file mode 100644 index 0000000..658d1c8 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:05:50.330Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:05:58.157Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-3/App.tsx b/eval/results/plain-2026-10-27/runs/small/tracker-react-3/App.tsx new file mode 100644 index 0000000..0322e58 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-3/App.tsx @@ -0,0 +1,178 @@ +import { useEffect, useState } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; + +/** the list for `filter`: `issues` once loaded, `error` if the load failed */ +type ListResult = { filter: Filter; issues?: Issue[]; error?: string }; +/** the detail for issue `id`: `detail` is null when there is no such issue */ +type DetailResult = { id: number; detail?: IssueDetail | null; error?: string }; +type AssignError = { id: number; message: string }; + +function messageOf(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +export default function App() { + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [list, setList] = useState<ListResult | null>(null); + const [selectedId, setSelectedId] = useState<number | null>(null); + const [attempt, setAttempt] = useState(0); + const [detail, setDetail] = useState<DetailResult | null>(null); + const [assignError, setAssignError] = useState<AssignError | null>(null); + + // Load the list for the current filter. A result that arrives after the filter + // has changed is ignored by the cleanup. + useEffect(() => { + let live = true; + fetchIssues(filter).then( + issues => { if (live) setList({ filter, issues }); }, + e => { if (live) setList({ filter, error: messageOf(e) }); }, + ); + return () => { live = false; }; + }, [filter]); + + // Load the selected issue. `attempt` changes on retry. + useEffect(() => { + if (selectedId === null) return; + let live = true; + fetchIssue(selectedId).then( + d => { if (live) setDetail({ id: selectedId, detail: d }); }, + e => { if (live) setDetail({ id: selectedId, error: messageOf(e) }); }, + ); + return () => { live = false; }; + }, [selectedId, attempt]); + + function chooseFilter(f: Filter) { + if (f === filter) return; + setFilter(f); + setList(null); + } + + function select(id: number) { + if (id === selectedId) return; + setSelectedId(id); + setDetail(null); + setAssignError(null); + } + + function retry() { + setDetail(null); + setAttempt(a => a + 1); + } + + function setAssignee(id: number, assignee: string | null) { + setList(prev => + prev?.issues + ? { ...prev, issues: prev.issues.map(i => (i.id === id ? { ...i, assignee } : i)) } + : prev, + ); + } + + async function assignSelected() { + if (selectedId === null) return; + const id = selectedId; + const who = user; + const before = list?.issues?.find(i => i.id === id)?.assignee ?? null; + setAssignError(null); + setAssignee(id, who); + try { + await assign(id, who); + } catch (e) { + setAssignee(id, before); + setAssignError({ id, message: messageOf(e) }); + } + } + + const detailLoading = selectedId !== null && (detail === null || detail.id !== selectedId); + const detailFailed = selectedId !== null && !detailLoading && detail !== null + && (detail.error !== undefined || detail.detail === null); + + let listContent; + if (list === null) { + listContent = <p>{`loading ${filter} issues`}</p>; + } else if (list.error !== undefined) { + listContent = <p role="alert">{`list error: ${list.error}`}</p>; + } else { + listContent = ( + <ul> + {(list.issues ?? []).map(i => ( + <li + key={i.id} + data-testid="issue" + role="button" + tabIndex={0} + onClick={() => select(i.id)} + onKeyDown={e => { + if (e.key === "Enter" || e.key === " ") select(i.id); + }} + > + {`#${i.id} ${i.title} (${i.assignee ?? "unassigned"})`} + </li> + ))} + </ul> + ); + } + + let detailContent; + if (selectedId === null) { + detailContent = <p>select an issue</p>; + } else if (detailLoading || detail === null) { + detailContent = <p>{`loading #${selectedId}`}</p>; + } else if (detail.error !== undefined) { + detailContent = <p role="alert">{`detail error: ${detail.error}`}</p>; + } else if (detail.detail === null || detail.detail === undefined) { + detailContent = <p role="alert">{`detail error: issue #${selectedId} not found`}</p>; + } else { + detailContent = ( + <> + <h2>{`#${detail.detail.id} ${detail.detail.title}`}</h2> + <p>{detail.detail.body}</p> + </> + ); + } + + return ( + <main style={{ fontFamily: "system-ui, sans-serif", display: "flex", gap: "2rem", padding: "1rem" }}> + <section style={{ flex: "1 1 0", minWidth: 0 }}> + <label> + User <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div style={{ margin: "0.5rem 0" }}> + <button + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => chooseFilter("open")} + > + open + </button>{" "} + <button + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => chooseFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + + <section data-testid="detail" style={{ flex: "1 1 0", minWidth: 0 }}> + {detailContent} + {detailFailed && ( + <button data-testid="retry" onClick={retry}>retry</button> + )} + {selectedId !== null && ( + <div style={{ marginTop: "1rem" }}> + <button data-testid="assign" onClick={assignSelected}>{`assign to ${user}`}</button> + {assignError !== null && assignError.id === selectedId && ( + <p role="alert">{assignError.message}</p> + )} + </div> + )} + </section> + </main> + ); +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-react-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-react-3/runlog.jsonl new file mode 100644 index 0000000..4160641 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T19:10:10.009Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T19:10:17.236Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/main.rx b/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/main.rx new file mode 100644 index 0000000..406055c --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/main.rx @@ -0,0 +1,77 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it: { id: number; title: string; body: string } | null, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user := "guest" + source filter := "open" + source selected: number | null := null + source assignError := "" + source issues = await fetchIssues(filter) + issue = selected == null ? null : found(await fetchIssue(selected), selected) + + action assignIssue(id: number, who: string) { + assignError := "" + affects issues + try { + optimistic issues := issues.map(i => i.id == id ? { ...i, assignee: who } : i) + let answer = await assign(id, who) + issues := issues.map(i => i.id == answer.id ? { ...i, assignee: answer.assignee } : i) + } catch (e) { + assignError := e.message + } + } + + action retry() { + refresh issue + } + + <main> + <label> + user <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /> + </label> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div class="panes"> + <section data-testid="list"> + {errored fallback(e) { + <p>list error: {e.message}</p> + } { + {loading on(filter) fallback { + <p>loading {filter} issues</p> + } { + {for i in issues { + <div data-testid="issue" onClick={() => { selected := i.id; assignError := "" }}> + {`#${i.id} ${i.title} (${i.assignee ?? "unassigned"})`} + </div> + }} + }} + }} + </section> + <section data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + {errored fallback(e) { + <p>detail error: {e.message}</p> + <button data-testid="retry" onClick={() => { retry() }}>retry</button> + } { + {loading on(selected) fallback { + <p>loading #{selected}</p> + } { + <h2>{`#${issue.id} ${issue.title}`}</h2> + <p>{issue.body}</p> + <button data-testid="assign" onClick={() => { assignIssue(selected, user) }}>assign to {user}</button> + <p>{assignError}</p> + }} + }} + }} + </section> + </div> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl new file mode 100644 index 0000000..ac168bf --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T19:03:50.023Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-small/tracker-rx-1/src/main.rx:16: error OPTIMISTIC_NO_QUESTION [AC-3] assignError is declared with := — a guess needs a question to confirm it (source assignError = …, ST-5c)\n/home/claude/runs/plain-small/tracker-rx-1/src/main.rx:16: error UNCAUGHT_ACTION_FAILURE [R-f-6] action assignIssue can fail with Error and does not catch it: a handler cannot (it does not await the action) — `try { … } catch (e) { … }` in the body"} +{"at":"2026-10-27T19:03:57.554Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:04:05.427Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:04:20.745Z","cmd":"test","ok":true,"build":true,"passed":3,"total":3} +{"at":"2026-10-27T19:04:25.712Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/main.rx b/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/main.rx new file mode 100644 index 0000000..7422277 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/main.rx @@ -0,0 +1,71 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" + +error IssueNotFound(id: number) => `issue #${id} not found` + +fn found(it, id) => it == null ? throw IssueNotFound(id) : it + +fn rowText(i, g) => `#${i.id} ${i.title} (${g != null && g.id == i.id ? g.assignee : (i.assignee == null ? "unassigned" : i.assignee)})` + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source assignError: string := "" + source inFlight: { id: number; assignee: string } | null := null + + source issues key id = await fetchIssues(filter) + issue = selected == null ? null : found(await fetchIssue(selected), selected) + + action assignIssue(id, who) { + try { + await assign(id, who) + issues := rows => rows.map(i => i.id == id ? { ...i, assignee: who } : i) + inFlight := null + } catch (e) { + inFlight := null + assignError := e.message + } + } + + <main> + <section> + <div> + <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div data-testid="list"> + {loading on(filter) fallback { <p>{`loading ${filter} issues`}</p> } { + errored fallback(e) { <p>{`list error: ${e.message}`}</p> } { + <ul> + {for i in issues { + <li data-testid="issue" onClick={() => { onFlush(() => { assignError := "" }); selected := i.id }}>{rowText(i, inFlight)}</li> + }} + </ul> + } + }} + </div> + </section> + <section data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + <button data-testid="retry" onClick={() => { refresh issue }}>retry</button> + <button data-testid="assign" onClick={() => { + let id = selected + let who = user + onFlush(() => { inFlight := { id: id, assignee: who }; assignError := "" }) + assignIssue(id, who) + }}>{`assign to ${user}`}</button> + loading on(selected) fallback { <p>{`loading #${selected}`}</p> } { + errored fallback(e) { <p>{`detail error: ${e.message}`}</p> } { + <h2>{`#${issue.id} ${issue.title}`}</h2> + <p>{issue.body}</p> + } + } + <p>{assignError}</p> + }} + </section> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl new file mode 100644 index 0000000..8a1ccf6 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl @@ -0,0 +1,14 @@ +{"at":"2026-10-27T19:10:12.392Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-small/tracker-rx-2/src/main.rx:17: error OPTIMISTIC_NO_QUESTION [AC-3] assignError is declared with := — a guess needs a question to confirm it (source assignError = …, ST-5c)\n/home/claude/runs/plain-small/tracker-rx-2/src/main.rx:17: error UNCAUGHT_ACTION_FAILURE [R-f-6] action assignIssue can fail with Error and does not catch it: a handler cannot (it does not await the action) — `try { … } catch (e) { … }` in the body"} +{"at":"2026-10-27T19:10:18.361Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:10:26.313Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:10:38.076Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:10:43.749Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:10:48.111Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:10:53.400Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:11:03.998Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:11:46.518Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:11:51.525Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T19:12:17.552Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:12:22.277Z","cmd":"test","ok":false,"build":true,"passed":1,"total":4} +{"at":"2026-10-27T19:13:06.174Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:13:11.186Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/main.rx b/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/main.rx new file mode 100644 index 0000000..9e79b34 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/main.rx @@ -0,0 +1,68 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + + // A question: asked again when `filter` changes. Writable: the assign action guesses into it. + source issues = await fetchIssues(filter) + // The selected issue's detail. A `null` answer is a declared failure (NotFound). + detail = selected == null ? null : found(await fetchIssue(selected), selected) + // The last assignment's server message, or "" when there is none. + source assignError: string = "" + + action assignTo(id: number, who: string) { + optimistic assignError := "" + try { + optimistic issues := issues.map(r => (r.id == id ? { ...r, assignee: who } : r)) + let answer = await assign(id, who) + issues := issues.map(r => (r.id == answer.id ? { ...r, assignee: answer.assignee } : r)) + } catch (e) { + assignError := e.message + } + } + + <main> + <div class="bar"> + <label>user <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /></label> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div class="panes"> + <section data-testid="list"> + {loading on(filter) fallback { <p>loading {filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + {for i in issues { + <button data-testid="issue" class="row" onClick={() => { selected := i.id; onFlush(() => { assignError := "" }) }}> + #{i.id} {i.title} ({i.assignee ?? "unassigned"}) + </button> + }} + }} + }} + </section> + <section data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + <button data-testid="assign" onClick={() => { assignTo(selected ?? 0, user) }}>assign to {user}</button> + {loading on(selected) fallback { <p>loading #{selected}</p> } { + {errored fallback(e) { + <p>detail error: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + <p class="assign-error">{assignError}</p> + }} + </section> + </div> + </main> +} diff --git a/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl b/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl new file mode 100644 index 0000000..61553d7 --- /dev/null +++ b/eval/results/plain-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T19:10:28.134Z","cmd":"check","ok":false,"out":"/home/claude/runs/plain-small/tracker-rx-3/src/main.rx:20: error UNCAUGHT_ACTION_FAILURE [R-f-6] action assignTo can fail with Error and does not catch it: a handler cannot (it does not await the action) — `try { … } catch (e) { … }` in the body"} +{"at":"2026-10-27T19:10:36.200Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T19:10:44.205Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T19:21:29.348Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/plain-2026-10-27/usage.txt b/eval/results/plain-2026-10-27/usage.txt new file mode 100644 index 0000000..bf5ab99 --- /dev/null +++ b/eval/results/plain-2026-10-27/usage.txt @@ -0,0 +1,36 @@ +big settings-react-1 tokens=45967 tools=6 ms=58975 +big search-react-1 tokens=47038 tools=5 ms=69406 +big tracker-rx-1 tokens=73739 tools=8 ms=74537 +big tracker-react-1 tokens=49648 tools=6 ms=85684 +big search-rx-1 tokens=65179 tools=12 ms=96791 note=added+removed own tests +small settings-react-1 tokens=52909 tools=8 ms=83865 +small search-react-1 tokens=52436 tools=7 ms=87818 +big search-react-2 tokens=44579 tools=5 ms=46022 +big settings-react-2 tokens=46860 tools=6 ms=59418 +big settings-rx-1 tokens=74744 tools=15 ms=127689 note=added+removed own tests +small tracker-react-1 tokens=61146 tools=6 ms=103307 +small search-rx-1 tokens=92463 tools=13 ms=116380 +small tracker-rx-1 tokens=80897 tools=15 ms=131973 note=added+restored own tests +small settings-rx-1 tokens=86026 tools=14 ms=133066 note=added+restored own tests +big search-rx-2 tokens=63588 tools=7 ms=61352 +big tracker-rx-2 tokens=72882 tools=9 ms=84209 +big search-react-3 tokens=44926 tools=5 ms=46655 +small settings-react-2 tokens=51418 tools=6 ms=60808 +big tracker-react-2 tokens=50827 tools=6 ms=98047 +big settings-react-3 tokens=45920 tools=4 ms=50061 +small search-react-2 tokens=53355 tools=7 ms=71989 +big settings-rx-2 tokens=59795 tools=12 ms=81565 note=added+restored own tests +small tracker-react-2 tokens=63215 tools=7 ms=107948 +small search-rx-2 tokens=93453 tools=15 ms=127191 note=scratch tests in temp copy +big tracker-react-3 tokens=48180 tools=5 ms=67411 +big settings-rx-3 tokens=67813 tools=11 ms=72389 +small search-react-3 tokens=54202 tools=7 ms=73965 +small settings-react-3 tokens=50491 tools=5 ms=53528 +big tracker-rx-3 tokens=72259 tools=12 ms=91342 +big search-rx-3 tokens=66374 tools=12 ms=97834 +small tracker-react-3 tokens=56175 tools=6 ms=78610 +small search-rx-3 tokens=84251 tools=12 ms=122248 +small settings-rx-3 tokens=89407 tools=14 ms=141156 +small tracker-rx-2 tokens=107704 tools=20 ms=256828 +small settings-rx-2 tokens=91014 tools=22 ms=313319 +small tracker-rx-3 tokens=96051 tools=22 ms=753028 From 902ffeec7796222d540d5359395df68dbb5b6b8b Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 20:03:56 +0000 Subject: [PATCH 03/11] eval: exploratory overlapping-write probes (not pre-registered) Run against round 2's finished apps and the references to choose round 3's direction. Not a pre-registered test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/probes/settings.mjs | 22 ++++++++++++++++++++++ eval/probes/tracker.mjs | 29 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 eval/probes/settings.mjs create mode 100644 eval/probes/tracker.mjs diff --git a/eval/probes/settings.mjs b/eval/probes/settings.mjs new file mode 100644 index 0000000..2cc0f34 --- /dev/null +++ b/eval/probes/settings.mjs @@ -0,0 +1,22 @@ +// Exploratory probes (round 3 candidates), NOT pre-registered: overlapping saves. +import { saving } from "../tasks/settings/fixtures.mjs"; + +const key = async (app, name) => { const k = (await app.pending()).find(k => k.startsWith("saveSettings:") && k.includes(`"name":"${name}"`)); if (!k) throw new Error(`no save of ${name} in flight: ${JSON.stringify(await app.pending())}`); return k; }; + +export default [ + { name: "an earlier save failing after a later one succeeded leaves the later one", async run(app) { + await saving(app, "Bea", "dark"); + await app.type("name", "Cy"); await app.click("save"); + await app.land(await key(app, "Cy"), { name: "Cy", theme: "dark", density: "compact" }); + await app.fail(await key(app, "Bea"), "quota exceeded"); + await app.eq("saved", "Saved: Cy · dark · compact"); + } }, + { name: "a later save failing while an earlier one is still out shows the error, then the earlier result", async run(app) { + await saving(app, "Bea", "dark"); + await app.type("name", "Cy"); await app.click("save"); + await app.fail(await key(app, "Cy"), "quota exceeded"); + await app.land(await key(app, "Bea"), { name: "Bea", theme: "dark", density: "compact" }); + await app.eq("saved", "Saved: Bea · dark · compact"); + await app.see("save-error", "Save failed: quota exceeded"); + } }, +]; diff --git a/eval/probes/tracker.mjs b/eval/probes/tracker.mjs new file mode 100644 index 0000000..8f2a92e --- /dev/null +++ b/eval/probes/tracker.mjs @@ -0,0 +1,29 @@ +// Exploratory probes (round 3 candidates), NOT pre-registered: overlapping writes. Applied to finished runs only to +// see whether they discriminate. +import { D1, OPEN, loaded, selected } from "../tasks/tracker/fixtures.mjs"; + +export default [ + { name: "two issues assigned at once: declining one reverts only that row", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.click("assign"); + await app.fail("assign:3,ann", "declined"); + await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + await app.land("assign:1,ann", { id: 1, assignee: "ann" }); + await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + } }, + { name: "a list answer that left before an assignment doesn't undo it", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await app.click("filter-closed"); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:closed", []); + await app.landIfAsked("fetchIssues:open", OPEN); + await app.see("list", "#3 Crash on save (ann)"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.see("list", "#3 Crash on save (ann)"); + } }, +]; From dcae050fff78e241e82d99681581c67163edaf84 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 20:29:36 +0000 Subject: [PATCH 04/11] prelude: indexOf(xs, f), the index of the first match or -1 Per-row guesses (optimistic xs[i].field := v) need a row's index from an id. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- compiler/src/check.rs | 2 +- compiler/src/emit.rs | 2 +- compiler/src/project.rs | 2 +- ref/compile/emit.ts | 2 +- ref/compile/rt-server.ts | 1 + ref/compile/rt-web.ts | 1 + ref/compile/rt.ts | 1 + ref/src/check.ts | 2 +- ref/src/runtime.ts | 1 + spec/11-compiler.md | 2 +- 10 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compiler/src/check.rs b/compiler/src/check.rs index 2568428..eb2c39b 100644 --- a/compiler/src/check.rs +++ b/compiler/src/check.rs @@ -39,7 +39,7 @@ impl Diagnostic { } } -const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"]; +const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"]; /// EF-5's cleanup shapes: a `return` of a closure anywhere in the body, a closure as its last expression, or an expression body that is one. fn returns_closure(f: &Expr) -> bool { fn in_stmts(ss: &[Stmt]) -> bool { diff --git a/compiler/src/emit.rs b/compiler/src/emit.rs index ccbefe6..0fd6b91 100644 --- a/compiler/src/emit.rs +++ b/compiler/src/emit.rs @@ -21,7 +21,7 @@ fn fail<T>(msg: impl Into<String>) -> R<T> { Err(CompileError(msg.into())) } -const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every"]; +const BUILTINS: &[&str] = &["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"]; fn builtin(n: &str) -> bool { BUILTINS.contains(&n) } diff --git a/compiler/src/project.rs b/compiler/src/project.rs index 0f322f6..3fac266 100644 --- a/compiler/src/project.rs +++ b/compiler/src/project.rs @@ -18,7 +18,7 @@ pub const PRELUDE: &str = include_str!("prelude.ts"); static ROW_WORDS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^((?:io|pending|lane|fails\([^)]*\))\s+)+").unwrap()); static SOURCE_WORD: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^source\s+").unwrap()); -const BUILTINS: [&str; 12] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "log", "until", "onSettled", "onFlush"]; +const BUILTINS: [&str; 13] = ["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf", "log", "until", "onSettled", "onFlush"]; fn builtin(n: &str) -> bool { BUILTINS.contains(&n) } diff --git a/ref/compile/emit.ts b/ref/compile/emit.ts index 2308160..5dcd7e8 100644 --- a/ref/compile/emit.ts +++ b/ref/compile/emit.ts @@ -23,7 +23,7 @@ class Env { } type Ctx = { imp: boolean; gen?: boolean; async?: boolean; tail?: boolean; self?: string; effect?: boolean; dyn?: boolean }; -const BUILTINS = new Set(["len", "str", "map", "filter", "sum", "fail", "fetch", "every"]); +const BUILTINS = new Set(["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"]); const isHandler = (a: string) => /^on[A-Z:]/.test(a); const J = (v: unknown) => v === undefined ? "undefined" : JSON.stringify(v); const mg = (n: string) => `v$${n}`; diff --git a/ref/compile/rt-server.ts b/ref/compile/rt-server.ts index dba5629..e68c65b 100644 --- a/ref/compile/rt-server.ts +++ b/ref/compile/rt-server.ts @@ -103,6 +103,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> filter: (xs: unknown[], f: (x: unknown) => unknown) => xs.filter(f), sum: (xs: number[]) => xs.reduce((a, b) => a + b, 0), every: (xs: unknown[], f: (x: unknown) => unknown) => xs.every(f), + indexOf: (xs: unknown[], f: (x: unknown) => unknown) => xs.findIndex(f), fail: (msg: string) => { throw Object.assign(new Error(msg), { panic: true }); }, fetch: (key: string) => opts.api?.fetch ? opts.api.fetch(key) : flight(String(key)), log: (...a: unknown[]) => { log.push(`log(${a.map(fmt).join(", ")})`); }, diff --git a/ref/compile/rt-web.ts b/ref/compile/rt-web.ts index 8a602e8..c9cd021 100644 --- a/ref/compile/rt-web.ts +++ b/ref/compile/rt-web.ts @@ -118,6 +118,7 @@ export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) filter: (xs: unknown[], f: (x: unknown) => unknown) => xs.filter(f), sum: (xs: number[]) => xs.reduce((a, b) => a + b, 0), every: (xs: unknown[], f: (x: unknown) => unknown) => xs.every(f), + indexOf: (xs: unknown[], f: (x: unknown) => unknown) => xs.findIndex(f), fail: (msg: string) => { throw Object.assign(new Error(msg), { panic: true }); }, // R-f-3: a JS throw — a panic, not in the failure row fetch: (key: string) => opts.api?.fetch ? opts.api.fetch(key) : flight(String(key)), log: (...a: unknown[]) => { log.push(`log(${a.map(fmt).join(", ")})`); }, diff --git a/ref/compile/rt.ts b/ref/compile/rt.ts index 03e0f52..cb79ec2 100644 --- a/ref/compile/rt.ts +++ b/ref/compile/rt.ts @@ -128,6 +128,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> filter: (xs: unknown[], f: (x: unknown) => unknown) => xs.filter(f), sum: (xs: number[]) => xs.reduce((a, b) => a + b, 0), every: (xs: unknown[], f: (x: unknown) => unknown) => xs.every(f), + indexOf: (xs: unknown[], f: (x: unknown) => unknown) => xs.findIndex(f), fail: (msg: string) => { throw Object.assign(new Error(msg), { panic: true }); }, // R-f-3: a JS throw — a panic, not in the failure row fetch: (key: string) => opts.api?.fetch ? opts.api.fetch(key) : flight(String(key)), log: (...a: unknown[]) => { log.push(`log(${a.map(fmt).join(", ")})`); }, diff --git a/ref/src/check.ts b/ref/src/check.ts index 799d532..1299359 100644 --- a/ref/src/check.ts +++ b/ref/src/check.ts @@ -9,7 +9,7 @@ import type { JsDecl } from "./parse.ts"; import { misnested, misnestedInside, P_CLOSERS, P_SCOPE, EMPTY_SHAPE, type Shape } from "./content.ts"; import { ModelBuilder, type Model, type RegionKind, type CompKind } from "./model.ts"; -const BUILTINS = new Set(["len", "str", "map", "filter", "sum", "fail", "fetch", "every"]); // the scenario prelude; all pure +const BUILTINS = new Set(["len", "str", "map", "filter", "sum", "fail", "fetch", "every", "indexOf"]); // the scenario prelude; all pure const isHandler = (attr: string) => /^on[A-Z:]/.test(attr); // SC-4 // ---------------------------------------------------------------- regions and symbols diff --git a/ref/src/runtime.ts b/ref/src/runtime.ts index e2e748d..ae084df 100644 --- a/ref/src/runtime.ts +++ b/ref/src/runtime.ts @@ -1157,6 +1157,7 @@ const BUILTINS: Record<string, unknown> = { sum: (xs: number[]) => xs.reduce((a, b) => a + b, 0), fail: (msg: string) => { throw Object.assign(new Error(msg), { panic: true }); }, // R-f-3: a JS throw — a panic, not in the failure row every: (xs: unknown[], f: (x: unknown) => unknown) => xs.every(f), + indexOf: (xs: unknown[], f: (x: unknown) => unknown) => xs.findIndex(f), fetch: (key: string) => key, // intercepted in `call`: returns the Flight for `key` }; diff --git a/spec/11-compiler.md b/spec/11-compiler.md index 868f476..b194a32 100644 --- a/spec/11-compiler.md +++ b/spec/11-compiler.md @@ -147,7 +147,7 @@ record argued each way; this is the decision. | `serverRef(name, fn)`, `serverCall(name, gen, args)` | a server fn as the client calls it, and its body (12 SF-2, SF-6, SF-7) | Solid's `"use server"` functions | | `show(v)`, `scopedClass(v, hash)` | `{e}` (a value formatted, a view inserted), a dynamic class with the view's scope (CSS-2) | — | | `TypedErrored`, `Portal` | `errored fallback(e: A)` (R-f-4), `portal` (RG-8) | `Errored`, `Portal` | - | `B` | the language's built-in functions (`len`, `str`, `map`, `filter`, `sum`, `every`) and `log` | — | + | `B` | the language's built-in functions (`len`, `str`, `map`, `filter`, `sum`, `every`, `indexOf`) and `log` | — | The app itself: `mount(view, hydrating)` in the client build, `stream(view, sink)` in the server's. From 82176b60eadf3e897f942c30d39a8678b3547dc2 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 20:33:45 +0000 Subject: [PATCH 05/11] eval: pre-register round 3 (overlapping writes); guide teaches per-row guesses hidden-overlap.mjs per task (6 tests), score.mjs --overlap, the guide's 'Writing one row of a list' with indexOf, the .rx prompt names indexOf. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-overlap.md | 70 ++++++++++++++++++++++++++ eval/harness/prompt-arms.json | 2 +- eval/harness/score.mjs | 13 +++-- eval/tasks/search/hidden-overlap.mjs | 16 ++++++ eval/tasks/settings/hidden-overlap.mjs | 37 ++++++++++++++ eval/tasks/tracker/hidden-overlap.mjs | 28 +++++++++++ guide/async.md | 25 ++++++++- 7 files changed, 185 insertions(+), 6 deletions(-) create mode 100644 eval/PREREG-overlap.md create mode 100644 eval/tasks/search/hidden-overlap.mjs create mode 100644 eval/tasks/settings/hidden-overlap.mjs create mode 100644 eval/tasks/tracker/hidden-overlap.mjs diff --git a/eval/PREREG-overlap.md b/eval/PREREG-overlap.md new file mode 100644 index 0000000..f776588 --- /dev/null +++ b/eval/PREREG-overlap.md @@ -0,0 +1,70 @@ +# Round 3: overlapping writes + +Pre-registration. Everything in this file was fixed and committed before the round's first agent run. + +## Why this round + +Round 2 (`results/plain-2026-10-27/`) found no defect gap: with briefs that don't name the races, React shipped 0 +of 18 defects and .rx 1 of 18. Current models guard a single stale answer in React unprompted. What the agents +reported as unsure, and what no hidden test probed, was **overlapping writes**: two writes in flight at once, and a +write racing a re-load of the data it changed. Exploratory probes on round 2's finished apps (`probes/`, not a +test of anything) showed failures there on both sides. This round tests that question on fresh runs. + +## What changes from round 2 + +- **Hidden tests**: each task gets `tasks/<task>/hidden-overlap.mjs`, run in addition to the unchanged + `hidden.mjs`. Each test checks something the plain brief already states, in a case where two things are in + flight: + - search (1): a retry's answer landing after another selection must not show. The brief says the pane shows the + selected hit's item. + - tracker (2): two issues assigned at once, one declined; and the list loading while an assignment is out. The + brief says a declined row "goes back to the assignee it had before" and that the row shows the new assignee + "straight away" and "keeps" it if the server accepts. + - settings (3): an earlier save failing after a later one succeeded; a later save failing while an earlier one is + out; an earlier save landing while a later one is out. The brief says the saved line shows the values being + saved with `(saving…)`, the saved settings when a save succeeds, and goes back and shows `save-error` when it + fails. +- **The .rx guide** teaches guessing one row of a list (`optimistic xs[i].field := v`, with `indexOf` to find the + row) instead of the whole list. A whole-list guess makes two actions one change (H-11). The example is a generic + todo list, not a task's code. **The prelude** gains `indexOf(xs, f)` (Ryan, 2026-10-27), and the .rx prompt's + prelude list names it. +- Everything else is as in round 2: plain briefs, prompt, visible tests, toolchain, budget, 3 tasks × 2 arms × + 2 models (frontier, Haiku) × 3 runs = 36 fresh runs. + +Hidden test files, SHA-256 at freezing: + +``` +5776f8888055c4f5c13261b8637e048ebbb4f919602afb6f294a78ae4339b4d2 tasks/search/hidden-overlap.mjs +5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e tasks/tracker/hidden-overlap.mjs +aa1d42209d188ea4c54220c816d0ffe718e7ebe65c2d9b9d3fbf5fa23949aade tasks/settings/hidden-overlap.mjs +1d77a1daa81d15d40045d1f4665f9fdcebbe9abb6356db5b3a6eb8b29d30d0ae tasks/search/hidden.mjs +c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e tasks/tracker/hidden.mjs +58d2a1daa4a1b8d2ea559ab56bb37ee9193dcf3d31d03e362107e25ee3b106b6 tasks/settings/hidden.mjs +``` + +The references are not subjects, and neither passes everything: both fail some overlap tests (`reference/` is +unchanged from the pilot). The tests were checked to run and to fail for a stated reason, not to be passable by the +references. + +## Hypothesis and test + +- **H1**: the React arm ships more defects than the .rx arm. +- **Primary measure**: defects shipped, where a defect is a failed build or any failed visible, hidden or + hidden-overlap test at the agent's "done" (`node harness/score.mjs <dir> --overlap`). One number per arm, pooled + over tasks and models (18 runs each). +- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React rate + is higher and p < 0.05. **H1 reversed** (the .rx rate higher, p < 0.05) is reported as plainly. +- **Reported alongside**: the same table per model and per task; overlap tests failed per run; each failure + classified by hand (stale answer, lost rollback, wrong revert, error lost, runtime bug, build failure). +- **Runtime bugs**: solidjs/solid#4329 (a per-key guess dropped when a re-ask lands mid-action) is live in rc.19 and + touches the tracker's "list loading while an assignment is out" test for a .rx app that guesses per row. As in + round 2, such a failure counts as a defect in the primary measure (the user gets the bug either way) and is + listed separately with the primary result recomputed without it. +- Published as is, whatever the outcome. + +## Known limits + +- Isolation is still by instruction. +- The overlap tests were chosen after the exploratory probes had been run on round 2's apps. They run here only on + fresh runs, and four of the six are the probes as they were run. +- 18 runs per arm detects a large effect only (round 2's PREREG gives the arithmetic). diff --git a/eval/harness/prompt-arms.json b/eval/harness/prompt-arms.json index 12e418e..919de3c 100644 --- a/eval/harness/prompt-arms.json +++ b/eval/harness/prompt-arms.json @@ -7,7 +7,7 @@ }, "rx": { "DOCS_RULE": ", except that you may read the language documentation in {{DIR}}/docs", - "LANG_RULE": "ReactiveScript (`.rx`), a new language that compiles to Solid 2.0. Nothing about it is online. Read `docs/guide/async.md` first: one page on reading from and writing to a server. Its specification is in `docs/spec/` (the concrete syntax is `docs/spec/07-surface.md`; modules and imports are `docs/spec/09-modules.md`). `src/main.rx` must be the program's entry, `export default component App() { … }`; you may add `.rx` modules under `src/`. Import the backend from \"./api\": loaders with `import pure { … } from \"./api\"`, functions that write with `import { … } from \"./api\"`. The prelude gives `import pure { len, str, map, filter, sum, every } from \"./prelude\"`. Give an element a test id with `data-testid=\"…\"`.", + "LANG_RULE": "ReactiveScript (`.rx`), a new language that compiles to Solid 2.0. Nothing about it is online. Read `docs/guide/async.md` first: one page on reading from and writing to a server. Its specification is in `docs/spec/` (the concrete syntax is `docs/spec/07-surface.md`; modules and imports are `docs/spec/09-modules.md`). `src/main.rx` must be the program's entry, `export default component App() { … }`; you may add `.rx` modules under `src/`. Import the backend from \"./api\": loaders with `import pure { … } from \"./api\"`, functions that write with `import { … } from \"./api\"`. The prelude gives `import pure { len, str, map, filter, sum, every, indexOf } from \"./prelude\"`. Give an element a test id with `data-testid=\"…\"`.", "API_TYPES": "`src/api.d.rx`", "CHECKER": "the ReactiveScript compiler's checker (`rxc check src/main.rx`)" } diff --git a/eval/harness/score.mjs b/eval/harness/score.mjs index efdbc25..a518ac0 100644 --- a/eval/harness/score.mjs +++ b/eval/harness/score.mjs @@ -1,6 +1,7 @@ // Scores finished runs: for each run directory (named <task>-<arm>-<n>), the arm's check, the visible and the hidden // tests, and the agent's own log of `./check` and `./test` runs. Writes results.json and prints a summary table. -// Usage: node harness/score.mjs <runs-dir> [out.json] +// With --overlap, each task's hidden-overlap.mjs (round 3) runs too and counts toward the defect. +// Usage: node harness/score.mjs <runs-dir> [out.json] [--overlap] import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,7 +10,8 @@ import { runFile } from "./test.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); const tasksDir = path.resolve(here, "../tasks"); -const [runsArg, outArg] = process.argv.slice(2); +const overlap = process.argv.includes("--overlap"); +const [runsArg, outArg] = process.argv.slice(2).filter(a => a !== "--overlap"); const runs = path.resolve(runsArg); const rows = []; @@ -23,7 +25,9 @@ for (const name of readdirSync(runs).sort()) { const checked = check(dir); const visible = await runFile(dir, path.join(tasksDir, task, "visible.mjs")); const hidden = await runFile(dir, path.join(tasksDir, task, "hidden.mjs")); + const extra = overlap ? await runFile(dir, path.join(tasksDir, task, "hidden-overlap.mjs")) : null; const failedHidden = hidden.results.filter(r => !r.ok); + const failedOverlap = extra ? extra.results.filter(r => !r.ok) : []; rows.push({ run: name, task, arm, n: Number(n), checks: log.filter(e => e.cmd === "check").length, tests: log.filter(e => e.cmd === "test").length, @@ -31,11 +35,12 @@ for (const name of readdirSync(runs).sort()) { checkOk: checked.ok, built: hidden.build.ok, visible: `${visible.results.filter(r => r.ok).length}/${visible.results.length}`, hidden: `${hidden.results.length - failedHidden.length}/${hidden.results.length}`, - shippedDefect: !hidden.build.ok || visible.results.some(r => !r.ok) || failedHidden.length > 0, + ...(extra ? { overlap: `${extra.results.length - failedOverlap.length}/${extra.results.length}`, failedOverlap: failedOverlap.map(r => ({ name: r.name, error: r.error })) } : {}), + shippedDefect: !hidden.build.ok || visible.results.some(r => !r.ok) || failedHidden.length > 0 || failedOverlap.length > 0, failedHidden: failedHidden.map(r => ({ name: r.name, error: r.error })), failedVisible: visible.results.filter(r => !r.ok).map(r => ({ name: r.name, error: r.error })), }); const r = rows.at(-1); - console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}`); + console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}`); } writeFileSync(outArg ? path.resolve(outArg) : path.join(runs, "results.json"), JSON.stringify(rows, null, 2) + "\n"); diff --git a/eval/tasks/search/hidden-overlap.mjs b/eval/tasks/search/hidden-overlap.mjs new file mode 100644 index 0000000..6a71e51 --- /dev/null +++ b/eval/tasks/search/hidden-overlap.mjs @@ -0,0 +1,16 @@ +// Round 3 hidden tests: overlapping requests and writes. Never shown to the agent. +import { APP, ITEM4, searched } from "./fixtures.mjs"; + +export default [ + { name: "a retry's answer landing after another selection is ignored", async run(app) { + await searched(app, "app", APP); + await app.clickText("hit", "applesauce"); + await app.fail("fetchItem:3", "timeout"); + await app.click("retry"); + await app.clickText("hit", "apple pie"); + await app.land("fetchItem:4", ITEM4); + await app.land("fetchItem:3", { id: 3, name: "applesauce", blurb: "stewed apples" }); + await app.see("detail", "baked with cinnamon"); + await app.notSee("detail", "stewed apples", "failed"); + } }, +]; diff --git a/eval/tasks/settings/hidden-overlap.mjs b/eval/tasks/settings/hidden-overlap.mjs new file mode 100644 index 0000000..8fdbe06 --- /dev/null +++ b/eval/tasks/settings/hidden-overlap.mjs @@ -0,0 +1,37 @@ +// Round 3 hidden tests: overlapping saves. Never shown to the agent. +import { saving } from "./fixtures.mjs"; + +/** the key of the save of `name` in flight (the app may order the object's fields its own way) */ +const saveOf = async (app, name) => { + const k = (await app.pending()).find(k => k.startsWith("saveSettings:") && k.includes(`"name":"${name}"`)); + if (!k) throw new Error(`expected a save of ${name} in flight; in flight: ${JSON.stringify(await app.pending())}`); + return k; +}; +const BEA = { name: "Bea", theme: "dark", density: "compact" }; +const CY = { name: "Cy", theme: "dark", density: "compact" }; + +export default [ + { name: "an earlier save failing after a later one succeeded leaves the later one", async run(app) { + await saving(app, "Bea", "dark"); + await app.type("name", "Cy"); await app.click("save"); + await app.land(await saveOf(app, "Cy"), CY); + await app.fail(await saveOf(app, "Bea"), "quota exceeded"); + await app.eq("saved", "Saved: Cy · dark · compact"); + } }, + { name: "a later save failing while an earlier one is out shows its error and then the earlier result", async run(app) { + await saving(app, "Bea", "dark"); + await app.type("name", "Cy"); await app.click("save"); + await app.fail(await saveOf(app, "Cy"), "quota exceeded"); + await app.land(await saveOf(app, "Bea"), BEA); + await app.eq("saved", "Saved: Bea · dark · compact"); + await app.see("save-error", "Save failed: quota exceeded"); + } }, + { name: "an earlier save landing while a later one is out keeps showing the later one", async run(app) { + await saving(app, "Bea", "dark"); + await app.type("name", "Cy"); await app.click("save"); + await app.land(await saveOf(app, "Bea"), BEA); + await app.eq("saved", "Saved: Cy · dark · compact (saving…)"); + await app.land(await saveOf(app, "Cy"), CY); + await app.eq("saved", "Saved: Cy · dark · compact"); + } }, +]; diff --git a/eval/tasks/tracker/hidden-overlap.mjs b/eval/tasks/tracker/hidden-overlap.mjs new file mode 100644 index 0000000..3913295 --- /dev/null +++ b/eval/tasks/tracker/hidden-overlap.mjs @@ -0,0 +1,28 @@ +// Round 3 hidden tests: overlapping requests and writes. Never shown to the agent. +import { D1, OPEN, selected } from "./fixtures.mjs"; + +export default [ + { name: "two issues assigned at once: declining one reverts only that row", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.click("assign"); + await app.fail("assign:3,ann", "declined"); + await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + await app.land("assign:1,ann", { id: 1, assignee: "ann" }); + await app.see("list", "#3 Crash on save (unassigned)", "#1 Login fails (ann)"); + } }, + { name: "a list that loads while an assignment is out keeps the new assignee", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await app.click("filter-closed"); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:closed", []); + await app.landIfAsked("fetchIssues:open", OPEN); + await app.see("list", "#3 Crash on save (ann)"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.see("list", "#3 Crash on save (ann)"); + } }, +]; diff --git a/guide/async.md b/guide/async.md index 512a3a1..558d033 100644 --- a/guide/async.md +++ b/guide/async.md @@ -65,12 +65,35 @@ by what the screen should do: | Record the server's answer | `x := answer` after the `await` | | Handle a failure | `try { … } catch (e) { … }`: the catch's writes and the revert are one frame (AC-7) | +### Writing one row of a list + +When several actions can each change a different row of one list, guess the row, not the list: + +``` +export source todos: Todo[] key id = await loadTodos() + +export action markDone(id: number) { + try { + let i = indexOf(todos, t => t.id == id) // from the prelude; -1 when absent + optimistic todos[i].done := true // this row only + await saveDone(id) + refresh todos // the server's list confirms the guess + } catch (e) { … } +} +``` + +A guess on `todos[i].done` belongs to its own action: another action's guess on another row reverts on its own +(AC-3). A guess on the whole list (`optimistic todos := …`) makes every action on the list one change, so a +declined one stays on screen until the others end. After the `await`, re-ask the list (`refresh todos`) rather than +writing the row: a plain write to a list that is guessed by row is not supported yet. + ## 4. Things that look like bugs and aren't - **A plain write in an action doesn't show until the server answers.** That's the hold. Use `optimistic` or `onFlush` (the table above). Don't move the code into a plain handler to escape it. - **Two actions guessing the same source revert together.** Their guesses are one change (H-11): a declined one - goes back when the other ends. Don't rebuild guesses by hand to separate them. + goes back when the other ends. For a list, guess the row instead (above). Don't rebuild guesses by hand to + separate them. - **A click waits for an action it has nothing to do with.** A plain write to a source an action in flight has written or guessed joins that action's hold (H-2): a click that clears the action's error waits for the server. Keep such a write off the action's sources, or make it an `onFlush` run. From 120cea38e58692c7f7570971b8557ce23a966d8c Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 20:54:53 +0000 Subject: [PATCH 06/11] eval: round 3 results (overlapping writes) H1 not supported: React 4/18, .rx 5/18 defects (p = 1.0); 4/18 vs 2/18 without solidjs/solid#4329 (p = 0.66). All four .rx failures on the frontier model are #4329. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/results/overlap-2026-10-27/README.md | 86 ++++ .../overlap-2026-10-27/results-big.json | 341 ++++++++++++++++ .../overlap-2026-10-27/results-small.json | 369 ++++++++++++++++++ .../runs/big/search-react-1/App.tsx | 126 ++++++ .../runs/big/search-react-1/runlog.jsonl | 2 + .../runs/big/search-react-2/App.tsx | 122 ++++++ .../runs/big/search-react-2/runlog.jsonl | 2 + .../runs/big/search-react-3/App.tsx | 112 ++++++ .../runs/big/search-react-3/runlog.jsonl | 2 + .../runs/big/search-rx-1/main.rx | 55 +++ .../runs/big/search-rx-1/runlog.jsonl | 3 + .../runs/big/search-rx-2/main.rx | 51 +++ .../runs/big/search-rx-2/runlog.jsonl | 4 + .../runs/big/search-rx-3/main.rx | 64 +++ .../runs/big/search-rx-3/runlog.jsonl | 4 + .../runs/big/settings-react-1/App.tsx | 204 ++++++++++ .../runs/big/settings-react-1/runlog.jsonl | 2 + .../runs/big/settings-react-2/App.tsx | 161 ++++++++ .../runs/big/settings-react-2/runlog.jsonl | 2 + .../runs/big/settings-react-3/App.tsx | 109 ++++++ .../runs/big/settings-react-3/runlog.jsonl | 2 + .../runs/big/settings-rx-1/main.rx | 70 ++++ .../runs/big/settings-rx-1/runlog.jsonl | 3 + .../runs/big/settings-rx-2/main.rx | 72 ++++ .../runs/big/settings-rx-2/runlog.jsonl | 5 + .../runs/big/settings-rx-3/main.rx | 54 +++ .../runs/big/settings-rx-3/runlog.jsonl | 4 + .../runs/big/settings-rx-3/store.rx | 25 ++ .../runs/big/tracker-react-1/App.tsx | 175 +++++++++ .../runs/big/tracker-react-1/runlog.jsonl | 2 + .../runs/big/tracker-react-2/App.tsx | 160 ++++++++ .../runs/big/tracker-react-2/runlog.jsonl | 2 + .../runs/big/tracker-react-3/App.tsx | 172 ++++++++ .../runs/big/tracker-react-3/runlog.jsonl | 2 + .../runs/big/tracker-rx-1/main.rx | 72 ++++ .../runs/big/tracker-rx-1/runlog.jsonl | 5 + .../runs/big/tracker-rx-2/main.rx | 73 ++++ .../runs/big/tracker-rx-2/runlog.jsonl | 4 + .../runs/big/tracker-rx-3/main.rx | 71 ++++ .../runs/big/tracker-rx-3/runlog.jsonl | 4 + .../runs/small/search-react-1/App.tsx | 128 ++++++ .../runs/small/search-react-1/runlog.jsonl | 4 + .../runs/small/search-react-2/App.tsx | 125 ++++++ .../runs/small/search-react-2/runlog.jsonl | 2 + .../runs/small/search-react-3/App.tsx | 144 +++++++ .../runs/small/search-react-3/runlog.jsonl | 2 + .../runs/small/search-rx-1/main.rx | 74 ++++ .../runs/small/search-rx-1/runlog.jsonl | 7 + .../runs/small/search-rx-2/main.rx | 70 ++++ .../runs/small/search-rx-2/runlog.jsonl | 4 + .../runs/small/search-rx-3/main.rx | 63 +++ .../runs/small/search-rx-3/runlog.jsonl | 7 + .../runs/small/settings-react-1/App.tsx | 180 +++++++++ .../runs/small/settings-react-1/runlog.jsonl | 2 + .../runs/small/settings-react-2/App.tsx | 152 ++++++++ .../runs/small/settings-react-2/runlog.jsonl | 2 + .../runs/small/settings-react-3/App.tsx | 20 + .../small/settings-react-3/SettingsForm.tsx | 79 ++++ .../small/settings-react-3/SettingsPage.tsx | 57 +++ .../runs/small/settings-react-3/runlog.jsonl | 2 + .../runs/small/settings-rx-1/main.rx | 61 +++ .../runs/small/settings-rx-1/runlog.jsonl | 4 + .../runs/small/settings-rx-2/main.rx | 67 ++++ .../runs/small/settings-rx-2/runlog.jsonl | 6 + .../runs/small/settings-rx-3/main.rx | 82 ++++ .../runs/small/settings-rx-3/runlog.jsonl | 7 + .../runs/small/tracker-react-1/App.tsx | 195 +++++++++ .../runs/small/tracker-react-1/runlog.jsonl | 2 + .../runs/small/tracker-react-2/App.tsx | 180 +++++++++ .../runs/small/tracker-react-2/runlog.jsonl | 2 + .../runs/small/tracker-react-3/App.tsx | 224 +++++++++++ .../runs/small/tracker-react-3/runlog.jsonl | 2 + .../runs/small/tracker-rx-1/main.rx | 83 ++++ .../runs/small/tracker-rx-1/runlog.jsonl | 5 + .../runs/small/tracker-rx-2/main.rx | 73 ++++ .../runs/small/tracker-rx-2/runlog.jsonl | 5 + .../runs/small/tracker-rx-3/main.rx | 81 ++++ .../runs/small/tracker-rx-3/runlog.jsonl | 15 + eval/results/overlap-2026-10-27/usage.txt | 36 ++ 79 files changed, 5052 insertions(+) create mode 100644 eval/results/overlap-2026-10-27/README.md create mode 100644 eval/results/overlap-2026-10-27/results-big.json create mode 100644 eval/results/overlap-2026-10-27/results-small.json create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/search-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/settings-rx-3/store.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/search-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsForm.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsPage.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/settings-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-1/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-2/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-3/App.tsx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-react-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/main.rx create mode 100644 eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl create mode 100644 eval/results/overlap-2026-10-27/usage.txt diff --git a/eval/results/overlap-2026-10-27/README.md b/eval/results/overlap-2026-10-27/README.md new file mode 100644 index 0000000..e2bef55 --- /dev/null +++ b/eval/results/overlap-2026-10-27/README.md @@ -0,0 +1,86 @@ +# Round 3 result: overlapping writes + +Pre-registration: `../../PREREG-overlap.md` (commit 82176b6, before any run). 36 fresh runs: 3 tasks × 2 arms × 2 +models (the frontier model, and Haiku as the small one) × 3 runs. No run had to be repeated. Two runs ended without +sending their report and were asked for it once; neither changed its app after being asked. + +## Primary result: H1 not supported + +| Defects shipped | React | .rx | Fisher, two-sided | +| --- | --- | --- | --- | +| All (pre-registered test) | 4 / 18 | 5 / 18 | p = 1.0 | +| Frontier model | 0 / 9 | 3 / 9 | p = 0.21 | +| Small model | 4 / 9 | 2 / 9 | p = 0.62 | +| Without solidjs/solid#4329 (pre-registered recount) | 4 / 18 | 2 / 18 | p = 0.66 | + +A defect is a failed build or any failed visible, hidden or hidden-overlap test (`score.mjs --overlap`). Per task: +search 0/6 vs 0/6, settings 1/6 vs 0/6, tracker 3/6 vs 5/6. + +Neither direction is significant. The frontier model wrote correct React for every overlap case. Every .rx failure +it shipped is the same Solid runtime bug. + +## Every failure, classified + +| Run | Test failed | Class | Cause | +| --- | --- | --- | --- | +| React small settings-1 | a later save failing while an earlier one is out | wrong revert | the failed save reverts to the last confirmed settings, dropping the earlier save still in flight | +| React small tracker-1 | two issues assigned at once | wrong revert | declining #3 also reverts #1 | +| React small tracker-1 | list loads while an assignment is out | lost guess | the reloaded list overwrites the optimistic row | +| React small tracker-2 | list loads while an assignment is out | lost guess | same | +| React small tracker-3 | list loads while an assignment is out | lost guess | same | +| .rx frontier tracker-1 | list loads while an assignment is out | runtime bug | #4329 | +| .rx frontier tracker-2 | list loads while an assignment is out | runtime bug | #4329 | +| .rx frontier tracker-3 | list loads while an assignment is out | runtime bug | #4329 | +| .rx small tracker-2 | list loads while an assignment is out | runtime bug | #4329 | +| .rx small tracker-2 | (hidden) a new assignment clears the previous error | error lost | the old error stays up through the new assignment | +| .rx small tracker-2 | (hidden) selecting after not found shows loading | stale error | `errored` sits outside `loading on(selected)`, the same trap as round 2's one defect | +| .rx small tracker-1 | two issues assigned at once | wrong revert | a whole-list guess (`optimistic overrides := …`) makes both assignments one change (H-11), so the declined row stays until the other ends | + +**The runtime bug.** The four "runtime bug" runs write exactly the guide's idiom: `let i = indexOf(issues, …)`, +`optimistic issues[i].assignee := who`, `await assign(…)`, `refresh issues`. By the spec the guess holds until the +action ends (AC-3, D-34; Ryan's ruling of 2026-10-27). In rc.19, a list answer that lands while the action is +pending drops a per-key guess from the screen (solidjs/solid#4329). The failure message matches the issue's repro +(the row shows the server's old assignee). This is an attribution by mechanism, not by rerunning on a fixed runtime: +no fix exists yet. + +**The same race, both arms.** "A list that loads while an assignment is out" is the one case both arms failed. +3 of 3 small-model React apps failed it in their own code (the refetch overwrote the optimistic cache). 0 of 3 +frontier React apps failed it. 4 of the 6 .rx apps failed it with code that is correct by the spec. The other two +did not use a per-row guess and passed it. + +## Reported alongside + +| Mean per run | React, frontier | .rx, frontier | React, small | .rx, small | +| --- | --- | --- | --- | --- | +| Agent tokens | 46.6k | 65.7k | 53.9k | 87.3k | +| `./check` + `./test` cycles | 2.0 | 4.0 | 2.2 | 6.7 | + +- Guide uptake: 5 of 6 .rx tracker runs used the per-row guess with `indexOf`. Small tracker-1 used a whole-list + guess. Small tracker-3 reports that its per-row guess "left the list unchanged" in its own tests, and replaced it + with an `onFlush`-written overlay. That report is not checked: the frontier runs show the per-row guess rendering. +- No .rx run hand-rolled a request counter. React runs guarded stale answers with sequence numbers or query keys, as + in round 2. +- App size is in `runs/` (about 60 lines .rx, 140 React). It is not evidence for either arm. + +## Known limits, as found + +- **A leak between runs through a shared scratch folder.** Three frontier .rx runs backed up `tests/visible.mjs` to + the session's scratch folder (outside their run directories) before adding tests of their own. Big search-rx-1 and + big settings-rx-1 did this at the same time, and search-rx-1 restored the settings task's `visible.mjs` into its + own directory. Scoring uses the task's own test files, so no score changes; the search run never saw settings + code beyond what it copied back. The next round should give each agent its own scratch folder. +- The overlap tests were written after exploratory probes (`probes/`) on round 2's apps. +- 18 runs per arm detects a large effect only. + +## What this means + +Overlapping writes did not separate the arms either. The frontier model handled every overlap case in React. The +small model's React failed overlap cases in 4 of 9 runs, the .rx guide's idiom avoided them in code, and the runtime +then lost the guess in the one case it was meant to hold. Until #4329 is fixed, the idiom the guide teaches ships a +bug in that case. + +## Files + +- `results-big.json`, `results-small.json`: `score.mjs --overlap` output. +- `runs/<model>/<run>/`: each agent's own source files and its `./check`/`./test` log. +- `usage.txt`: agent tokens per run. diff --git a/eval/results/overlap-2026-10-27/results-big.json b/eval/results/overlap-2026-10-27/results-big.json new file mode 100644 index 0000000..ae4c365 --- /dev/null +++ b/eval/results/overlap-2026-10-27/results-big.json @@ -0,0 +1,341 @@ +[ + { + "run": "search-react-1", + "task": "search", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-2", + "task": "search", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-3", + "task": "search", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-1", + "task": "search", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-2", + "task": "search", + "arm": "rx", + "n": 2, + "checks": 1, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-3", + "task": "search", + "arm": "rx", + "n": 3, + "checks": 1, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-1", + "task": "settings", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-2", + "task": "settings", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-3", + "task": "settings", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-1", + "task": "settings", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-2", + "task": "settings", + "arm": "rx", + "n": 2, + "checks": 2, + "tests": 3, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-3", + "task": "settings", + "arm": "rx", + "n": 3, + "checks": 2, + "tests": 2, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-1", + "task": "tracker", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "2/2", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-2", + "task": "tracker", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "2/2", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-3", + "task": "tracker", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "2/2", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-1", + "task": "tracker", + "arm": "rx", + "n": 1, + "checks": 1, + "tests": 4, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-2", + "task": "tracker", + "arm": "rx", + "n": 2, + "checks": 1, + "tests": 3, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-3", + "task": "tracker", + "arm": "rx", + "n": 3, + "checks": 2, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + } +] diff --git a/eval/results/overlap-2026-10-27/results-small.json b/eval/results/overlap-2026-10-27/results-small.json new file mode 100644 index 0000000..a1bc2c3 --- /dev/null +++ b/eval/results/overlap-2026-10-27/results-small.json @@ -0,0 +1,369 @@ +[ + { + "run": "search-react-1", + "task": "search", + "arm": "react", + "n": 1, + "checks": 2, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-2", + "task": "search", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-react-3", + "task": "search", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-1", + "task": "search", + "arm": "rx", + "n": 1, + "checks": 3, + "tests": 4, + "cyclesToGreen": 6, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-2", + "task": "search", + "arm": "rx", + "n": 2, + "checks": 2, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "search-rx-3", + "task": "search", + "arm": "rx", + "n": 3, + "checks": 3, + "tests": 4, + "cyclesToGreen": 4, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "12/12", + "overlap": "1/1", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-1", + "task": "settings", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "2/3", + "failedOverlap": [ + { + "name": "a later save failing while an earlier one is out shows its error and then the earlier result", + "error": "expected [saved] to read \"Saved: Bea · dark · compact\", it reads \"Saved: Ann · light · compact\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-2", + "task": "settings", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-react-3", + "task": "settings", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-1", + "task": "settings", + "arm": "rx", + "n": 1, + "checks": 2, + "tests": 2, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-2", + "task": "settings", + "arm": "rx", + "n": 2, + "checks": 4, + "tests": 2, + "cyclesToGreen": 5, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "settings-rx-3", + "task": "settings", + "arm": "rx", + "n": 3, + "checks": 3, + "tests": 4, + "cyclesToGreen": 4, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "8/8", + "overlap": "3/3", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-1", + "task": "tracker", + "arm": "react", + "n": 1, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "0/2", + "failedOverlap": [ + { + "name": "two issues assigned at once: declining one reverts only that row", + "error": "expected [list] to show \"#1 Login fails (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + }, + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-2", + "task": "tracker", + "arm": "react", + "n": 2, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-react-3", + "task": "tracker", + "arm": "react", + "n": 3, + "checks": 1, + "tests": 1, + "cyclesToGreen": 2, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-1", + "task": "tracker", + "arm": "rx", + "n": 1, + "checks": 3, + "tests": 2, + "cyclesToGreen": 3, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "two issues assigned at once: declining one reverts only that row", + "error": "expected [list] to show \"#3 Crash on save (unassigned)\", it shows \"#1 Login fails (ann)#3 Crash on save (ann)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [], + "failedVisible": [] + }, + { + "run": "tracker-rx-2", + "task": "tracker", + "arm": "rx", + "n": 2, + "checks": 3, + "tests": 2, + "cyclesToGreen": 4, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "9/11", + "overlap": "1/2", + "failedOverlap": [ + { + "name": "a list that loads while an assignment is out keeps the new assignee", + "error": "expected [list] to show \"#3 Crash on save (ann)\", it shows \"#1 Login fails (unassigned)#3 Crash on save (unassigned)#4 Typo in footer (dana)\"" + } + ], + "shippedDefect": true, + "failedHidden": [ + { + "name": "a new assignment clears the previous error at once", + "error": "expected [detail] not to show \"declined\", it shows \"#3 Crash on savestack trace attachedretryassign to bodeclined\"" + }, + { + "name": "selecting after not found shows loading, not the old message", + "error": "expected [detail] to show \"loading #1\", it shows \"detail error: issue #3 not foundretryassign to guest\"" + } + ], + "failedVisible": [] + }, + { + "run": "tracker-rx-3", + "task": "tracker", + "arm": "rx", + "n": 3, + "checks": 8, + "tests": 7, + "cyclesToGreen": 15, + "checkOk": true, + "built": true, + "visible": "4/4", + "hidden": "11/11", + "overlap": "2/2", + "failedOverlap": [], + "shippedDefect": false, + "failedHidden": [], + "failedVisible": [] + } +] diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/big/search-react-1/App.tsx new file mode 100644 index 0000000..601acd3 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-1/App.tsx @@ -0,0 +1,126 @@ +import { useRef, useState } from "react"; +import { fetchItem, search } from "./api.js"; +import type { Hit, Item } from "./api.js"; + +type Results = + | { kind: "none" } + | { kind: "short"; q: string } + | { kind: "hits"; q: string; hits: Hit[] } + | { kind: "error"; message: string }; + +type Detail = + | { kind: "none" } + | { kind: "loading"; id: number } + | { kind: "item"; item: Item } + | { kind: "missing"; id: number } + | { kind: "error"; id: number; message: string }; + +const errMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [query, setQuery] = useState(""); + const [results, setResults] = useState<Results>({ kind: "none" }); + const [searching, setSearching] = useState(false); + const [selected, setSelected] = useState<number | null>(null); + const [detail, setDetail] = useState<Detail>({ kind: "none" }); + const searchSeq = useRef(0); + const detailSeq = useRef(0); + + function onQuery(q: string) { + setQuery(q); + const seq = ++searchSeq.current; + if (q.length === 0) { + setSearching(false); + setResults({ kind: "none" }); + return; + } + if (q.length === 1) { + setSearching(false); + setResults({ kind: "short", q }); + return; + } + setSearching(true); + search(q).then( + hits => { + if (seq !== searchSeq.current) return; + setSearching(false); + setResults({ kind: "hits", q, hits }); + }, + (e: unknown) => { + if (seq !== searchSeq.current) return; + setSearching(false); + setResults({ kind: "error", message: errMessage(e) }); + }, + ); + } + + function load(id: number) { + const seq = ++detailSeq.current; + setDetail({ kind: "loading", id }); + fetchItem(id).then( + item => { + if (seq !== detailSeq.current) return; + setDetail(item ? { kind: "item", item } : { kind: "missing", id }); + }, + (e: unknown) => { + if (seq !== detailSeq.current) return; + setDetail({ kind: "error", id, message: errMessage(e) }); + }, + ); + } + + function select(id: number) { + if (id === selected) return; + setSelected(id); + load(id); + } + + return ( + <main> + <input + data-testid="query" + value={query} + onChange={e => onQuery(e.target.value)} + placeholder="search" + /> + <section data-testid="results"> + {searching && <p>searching…</p>} + {results.kind === "short" && <p>too short: {results.q}</p>} + {results.kind === "error" && <p>search failed: {results.message}</p>} + {results.kind === "hits" && ( + <> + <p> + {results.hits.length} results for {results.q} + </p> + <ul> + {results.hits.map(hit => ( + <li key={hit.id} data-testid="hit" onClick={() => select(hit.id)} style={{ cursor: "pointer" }}> + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + )} + </section> + <section data-testid="detail"> + {detail.kind === "none" && <p>pick a result</p>} + {detail.kind === "loading" && <p>loading…</p>} + {detail.kind === "item" && ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + )} + {detail.kind === "missing" && <p>item {detail.id} not found</p>} + {detail.kind === "error" && ( + <> + <p>failed: {detail.message}</p> + <button data-testid="retry" onClick={() => load(detail.id)}> + retry + </button> + </> + )} + </section> + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-react-1/runlog.jsonl new file mode 100644 index 0000000..7161760 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:34:55.453Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:35:01.201Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/big/search-react-2/App.tsx new file mode 100644 index 0000000..7632a8a --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-2/App.tsx @@ -0,0 +1,122 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; +import { search, fetchItem } from "./api.js"; +import type { Hit } from "./api.js"; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + staleTime: 0, + gcTime: 0, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }, + }, +}); + +type Results = { q: string; hits: Hit[] }; + +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function ResultsArea({ + q, + selected, + onSelect, +}: { + q: string; + selected: number | null; + onSelect: (id: number) => void; +}) { + const enabled = q.length >= 2; + const query = useQuery({ + queryKey: ["search", q], + queryFn: async (): Promise<Results> => ({ q, hits: await search(q) }), + enabled, + }); + const [shown, setShown] = useState<Results | null>(null); + + const fresh = enabled && !query.isFetching && query.isSuccess ? query.data : null; + const failed = enabled && !query.isFetching && query.isError; + if (fresh && fresh !== shown) setShown(fresh); + if ((!enabled || failed) && shown !== null) setShown(null); + + let body; + if (q.length === 0) body = null; + else if (!enabled) body = <p>too short: {q}</p>; + else if (failed) body = <p>search failed: {message(query.error)}</p>; + else { + const list = fresh ?? shown; + body = ( + <> + {query.isFetching && <p>searching…</p>} + {list && ( + <> + <p> + {list.hits.length} results for {list.q} + </p> + <ul> + {list.hits.map((h) => ( + <li key={h.id} data-testid="hit" onClick={() => onSelect(h.id)} style={{ cursor: "pointer" }}> + {h.id === selected ? `> ${h.name}` : h.name} + </li> + ))} + </ul> + </> + )} + </> + ); + } + return <div data-testid="results">{body}</div>; +} + +function Detail({ id }: { id: number }) { + const query = useQuery({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + }); + let body; + if (query.isFetching || query.isPending) body = <p>loading…</p>; + else if (query.isError) + body = ( + <> + <p>failed: {message(query.error)}</p> + <button data-testid="retry" onClick={() => void query.refetch()}> + retry + </button> + </> + ); + else if (query.data === null || query.data === undefined) body = <p>item {id} not found</p>; + else + body = ( + <> + <h2>{query.data.name}</h2> + <p>{query.data.blurb}</p> + </> + ); + return <div data-testid="detail">{body}</div>; +} + +function SearchPage() { + const [q, setQ] = useState(""); + const [selected, setSelected] = useState<number | null>(null); + return ( + <main> + <input data-testid="query" value={q} onChange={(e) => setQ(e.target.value)} placeholder="Search" /> + <ResultsArea q={q} selected={selected} onSelect={setSelected} /> + {selected === null ? ( + <div data-testid="detail">pick a result</div> + ) : ( + <Detail key={selected} id={selected} /> + )} + </main> + ); +} + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <SearchPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-react-2/runlog.jsonl new file mode 100644 index 0000000..4bb1258 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:36:09.218Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:20.844Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/big/search-react-3/App.tsx new file mode 100644 index 0000000..9191b90 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-3/App.tsx @@ -0,0 +1,112 @@ +import { useEffect, useState } from "react"; +import type React from "react"; +import { search, fetchItem } from "./api.js"; +import type { Hit, Item } from "./api.js"; + +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +type SearchResult = { q: string; hits: Hit[] } | { q: string; error: string }; + +type DetailResult = + | { id: number; attempt: number; item: Item | null } + | { id: number; attempt: number; error: string }; + +export default function App() { + const [query, setQuery] = useState(""); + // last settled search answer (only for queries that were asked) + const [settled, setSettled] = useState<SearchResult | null>(null); + const [selected, setSelected] = useState<number | null>(null); + const [attempt, setAttempt] = useState(0); + const [detail, setDetail] = useState<DetailResult | null>(null); + + const asks = query.length >= 2; + + useEffect(() => { + if (!asks) return; + let active = true; + search(query).then( + hits => { if (active) setSettled({ q: query, hits }); }, + e => { if (active) setSettled({ q: query, error: message(e) }); }, + ); + return () => { active = false; }; + }, [query, asks]); + + useEffect(() => { + if (selected === null) return; + let active = true; + fetchItem(selected).then( + item => { if (active) setDetail({ id: selected, attempt, item }); }, + e => { if (active) setDetail({ id: selected, attempt, error: message(e) }); }, + ); + return () => { active = false; }; + }, [selected, attempt]); + + function onType(q: string) { + setQuery(q); + // results no longer on screen once the query is too short + if (q.length < 2) setSettled(null); + } + + function select(id: number) { + if (id === selected) return; + setSelected(id); + setAttempt(a => a + 1); + } + + let results: React.ReactNode = null; + if (query.length === 1) { + results = <p>too short: {query}</p>; + } else if (asks) { + const current = settled !== null && settled.q === query; + const shown = settled !== null && "hits" in settled ? settled : null; + results = ( + <> + {!current && <p>searching…</p>} + {current && settled !== null && "error" in settled && <p>search failed: {settled.error}</p>} + {shown && ( + <> + <p>{shown.hits.length} results for {shown.q}</p> + <ul> + {shown.hits.map(hit => ( + <li key={hit.id} data-testid="hit" onClick={() => select(hit.id)} style={{ cursor: "pointer" }}> + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + )} + </> + ); + } + + let pane: React.ReactNode; + if (selected === null) { + pane = <p>pick a result</p>; + } else if (detail === null || detail.id !== selected || detail.attempt !== attempt) { + pane = <p>loading…</p>; + } else if ("error" in detail) { + pane = ( + <> + <p>failed: {detail.error}</p> + <button data-testid="retry" onClick={() => setAttempt(a => a + 1)}>retry</button> + </> + ); + } else if (detail.item === null) { + pane = <p>item {detail.id} not found</p>; + } else { + pane = ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + ); + } + + return ( + <main> + <input data-testid="query" value={query} onChange={e => onType(e.target.value)} placeholder="search" /> + <section data-testid="results">{results}</section> + <section data-testid="detail">{pane}</section> + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-react-3/runlog.jsonl new file mode 100644 index 0000000..21a81a4 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:36:43.287Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:48.862Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/big/search-rx-1/main.rx new file mode 100644 index 0000000..a5c6260 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-1/main.rx @@ -0,0 +1,55 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +error NotFound(id: number) => `item ${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source query: string := "" + source selected: number | null := null + + res = len(query) < 2 ? null : { q: query, hits: await search(query) } + item = selected == null ? null : found(await fetchItem(selected), selected) + + <main> + <input data-testid="query" value={query} onInput={(e) => { query := e.target.value }} /> + <div data-testid="results"> + {if len(query) == 1 { + <p>too short: {query}</p> + }} + {if len(query) >= 2 { + {errored fallback(e) { <p>search failed: {e.message}</p> } { + {loading fallback { <p>searching…</p> } { + {if pending? res { <p>searching…</p> }} + {if res != null { + <p>{len(res.hits)} results for {res.q}</p> + <ul> + {for h in res.hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{h.id == selected ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + </div> + <div data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + {loading on(selected) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {if item != null { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + }} + </div> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-rx-1/runlog.jsonl new file mode 100644 index 0000000..dbe75c3 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-1/runlog.jsonl @@ -0,0 +1,3 @@ +{"at":"2026-10-27T20:34:52.439Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:35:00.640Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:35:30.140Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/big/search-rx-2/main.rx new file mode 100644 index 0000000..0d94dae --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-2/main.rx @@ -0,0 +1,51 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +error ItemNotFound(id: number) => `item ${id} not found` + +fn found(it, id) => it == null ? throw ItemNotFound(id) : it + +export default component App() { + source q: string := "" + source selected: number | null := null + + hits = len(q) < 2 ? [] : await search(q) + item = selected == null ? null : found(await fetchItem(selected), selected) + + <div> + <input data-testid="query" value={q} onInput={(e) => { q := e.target.value }} /> + <div data-testid="results"> + {if len(latest q) == 1 { + <p>too short: {latest q}</p> + } else { + {if len(latest q) >= 2 { + {errored fallback(e) { <p>search failed: {e.message}</p> } { + {loading fallback { <p>searching…</p> } { + {if pending? hits { <p>searching…</p> }} + <p>{len(hits)} results for {q}</p> + <ul> + {for h in hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{selected == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + </div> + <div data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + {loading on(selected) fallback { <p>loading…</p> } { + {errored fallback(e) { <p>failed: {e.message}</p> <button data-testid="retry" onClick={() => { refresh item }}>retry</button> } { + {errored fallback(e: ItemNotFound) { <p>{e.message}</p> } { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + }} + </div> + </div> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-rx-2/runlog.jsonl new file mode 100644 index 0000000..028efa1 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-2/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:36:19.421Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:29.650Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:36:49.123Z","cmd":"test","ok":true,"build":true,"passed":8,"total":8} +{"at":"2026-10-27T20:36:57.429Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/big/search-rx-3/main.rx new file mode 100644 index 0000000..38e2603 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-3/main.rx @@ -0,0 +1,64 @@ +import pure { search, fetchItem } from "./api" +import pure { len } from "./prelude" + +export error NotFound(id: number) => `item ${id} not found` +fn found(it, id) => it == null ? throw NotFound(id) : it + +export default component App() { + source q: string := "" + source sel: number := 0 + + typed = latest q + picked = latest sel + res = len(q) < 2 ? null : ({ q: q, hits: await search(q), err: "" } catch (e) => ({ q: q, hits: [], err: e.message })) + item = sel == 0 ? null : found(await fetchItem(sel), sel) + + <main> + <input data-testid="query" onInput={(e) => { q := e.target.value }} /> + <div data-testid="results"> + {if typed == "" { + } else { + {if len(typed) < 2 { + <p>too short: {typed}</p> + } else { + {loading fallback { <p>searching…</p> } { + {if res == null { + <p>searching…</p> + } else { + {if pending? res { <p>searching…</p> }} + {if res.err != "" { + <p>search failed: {res.err}</p> + } else { + <p>{len(res.hits)} results for {res.q}</p> + <ul> + {for h in res.hits { + <li data-testid="hit" onClick={() => { sel := h.id }}>{picked == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + }} + </div> + <div data-testid="detail"> + {if picked == 0 { + <p>pick a result</p> + } else { + {loading on(sel) fallback { <p>loading…</p> } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {errored fallback(e: NotFound) { <p>{e.message}</p> } { + {if item != null { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + }} + }} + </div> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/search-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/search-rx-3/runlog.jsonl new file mode 100644 index 0000000..bafbf91 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/search-rx-3/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:39:50.319Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:39:56.653Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:15.291Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} +{"at":"2026-10-27T20:40:34.559Z","cmd":"test","ok":true,"build":true,"passed":9,"total":9} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/big/settings-react-1/App.tsx new file mode 100644 index 0000000..e39c406 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-1/App.tsx @@ -0,0 +1,204 @@ +import { useCallback, useEffect, useReducer, useRef } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings } from "./api.js"; + +type Pending = { seq: number; settings: Settings }; + +type State = { + load: { status: "loading" } | { status: "error"; message: string } | { status: "ready" }; + confirmed: Settings | null; + confirmedSeq: number; + draft: Settings | null; + pending: Pending[]; + lastSeq: number; + saveError: string | null; +}; + +type Action = + | { type: "loadStart" } + | { type: "loadOk"; settings: Settings } + | { type: "loadFail"; message: string } + | { type: "draft"; patch: Partial<Settings> } + | { type: "saveStart"; seq: number; settings: Settings } + | { type: "saveOk"; seq: number; settings: Settings } + | { type: "saveFail"; seq: number; message: string }; + +const initial: State = { + load: { status: "loading" }, + confirmed: null, + confirmedSeq: -1, + draft: null, + pending: [], + lastSeq: -1, + saveError: null, +}; + +function reducer(state: State, action: Action): State { + switch (action.type) { + case "loadStart": + return { ...state, load: { status: "loading" } }; + case "loadOk": + if (state.load.status === "ready") return state; + return { + ...state, + load: { status: "ready" }, + confirmed: action.settings, + draft: state.draft ?? action.settings, + }; + case "loadFail": + if (state.load.status === "ready") return state; + return { ...state, load: { status: "error", message: action.message } }; + case "draft": + if (!state.draft) return state; + return { ...state, draft: { ...state.draft, ...action.patch } }; + case "saveStart": + return { + ...state, + pending: [...state.pending, { seq: action.seq, settings: action.settings }], + lastSeq: action.seq, + saveError: null, + }; + case "saveOk": { + const pending = state.pending.filter(p => p.seq !== action.seq); + if (action.seq > state.confirmedSeq) { + return { ...state, pending, confirmed: action.settings, confirmedSeq: action.seq }; + } + return { ...state, pending }; + } + case "saveFail": { + const pending = state.pending.filter(p => p.seq !== action.seq); + const saveError = action.seq === state.lastSeq ? `Save failed: ${action.message}` : state.saveError; + return { ...state, pending, saveError }; + } + } +} + +const MAX = 20; + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX) return `Display name must be at most ${MAX} characters, not ${name.length}`; + return null; +} + +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); +const line = (s: Settings) => `${s.name} · ${s.theme} · ${s.density}`; + +export default function App() { + const [state, dispatch] = useReducer(reducer, initial); + const started = useRef(false); + const seq = useRef(0); + + const load = useCallback(() => { + dispatch({ type: "loadStart" }); + loadSettings().then( + settings => dispatch({ type: "loadOk", settings }), + (e: unknown) => dispatch({ type: "loadFail", message: message(e) }), + ); + }, []); + + useEffect(() => { + if (started.current) return; + started.current = true; + load(); + }, [load]); + + const save = () => { + const draft = state.draft; + if (!draft || nameError(draft.name)) return; + const n = seq.current++; + const settings = { ...draft }; + dispatch({ type: "saveStart", seq: n, settings }); + saveSettings(settings).then( + saved => dispatch({ type: "saveOk", seq: n, settings: saved }), + (e: unknown) => dispatch({ type: "saveFail", seq: n, message: message(e) }), + ); + }; + + if (state.load.status !== "ready" || !state.confirmed || !state.draft) { + return ( + <main> + {state.load.status === "error" ? ( + <> + <p data-testid="load">Could not load settings: {state.load.message}</p> + <button data-testid="retry-load" onClick={load}> + Retry + </button> + </> + ) : ( + <p data-testid="load">Loading settings…</p> + )} + </main> + ); + } + + const { draft, confirmed } = state; + const showing = state.pending + .filter(p => p.seq > state.confirmedSeq) + .reduce<Pending | null>((a, p) => (!a || p.seq > a.seq ? p : a), null); + const error = nameError(draft.name); + + return ( + <main> + <p data-testid="saved"> + {showing ? `Saved: ${line(showing.settings)} (saving…)` : `Saved: ${line(confirmed)}`} + </p> + <form + onSubmit={e => { + e.preventDefault(); + save(); + }} + > + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={e => dispatch({ type: "draft", patch: { name: e.target.value } })} + /> + </label>{" "} + <span data-testid="name-count"> + {draft.name.length}/{MAX} + </span> + {error && <p data-testid="name-error">{error}</p>} + <div> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => dispatch({ type: "draft", patch: { theme: "light" } })} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => dispatch({ type: "draft", patch: { theme: "dark" } })} + > + Dark + </button> + <button + type="button" + data-testid="density-toggle" + onClick={() => + dispatch({ + type: "draft", + patch: { density: draft.density === "compact" ? "comfortable" : "compact" }, + }) + } + > + Density: {draft.density} + </button> + </div> + <p data-testid="preview"> + Preview: {draft.theme} / {draft.density} + </p> + <button type="submit" data-testid="save"> + Save + </button> + {state.saveError && <p data-testid="save-error">{state.saveError}</p>} + </form> + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-react-1/runlog.jsonl new file mode 100644 index 0000000..08a3bc5 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:35:08.655Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:35:13.662Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/big/settings-react-2/App.tsx new file mode 100644 index 0000000..1978e83 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-2/App.tsx @@ -0,0 +1,161 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings } from "./api.js"; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + refetchOnMount: false, + staleTime: Infinity, + gcTime: Infinity, + }, + }, +}); + +const MAX = 20; + +function message(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function line(s: Settings): string { + return `Saved: ${s.name} · ${s.theme} · ${s.density}`; +} + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX) return `Display name must be at most ${MAX} characters, not ${name.length}`; + return null; +} + +function Editor({ initial }: { initial: Settings }) { + const [draft, setDraft] = useState<Settings>(initial); + const [saved, setSaved] = useState<Settings>(initial); + const [pending, setPending] = useState<Settings | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + const seq = useRef(0); + const savedSeq = useRef(0); + + const error = nameError(draft.name); + + function save() { + if (nameError(draft.name)) return; + const values: Settings = { ...draft }; + const id = ++seq.current; + setPending(values); + setSaveError(null); + saveSettings(values).then( + result => { + if (id > savedSeq.current) { + savedSeq.current = id; + setSaved(result); + } + if (id === seq.current) setPending(null); + }, + (e: unknown) => { + if (id === seq.current) { + setPending(null); + setSaveError(`Save failed: ${message(e)}`); + } + }, + ); + } + + return ( + <div> + <p data-testid="saved">{pending ? `${line(pending)} (saving…)` : line(saved)}</p> + <form + onSubmit={e => { + e.preventDefault(); + save(); + }} + > + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={e => setDraft(d => ({ ...d, name: e.target.value }))} + /> + </label>{" "} + <span data-testid="name-count"> + {draft.name.length}/{MAX} + </span> + {error && ( + <p data-testid="name-error" role="alert"> + {error} + </p> + )} + <div> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => setDraft(d => ({ ...d, theme: "light" }))} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => setDraft(d => ({ ...d, theme: "dark" }))} + > + Dark + </button> + <button + type="button" + data-testid="density-toggle" + onClick={() => + setDraft(d => ({ ...d, density: d.density === "comfortable" ? "compact" : "comfortable" })) + } + > + Toggle density + </button> + </div> + <p data-testid="preview"> + Preview: {draft.theme} / {draft.density} + </p> + <button type="submit" data-testid="save"> + Save + </button> + {saveError && ( + <p data-testid="save-error" role="alert"> + {saveError} + </p> + )} + </form> + </div> + ); +} + +function SettingsPage() { + const query = useQuery({ queryKey: ["settings"], queryFn: () => loadSettings() }); + + if (query.data) return <Editor initial={query.data} />; + + if (query.isError && !query.isFetching) { + return ( + <div> + <p data-testid="load">Could not load settings: {message(query.error)}</p> + <button type="button" data-testid="retry-load" onClick={() => void query.refetch()}> + Retry + </button> + </div> + ); + } + + return <p data-testid="load">Loading settings…</p>; +} + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <SettingsPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-react-2/runlog.jsonl new file mode 100644 index 0000000..ebc8c9d --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:35:59.002Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:10.584Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/big/settings-react-3/App.tsx new file mode 100644 index 0000000..bfa7c8d --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-3/App.tsx @@ -0,0 +1,109 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; +import { loadSettings, saveSettings, type Settings } from "./api"; + +const MAX = 20; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, staleTime: Infinity, gcTime: Infinity, refetchOnWindowFocus: false, refetchOnReconnect: false }, + }, +}); + +const fmt = (s: Settings) => `${s.name} · ${s.theme} · ${s.density}`; +const errMsg = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > MAX) return `Display name must be at most ${MAX} characters, not ${name.length}`; + return null; +} + +function SettingsPage() { + const query = useQuery({ queryKey: ["settings"], queryFn: () => loadSettings() }); + + if (query.data) return <Editor loaded={query.data} />; + + if (query.isError && !query.isFetching) { + return ( + <div> + <p data-testid="load">Could not load settings: {errMsg(query.error)}</p> + <button data-testid="retry-load" onClick={() => void query.refetch()}>Retry</button> + </div> + ); + } + return <p data-testid="load">Loading settings…</p>; +} + +type Confirmed = { seq: number; value: Settings }; +type Pending = { seq: number; value: Settings }; + +function Editor({ loaded }: { loaded: Settings }) { + const [draft, setDraft] = useState<Settings>(loaded); + const [confirmed, setConfirmed] = useState<Confirmed>({ seq: -1, value: loaded }); + const [pending, setPending] = useState<Pending | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + const seqRef = useRef(0); + + const error = nameError(draft.name); + + const save = () => { + if (error) return; + const seq = seqRef.current++; + const value = { ...draft }; + setPending({ seq, value }); + setSaveError(null); + saveSettings(value).then( + saved => { + setConfirmed(c => (seq > c.seq ? { seq, value: saved } : c)); + setPending(p => (p && p.seq === seq ? null : p)); + }, + (e: unknown) => { + // Only the latest save's failure is reported. + if (seq === seqRef.current - 1) setSaveError(`Save failed: ${errMsg(e)}`); + setPending(p => (p && p.seq === seq ? null : p)); + }, + ); + }; + + const shown = pending ? `${fmt(pending.value)} (saving…)` : fmt(confirmed.value); + + return ( + <div> + <p data-testid="saved">Saved: {shown}</p> + <form onSubmit={e => { e.preventDefault(); save(); }}> + <label> + Display name{" "} + <input + data-testid="name" + value={draft.name} + onChange={e => { const name = e.target.value; setDraft(d => ({ ...d, name })); }} + /> + </label> + <span data-testid="name-count">{draft.name.length}/{MAX}</span> + {error && <p data-testid="name-error">{error}</p>} + <div> + <button type="button" data-testid="theme-light" aria-pressed={draft.theme === "light"} + onClick={() => setDraft(d => ({ ...d, theme: "light" }))}>Light</button> + <button type="button" data-testid="theme-dark" aria-pressed={draft.theme === "dark"} + onClick={() => setDraft(d => ({ ...d, theme: "dark" }))}>Dark</button> + <button type="button" data-testid="density-toggle" + onClick={() => setDraft(d => ({ ...d, density: d.density === "compact" ? "comfortable" : "compact" }))}> + Density: {draft.density} + </button> + </div> + <p data-testid="preview">Preview: {draft.theme} / {draft.density}</p> + <button type="submit" data-testid="save">Save</button> + </form> + {saveError && <p data-testid="save-error">{saveError}</p>} + </div> + ); +} + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <SettingsPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-react-3/runlog.jsonl new file mode 100644 index 0000000..9495063 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:36:53.653Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:37:03.431Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/main.rx new file mode 100644 index 0000000..5718263 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/main.rx @@ -0,0 +1,70 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +type Settings = { name: string; theme: string; density: string } + +source settings: Settings = await loadSettings() +source saveError: string = "" + +fn line(s: Settings): string => `Saved: ${s.name} · ${s.theme} · ${s.density}` + +fn nameError(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +action save(next: Settings) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } +} + +component Form(props: { initial: Settings }) { + source name: string := peek(props.initial.name) + source theme: string := peek(props.initial.theme) + source density: string := peek(props.initial.density) + err = nameError(name) + <form onSubmit={(ev) => { ev.preventDefault() }}> + <label> + Display name{" "} + <input data-testid="name" value={name} onInput={(ev) => { name := ev.currentTarget.value }} /> + </label> + <span data-testid="name-count">{`${len(name)}/20`}</span> + {if err != "" { + <p data-testid="name-error">{err}</p> + }} + <div> + <button type="button" data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button type="button" data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button type="button" data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + <button type="button" data-testid="save" onClick={() => { + if (nameError(name) == "") { save({ name: name, theme: theme, density: density }) } + }}>Save</button> + </form> +} + +export default component App() { + <main> + {errored fallback(e) { + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { refresh settings }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{pending? settings ? `${line(settings)} (saving…)` : line(settings)}</p> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + <Form initial={settings} /> + }} + }} + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/runlog.jsonl new file mode 100644 index 0000000..3bc404c --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-1/runlog.jsonl @@ -0,0 +1,3 @@ +{"at":"2026-10-27T20:34:56.640Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:35:04.793Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:35:22.569Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/main.rx new file mode 100644 index 0000000..278f5be --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/main.rx @@ -0,0 +1,72 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +type Settings = { name: string; theme: string; density: string } + +fn nameError(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +source settings: Settings = await loadSettings() +source saveError: string = "" + +action save(next: Settings) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } +} + +component Form(props: { initial: Settings }) { + source name := peek(props.initial.name) + source theme := peek(props.initial.theme) + source density := peek(props.initial.density) + err = nameError(name) + + <div> + <label> + Display name{" "} + <input data-testid="name" value={name} onInput={(e) => { name := e.target.value }} /> + </label> + <span data-testid="name-count">{`${len(name)}/20`}</span> + {if err != "" { + <p data-testid="name-error">{err}</p> + }} + <div> + <button data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + <button data-testid="save" onClick={() => { + if (nameError(name) == "") { + save({ name: name, theme: theme, density: density }) + } + }}>Save</button> + </div> +} + +export default component App() { + <main> + <h1>Settings</h1> + {errored fallback(e) { + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { action { refresh settings } }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{`Saved: ${settings.name} · ${settings.theme} · ${settings.density}${pending? settings ? " (saving…)" : ""}`}</p> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + <Form initial={settings} /> + }} + }} + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/runlog.jsonl new file mode 100644 index 0000000..4719ed1 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-2/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T20:36:00.426Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/overlap-big/settings-rx-2/src/main.rx: line 14: expected '{' near 'source theme: string'"} +{"at":"2026-10-27T20:36:14.094Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:22.338Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:36:38.497Z","cmd":"test","ok":false,"build":true,"passed":5,"total":6} +{"at":"2026-10-27T20:36:51.853Z","cmd":"test","ok":true,"build":true,"passed":6,"total":6} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/main.rx new file mode 100644 index 0000000..80f9723 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/main.rx @@ -0,0 +1,54 @@ +import pure { len } from "./prelude" +import { Settings } from "./api" +import { settings, saveError, save, retryLoad } from "./store.rx" + +fn nameError(n: string): string => + len(n) == 0 ? "Display name is required" + : len(n) > 20 ? `Display name must be at most 20 characters, not ${len(n)}` + : "" + +component SettingsForm(props: { initial: Settings }) { + source name: string := peek(props.initial.name) + source theme: string := peek(props.initial.theme) + source density: string := peek(props.initial.density) + error = nameError(name) + <form onSubmit={(ev) => { ev.preventDefault() }}> + <label> + Display name{" "} + <input data-testid="name" value={name} onInput={(ev) => { name := ev.currentTarget.value }} /> + </label> + <span data-testid="name-count">{len(name)}/20</span> + {if error != "" { + <p data-testid="name-error">{error}</p> + }} + <div> + <button type="button" data-testid="theme-light" onClick={() => { theme := "light" }}>Light</button> + <button type="button" data-testid="theme-dark" onClick={() => { theme := "dark" }}>Dark</button> + <button type="button" data-testid="density-toggle" onClick={() => { density := density == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">Preview: {theme} / {density}</p> + <button type="button" data-testid="save" onClick={() => { + if (nameError(name) == "") { + save({ name: name, theme: theme, density: density }) + } + }}>Save</button> + </form> +} + +export default component App() { + <main> + <h1>Settings</h1> + {errored fallback(e) { + <p data-testid="load">Could not load settings: {e.message}</p> + <button data-testid="retry-load" onClick={() => { retryLoad() }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">Saved: {settings.name} · {settings.theme} · {settings.density}{pending? settings ? " (saving…)" : ""}</p> + {if saveError != "" { + <p data-testid="save-error">Save failed: {saveError}</p> + }} + <SettingsForm initial={settings} /> + }} + }} + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/runlog.jsonl new file mode 100644 index 0000000..cac8de4 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:39:18.964Z","cmd":"check","ok":false,"out":"/home/claude/runs/overlap-big/settings-rx-3/src/store.rx:8: error OPTIMISTIC_NO_QUESTION [AC-3] saveError is declared with := — a guess needs a question to confirm it (source saveError = …, ST-5c)"} +{"at":"2026-10-27T20:39:21.687Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:39:28.192Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:39:48.785Z","cmd":"test","ok":true,"build":true,"passed":6,"total":6} diff --git a/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/store.rx b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/store.rx new file mode 100644 index 0000000..af46fc2 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/settings-rx-3/store.rx @@ -0,0 +1,25 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import { Settings } from "./api" + +export source settings: Settings = await loadSettings() +export source saveError: string = "" + +export action save(next: Settings) { + optimistic settings := next + optimistic saveError := "" + affects settings + try { + let answer = await saveSettings(next) + settings := answer + } catch (e) { + saveError := e.message + } +} + +export action retryLoad() { + try { + refresh settings + } catch (e) { + } +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/App.tsx new file mode 100644 index 0000000..f8d3244 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/App.tsx @@ -0,0 +1,175 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type Pending = { seq: number; user: string }; +type AssignError = { issueId: number; message: string }; + +const errorMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + // optimistic assignments still in the air, per issue, oldest first + const [pending, setPending] = useState<Record<number, Pending[]>>({}); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seqRef = useRef(0); + const confirmedSeqRef = useRef<Record<number, number>>({}); + + const list = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + const detail = useQuery({ + queryKey: ["issue", selected], + queryFn: async () => { + const id = selected as number; + const d = await fetchIssue(id); + if (d === null) throw new Error(`issue #${id} not found`); + return d; + }, + enabled: selected !== null, + }); + + const assigneeOf = (issue: Issue) => { + const p = pending[issue.id]; + if (p && p.length > 0) return p[p.length - 1].user; + return issue.assignee; + }; + + const select = (id: number) => { + setSelected(id); + setAssignError(null); + }; + + const doAssign = async (id: number, who: string) => { + const seq = ++seqRef.current; + setAssignError(null); + setPending(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { seq, user: who }] })); + const done = () => + setPending(prev => { + const rest = (prev[id] ?? []).filter(p => p.seq !== seq); + const next = { ...prev }; + if (rest.length) next[id] = rest; + else delete next[id]; + return next; + }); + try { + const res = await assign(id, who); + if ((confirmedSeqRef.current[id] ?? 0) < seq) { + confirmedSeqRef.current[id] = seq; + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === id ? { ...i, assignee: res.assignee } : i)), + ); + } + done(); + } catch (e) { + done(); + // only report the failure if no later assignment has started since + if (seqRef.current === seq) setAssignError({ issueId: id, message: errorMessage(e) }); + } + }; + + let listContent; + if (list.isPending) listContent = <p>loading {filter} issues</p>; + else if (list.isError) listContent = <p>list error: {errorMessage(list.error)}</p>; + else + listContent = ( + <ul> + {list.data.map(issue => ( + <li key={issue.id}> + <button + type="button" + data-testid="issue" + aria-pressed={issue.id === selected} + onClick={() => select(issue.id)} + > + #{issue.id} {issue.title} ({assigneeOf(issue) ?? "unassigned"}) + </button> + </li> + ))} + </ul> + ); + + let detailContent; + if (selected === null) detailContent = <p>select an issue</p>; + else { + let body; + if (detail.isFetching || detail.isPending) body = <p>loading #{selected}</p>; + else if (detail.isError) body = <p>detail error: {errorMessage(detail.error)}</p>; + else + body = ( + <> + <h2> + #{detail.data.id} {detail.data.title} + </h2> + <p>{detail.data.body}</p> + </> + ); + const id = selected; + detailContent = ( + <> + {body} + <button type="button" data-testid="retry" onClick={() => void detail.refetch()}> + retry + </button> + <button type="button" data-testid="assign" onClick={() => void doAssign(id, user)}> + assign to {user} + </button> + {assignError && assignError.issueId === selected && ( + <p data-testid="assign-error">{assignError.message}</p> + )} + </> + ); + } + + return ( + <div style={{ display: "flex", gap: "2rem" }}> + <section> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + <section data-testid="detail">{detailContent}</section> + </div> + ); +} + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false, refetchOnReconnect: false }, + }, +}); + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/runlog.jsonl new file mode 100644 index 0000000..ff6ab92 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:35:27.614Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:35:33.445Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/App.tsx new file mode 100644 index 0000000..5af44b5 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/App.tsx @@ -0,0 +1,160 @@ +import { useEffect, useState } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; +type ListReq = { filter: Filter }; +type ListRes = + | { req: ListReq; startedAt: number; ok: true; data: Issue[] } + | { req: ListReq; startedAt: number; ok: false; error: string }; +type SelReq = { id: number }; +type DetRes = + | { req: SelReq; ok: true; data: IssueDetail | null } + | { req: SelReq; ok: false; error: string }; +type Pending = { seq: number; id: number; user: string }; +type Confirmed = Record<number, { assignee: string; at: number }>; +type AssignState = { seq: number; id: number; error: string | null } | null; + +// Logical clock used to tell whether a list response predates a confirmed assignment. +let clock = 0; +let assignSeq = 0; + +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [user, setUser] = useState("guest"); + const [listReq, setListReq] = useState<ListReq>({ filter: "open" }); + const [listRes, setListRes] = useState<ListRes | null>(null); + const [sel, setSel] = useState<SelReq | null>(null); + const [detRes, setDetRes] = useState<DetRes | null>(null); + const [pending, setPending] = useState<Pending[]>([]); + const [confirmed, setConfirmed] = useState<Confirmed>({}); + const [assignState, setAssignState] = useState<AssignState>(null); + + useEffect(() => { + let cancelled = false; + const startedAt = ++clock; + fetchIssues(listReq.filter).then( + data => { if (!cancelled) setListRes({ req: listReq, startedAt, ok: true, data }); }, + (e: unknown) => { if (!cancelled) setListRes({ req: listReq, startedAt, ok: false, error: message(e) }); }, + ); + return () => { cancelled = true; }; + }, [listReq]); + + useEffect(() => { + if (!sel) return; + let cancelled = false; + fetchIssue(sel.id).then( + data => { if (!cancelled) setDetRes({ req: sel, ok: true, data }); }, + (e: unknown) => { if (!cancelled) setDetRes({ req: sel, ok: false, error: message(e) }); }, + ); + return () => { cancelled = true; }; + }, [sel]); + + const chooseFilter = (filter: Filter) => { + if (filter !== listReq.filter) setListReq({ filter }); + }; + + const select = (id: number) => { + if (sel && sel.id === id) return; + setSel({ id }); + setAssignState(null); + }; + + const retry = () => { + if (sel) setSel({ id: sel.id }); + }; + + const doAssign = () => { + if (!sel) return; + const id = sel.id; + const seq = ++assignSeq; + const who = user; + setPending(p => [...p, { seq, id, user: who }]); + setAssignState({ seq, id, error: null }); + assign(id, who).then( + res => { + const at = ++clock; + setConfirmed(c => ({ ...c, [id]: { assignee: res.assignee, at } })); + setPending(p => p.filter(x => x.seq !== seq)); + }, + (e: unknown) => { + setPending(p => p.filter(x => x.seq !== seq)); + setAssignState(s => (s && s.seq === seq ? { ...s, error: message(e) } : s)); + }, + ); + }; + + const assigneeOf = (row: Issue, startedAt: number): string | null => { + let latest: Pending | null = null; + for (const p of pending) if (p.id === row.id && (!latest || p.seq > latest.seq)) latest = p; + if (latest) return latest.user; + const c = confirmed[row.id]; + if (c && c.at > startedAt) return c.assignee; + return row.assignee; + }; + + const filter = listReq.filter; + let list: React.ReactNode; + if (!listRes || listRes.req !== listReq) list = <p>loading {filter} issues</p>; + else if (!listRes.ok) list = <p>list error: {listRes.error}</p>; + else + list = ( + <ul> + {listRes.data.map(row => { + const a = assigneeOf(row, listRes.startedAt); + return ( + <li + key={row.id} + data-testid="issue" + onClick={() => select(row.id)} + style={{ cursor: "pointer", fontWeight: sel?.id === row.id ? "bold" : undefined }} + > + {`#${row.id} ${row.title} (${a ?? "unassigned"})`} + </li> + ); + })} + </ul> + ); + + let detail: React.ReactNode; + if (!sel) detail = <p>select an issue</p>; + else { + let body: React.ReactNode; + if (!detRes || detRes.req !== sel) body = <p>loading #{sel.id}</p>; + else if (!detRes.ok) body = <p>detail error: {detRes.error}</p>; + else if (!detRes.data) body = <p>detail error: issue #{sel.id} not found</p>; + else + body = ( + <> + <h2>{`#${detRes.data.id} ${detRes.data.title}`}</h2> + <p>{detRes.data.body}</p> + </> + ); + const assignError = assignState && assignState.id === sel.id ? assignState.error : null; + detail = ( + <> + {body} + <button data-testid="retry" onClick={retry}>retry</button>{" "} + <button data-testid="assign" onClick={doAssign}>{`assign to ${user}`}</button> + {assignError !== null && <p data-testid="assign-error">{assignError}</p>} + </> + ); + } + + return ( + <div> + <label> + user <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button data-testid="filter-open" onClick={() => chooseFilter("open")}>open</button>{" "} + <button data-testid="filter-closed" onClick={() => chooseFilter("closed")}>closed</button> + </div> + <div style={{ display: "flex", gap: "2em" }}> + <section data-testid="list">{list}</section> + <section data-testid="detail">{detail}</section> + </div> + </div> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/runlog.jsonl new file mode 100644 index 0000000..ea9202e --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:37:10.750Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:37:18.860Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/App.tsx new file mode 100644 index 0000000..ccb7b18 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/App.tsx @@ -0,0 +1,172 @@ +import { useEffect, useRef, useState } from "react"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; + +type ListResult = + | { filter: Filter; nonce: number; ok: true; issues: Issue[] } + | { filter: Filter; nonce: number; ok: false; message: string }; + +type DetailResult = + | { id: number; nonce: number; ok: true; issue: IssueDetail | null } + | { id: number; nonce: number; ok: false; message: string }; + +type Pending = { token: number; assignee: string; confirmed: boolean }; + +const errMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [listNonce, setListNonce] = useState(0); + const [list, setList] = useState<ListResult | null>(null); + + const [selectedId, setSelectedId] = useState<number | null>(null); + const [detailNonce, setDetailNonce] = useState(0); + const [detail, setDetail] = useState<DetailResult | null>(null); + + // Per-issue assignments, oldest first; the latest still standing wins. + const [assignments, setAssignments] = useState<Record<number, Pending[]>>({}); + const [assignError, setAssignError] = useState<{ id: number; token: number; message: string } | null>(null); + const tokenRef = useRef(0); + const latestAssignRef = useRef(0); + const selectedRef = useRef<number | null>(null); + + useEffect(() => { + let live = true; + fetchIssues(filter).then( + issues => { if (live) setList({ filter, nonce: listNonce, ok: true, issues }); }, + (e: unknown) => { if (live) setList({ filter, nonce: listNonce, ok: false, message: errMessage(e) }); }, + ); + return () => { live = false; }; + }, [filter, listNonce]); + + useEffect(() => { + if (selectedId === null) return; + let live = true; + const id = selectedId; + fetchIssue(id).then( + issue => { if (live) setDetail({ id, nonce: detailNonce, ok: true, issue }); }, + (e: unknown) => { if (live) setDetail({ id, nonce: detailNonce, ok: false, message: errMessage(e) }); }, + ); + return () => { live = false; }; + }, [selectedId, detailNonce]); + + const chooseFilter = (f: Filter) => { + if (f === filter) return; + setFilter(f); + setListNonce(n => n + 1); + }; + + const select = (id: number) => { + if (id === selectedId) return; + selectedRef.current = id; + setSelectedId(id); + setDetailNonce(n => n + 1); + setAssignError(null); + }; + + const retry = () => { + setDetailNonce(n => n + 1); + }; + + const doAssign = () => { + if (selectedId === null) return; + const id = selectedId; + const who = user; + const token = ++tokenRef.current; + latestAssignRef.current = token; + setAssignError(null); + setAssignments(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { token, assignee: who, confirmed: false }] })); + assign(id, who).then( + res => { + setAssignments(prev => { + const entries = prev[id] ?? []; + // A confirmed assignment supersedes older ones. + const idx = entries.findIndex(p => p.token === token); + if (idx < 0) return prev; + const next = entries.slice(idx).map(p => (p.token === token ? { ...p, assignee: res.assignee, confirmed: true } : p)); + return { ...prev, [id]: next }; + }); + }, + (e: unknown) => { + setAssignments(prev => { + const entries = (prev[id] ?? []).filter(p => p.token !== token); + return { ...prev, [id]: entries }; + }); + if (latestAssignRef.current === token && selectedRef.current === id) { + setAssignError({ id, token, message: errMessage(e) }); + } + }, + ); + }; + + const shownAssignee = (issue: Issue): string | null => { + const entries = assignments[issue.id]; + if (entries && entries.length) return entries[entries.length - 1].assignee; + return issue.assignee; + }; + + const listCurrent = list !== null && list.filter === filter && list.nonce === listNonce ? list : null; + + let listBody; + if (!listCurrent) listBody = <p>loading {filter} issues</p>; + else if (!listCurrent.ok) listBody = <p>list error: {listCurrent.message}</p>; + else + listBody = ( + <ul> + {listCurrent.issues.map(issue => ( + <li + key={issue.id} + data-testid="issue" + onClick={() => select(issue.id)} + style={{ cursor: "pointer", fontWeight: issue.id === selectedId ? "bold" : undefined }} + > + {`#${issue.id} ${issue.title} (${shownAssignee(issue) ?? "unassigned"})`} + </li> + ))} + </ul> + ); + + let detailBody; + if (selectedId === null) detailBody = <p>select an issue</p>; + else { + const current = detail !== null && detail.id === selectedId && detail.nonce === detailNonce ? detail : null; + let content; + if (!current) content = <p>loading #{selectedId}</p>; + else if (!current.ok) content = <p>detail error: {current.message}</p>; + else if (current.issue === null) content = <p>detail error: issue #{selectedId} not found</p>; + else + content = ( + <div> + <h2>{`#${current.issue.id} ${current.issue.title}`}</h2> + <p>{current.issue.body}</p> + </div> + ); + detailBody = ( + <> + {content} + <button data-testid="retry" onClick={retry}>retry</button> + <button data-testid="assign" onClick={doAssign}>{`assign to ${user}`}</button> + {assignError && assignError.id === selectedId && <p data-testid="assign-error">{assignError.message}</p>} + </> + ); + } + + return ( + <div> + <label> + user <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button data-testid="filter-open" onClick={() => chooseFilter("open")}>open</button> + <button data-testid="filter-closed" onClick={() => chooseFilter("closed")}>closed</button> + </div> + <div style={{ display: "flex", gap: "2em" }}> + <section data-testid="list">{listBody}</section> + <section data-testid="detail">{detailBody}</section> + </div> + </div> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/runlog.jsonl new file mode 100644 index 0000000..5b9a16b --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:39:35.407Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:39:40.752Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/main.rx new file mode 100644 index 0000000..6f8e56b --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/main.rx @@ -0,0 +1,72 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { indexOf } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: string = "" + + action assignTo(id: number, who: string) { + optimistic assignError := "" + try { + let i = indexOf(issues, t => t.id == id) + optimistic issues[i].assignee := who + await assign(id, who) + refresh issues + } catch (e) { + assignError := e.message + } + } + + <main> + <div> + <label> + user{" "} + <input data-testid="user" value={user} onInput={e => { user := e.target.value }} /> + </label> + </div> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div data-testid="list"> + {loading on(filter) fallback { <p>loading {latest filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for it in issues { + <li data-testid="issue" onClick={() => { + selected := it.id + onFlush(() => { assignError := "" }) + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + </ul> + }} + }} + </div> + <div data-testid="detail"> + {if selected != null { + detail = found(await fetchIssue(selected), selected) + {loading on(selected) fallback { <p>loading #{latest selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + }} + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + } else { + <p>select an issue</p> + }} + </div> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl new file mode 100644 index 0000000..afa51e2 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T20:35:28.626Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:35:36.388Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:35:53.530Z","cmd":"test","ok":true,"build":true,"passed":7,"total":7} +{"at":"2026-10-27T20:36:18.645Z","cmd":"test","ok":false,"build":true,"passed":8,"total":10} +{"at":"2026-10-27T20:36:45.878Z","cmd":"test","ok":true,"build":true,"passed":6,"total":6} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/main.rx new file mode 100644 index 0000000..19a60c6 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/main.rx @@ -0,0 +1,73 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { indexOf } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +type AssignError = { id: number; message: string } + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selectedId: number | null := null + source assignError: AssignError | null := null + source issues key id = await fetchIssues(filter) + + detail = selectedId == null ? null : found(await fetchIssue(selectedId), selectedId) + + action assignTo(id: number, who: string) { + try { + let i = indexOf(issues, t => t.id == id) + optimistic issues[i].assignee := who + await assign(id, who) + refresh issues + } catch (e) { + assignError := { id: id, message: e.message } + } + } + + <main> + <label>user <input data-testid="user" value={user} onInput={(ev) => { user := ev.target.value }} /></label> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <ul data-testid="list"> + {loading on(filter) fallback { <li>loading {latest filter} issues</li> } { + {errored fallback(e) { <li>list error: {e.message}</li> } { + {for it in issues { + <li data-testid="issue" onClick={() => { + onFlush(() => { assignError := null }) + selectedId := it.id + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + }} + }} + </ul> + <section data-testid="detail"> + {if selectedId == null { + <p>select an issue</p> + } else { + {loading on(selectedId) fallback { <p>loading #{latest selectedId}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + {if detail { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + <button data-testid="assign" onClick={() => { + let id = selectedId + let who = user + onFlush(() => { assignError := null }) + if (id != null) { assignTo(id, who) } + }}>assign to {user}</button> + {if assignError != null && assignError.id == selectedId { + <p data-testid="assign-error">{assignError.message}</p> + }} + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl new file mode 100644 index 0000000..e9312f7 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-2/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:37:44.330Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:37:52.588Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:38:12.093Z","cmd":"test","ok":true,"build":true,"passed":8,"total":8} +{"at":"2026-10-27T20:38:34.337Z","cmd":"test","ok":true,"build":true,"passed":10,"total":10} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/main.rx new file mode 100644 index 0000000..c910ff9 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/main.rx @@ -0,0 +1,71 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" +import pure { indexOf } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: { id: number; message: string } | null := null + + detail = selected == null ? null : found(await fetchIssue(selected), selected) + shownError = assignError != null && assignError.id == selected ? assignError.message : "" + + action assignTo(id: number, who: string) { + try { + let i = indexOf(issues, t => t.id == id) + if (i >= 0) { + optimistic issues[i].assignee := who + } + await assign(id, who) + refresh issues + } catch (e) { + assignError := { id: id, message: e.message } + } + } + + <div> + <label>user <input data-testid="user" value={user} onInput={(ev) => { user := ev.target.value }} /></label> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div style="display: flex; gap: 2em"> + <ul data-testid="list"> + {loading on(filter) fallback { <li>loading {filter} issues</li> } { + {errored fallback(e) { <li>list error: {e.message}</li> } { + {for it in issues { + <li data-testid="issue" onClick={() => { + selected := it.id + onFlush(() => { assignError := null }) + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + }} + }} + </ul> + <div data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + {loading on(selected) fallback { <p>loading #{selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + <button data-testid="assign" onClick={() => { + onFlush(() => { assignError := null }) + assignTo(selected, user) + }}>assign to {user}</button> + {if shownError != "" { <p data-testid="assign-error">{shownError}</p> }} + }} + </div> + </div> + </div> +} diff --git a/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl new file mode 100644 index 0000000..6abc37e --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/big/tracker-rx-3/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:39:42.444Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:39:50.540Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:03.150Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:08.620Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/small/search-react-1/App.tsx new file mode 100644 index 0000000..00c675b --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-1/App.tsx @@ -0,0 +1,128 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, keepPreviousData, useQuery } from "@tanstack/react-query"; +import { search, fetchItem } from "./api.js"; + +const queries = new QueryClient(); + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +export default function App() { + return ( + <QueryClientProvider client={queries}> + <Catalogue /> + </QueryClientProvider> + ); +} + +function Catalogue() { + const [query, setQuery] = useState(""); + const [selected, setSelected] = useState<number | null>(null); + + return ( + <main style={{ fontFamily: "system-ui, sans-serif", padding: "16px", maxWidth: "960px", margin: "0 auto" }}> + <input + data-testid="query" + type="text" + aria-label="Search" + placeholder="Search the catalogue" + value={query} + onChange={e => setQuery(e.target.value)} + style={{ width: "100%", boxSizing: "border-box", padding: "8px", fontSize: "16px" }} + /> + <div style={{ display: "flex", gap: "24px", marginTop: "16px", flexWrap: "wrap" }}> + <div style={{ flex: "1 1 280px", minWidth: 0 }}> + <SearchResults q={query} selected={selected} onSelect={setSelected} /> + </div> + <section data-testid="detail" style={{ flex: "1 1 280px", minWidth: 0 }}> + {selected === null ? "pick a result" : <Detail id={selected} />} + </section> + </div> + </main> + ); +} + +type SearchResult = { q: string; hits: { id: number; name: string }[] }; + +function SearchResults({ + q, + selected, + onSelect, +}: { + q: string; + selected: number | null; + onSelect: (id: number) => void; +}) { + const enabled = q.length >= 2; + const result = useQuery({ + queryKey: ["search", q], + queryFn: async (): Promise<SearchResult> => ({ q, hits: await search(q) }), + enabled, + placeholderData: keepPreviousData, + retry: false, + staleTime: 0, + gcTime: 0, + }); + + if (q.length === 0) return <div data-testid="results" />; + if (q.length === 1) return <div data-testid="results">{`too short: ${q}`}</div>; + + const searching = result.isFetching; + const data = result.data; + + return ( + <div data-testid="results"> + {result.isError && !searching ? ( + <p>{`search failed: ${errorMessage(result.error)}`}</p> + ) : data ? ( + <> + <p>{`${data.hits.length} results for ${data.q}`}</p> + <ul style={{ paddingLeft: "20px" }}> + {data.hits.map(hit => ( + <li + key={hit.id} + data-testid="hit" + onClick={() => onSelect(hit.id)} + style={{ cursor: "pointer" }} + > + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ))} + </ul> + </> + ) : null} + {searching ? <p>searching…</p> : null} + </div> + ); +} + +function Detail({ id }: { id: number }) { + const item = useQuery({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + retry: false, + staleTime: 0, + gcTime: 0, + }); + + if (item.isFetching) return <p>loading…</p>; + if (item.isError) { + return ( + <div> + <p>{`failed: ${errorMessage(item.error)}`}</p> + <button type="button" data-testid="retry" onClick={() => void item.refetch()}> + retry + </button> + </div> + ); + } + if (item.data === null) return <p>{`item ${id} not found`}</p>; + if (!item.data) return null; + return ( + <div> + <h2 style={{ marginTop: 0 }}>{item.data.name}</h2> + <p>{item.data.blurb}</p> + </div> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-react-1/runlog.jsonl new file mode 100644 index 0000000..fa5b798 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:36:07.568Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:16.539Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:36:26.542Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:32.384Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/small/search-react-2/App.tsx new file mode 100644 index 0000000..9faebdd --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-2/App.tsx @@ -0,0 +1,125 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider, keepPreviousData, useQuery } from "@tanstack/react-query"; +import { fetchItem, search } from "./api.js"; + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +type ResultsProps = { + q: string; + selected: number | null; + onSelect: (id: number) => void; +}; + +function Results({ q, selected, onSelect }: ResultsProps) { + const enabled = q.length >= 2; + const s = useQuery({ + queryKey: ["search", q], + queryFn: () => search(q), + enabled, + // keep the previous hits on screen until the new ones arrive + placeholderData: keepPreviousData, + }); + + if (q === "") return <div data-testid="results" />; + if (q.length < 2) { + return ( + <div data-testid="results"> + <p>too short: {q}</p> + </div> + ); + } + if (s.isError) { + return ( + <div data-testid="results"> + <p>search failed: {errorMessage(s.error)}</p> + </div> + ); + } + + const hits = s.data ?? []; + return ( + <div data-testid="results"> + {s.isFetching ? <p>searching…</p> : <p>{hits.length} results for {q}</p>} + <ul> + {hits.map((h, i) => { + const sel = h.id === selected; + return ( + <li + key={`${h.id}:${i}`} + data-testid="hit" + tabIndex={0} + aria-selected={sel} + onClick={() => onSelect(h.id)} + onKeyDown={e => { + if (e.key === "Enter" || e.key === " ") onSelect(h.id); + }} + > + {sel ? "> " : ""} + {h.name} + </li> + ); + })} + </ul> + </div> + ); +} + +function Detail({ id }: { id: number }) { + const item = useQuery({ + queryKey: ["item", id], + queryFn: () => fetchItem(id), + }); + + if (item.isFetching) return <p>loading…</p>; + if (item.isError) { + return ( + <> + <p>failed: {errorMessage(item.error)}</p> + <button type="button" data-testid="retry" onClick={() => void item.refetch()}> + retry + </button> + </> + ); + } + if (item.data === null) return <p>item {id} not found</p>; + if (!item.data) return <p>loading…</p>; + return ( + <> + <h2>{item.data.name}</h2> + <p>{item.data.blurb}</p> + </> + ); +} + +function Page() { + const [q, setQ] = useState(""); + const [selected, setSelected] = useState<number | null>(null); + + return ( + <main> + <input + data-testid="query" + aria-label="query" + value={q} + onChange={e => setQ(e.target.value)} + /> + <Results q={q} selected={selected} onSelect={setSelected} /> + <section data-testid="detail"> + {selected === null ? <p>pick a result</p> : <Detail id={selected} />} + </section> + </main> + ); +} + +export default function App() { + const [client] = useState( + () => new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }), + ); + return ( + <QueryClientProvider client={client}> + <Page /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-react-2/runlog.jsonl new file mode 100644 index 0000000..7b759c7 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:39:46.177Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:39:53.208Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/small/search-react-3/App.tsx new file mode 100644 index 0000000..0de29e4 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-3/App.tsx @@ -0,0 +1,144 @@ +import { useRef, useState, type KeyboardEvent, type ReactNode } from "react"; +import { search, fetchItem, type Hit, type Item } from "./api.js"; + +type SearchState = { q: string; hits?: Hit[]; error?: string }; + +type DetailState = + | { id: number; status: "loading" } + | { id: number; status: "ok"; item: Item } + | { id: number; status: "missing" } + | { id: number; status: "error"; error: string }; + +const errText = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +function HitList({ hits, selected, onSelect }: { hits: Hit[]; selected: number | null; onSelect: (id: number) => void }) { + return ( + <ul> + {hits.map((hit, i) => { + const onKey = (e: KeyboardEvent<HTMLLIElement>) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onSelect(hit.id); + } + }; + return ( + <li + key={`${i}-${hit.id}`} + data-testid="hit" + tabIndex={0} + onClick={() => onSelect(hit.id)} + onKeyDown={onKey} + style={{ cursor: "pointer", fontWeight: hit.id === selected ? 600 : 400 }} + > + {hit.id === selected ? `> ${hit.name}` : hit.name} + </li> + ); + })} + </ul> + ); +} + +export default function App() { + const [query, setQuery] = useState(""); + const [results, setResults] = useState<SearchState | null>(null); + const [selected, setSelected] = useState<number | null>(null); + const [detail, setDetail] = useState<DetailState | null>(null); + const searchSeq = useRef(0); + const detailSeq = useRef(0); + + function onQuery(q: string) { + setQuery(q); + // every keystroke invalidates older searches, so only the latest answer lands + const n = ++searchSeq.current; + if (q.length < 2) return; + search(q).then( + hits => { if (searchSeq.current === n) setResults({ q, hits }); }, + e => { if (searchSeq.current === n) setResults({ q, error: errText(e) }); }, + ); + } + + function load(id: number) { + const n = ++detailSeq.current; + setDetail({ id, status: "loading" }); + fetchItem(id).then( + item => { + if (detailSeq.current !== n) return; + setDetail(item ? { id, status: "ok", item } : { id, status: "missing" }); + }, + e => { if (detailSeq.current === n) setDetail({ id, status: "error", error: errText(e) }); }, + ); + } + + function select(id: number) { + setSelected(id); + load(id); + } + + let resultsBody: ReactNode = null; + const q = query; + if (q.length === 1) { + resultsBody = <p>{`too short: ${q}`}</p>; + } else if (q.length >= 2) { + const fresh = results !== null && results.q === q; + if (!fresh) { + // keep the previous hits on screen while the new search runs + resultsBody = ( + <> + <p>searching…</p> + {results?.hits && <HitList hits={results.hits} selected={selected} onSelect={select} />} + </> + ); + } else if (results.error !== undefined) { + resultsBody = <p>{`search failed: ${results.error}`}</p>; + } else { + const hits = results.hits ?? []; + resultsBody = ( + <> + <p>{`${hits.length} results for ${q}`}</p> + <HitList hits={hits} selected={selected} onSelect={select} /> + </> + ); + } + } + + let detailBody: ReactNode; + if (selected === null) { + detailBody = <p>pick a result</p>; + } else if (detail === null || detail.status === "loading") { + detailBody = <p>loading…</p>; + } else if (detail.status === "ok") { + detailBody = ( + <> + <h2>{detail.item.name}</h2> + <p>{detail.item.blurb}</p> + </> + ); + } else if (detail.status === "missing") { + detailBody = <p>{`item ${detail.id} not found`}</p>; + } else { + const failedId = detail.id; + detailBody = ( + <> + <p>{`failed: ${detail.error}`}</p> + <button type="button" data-testid="retry" onClick={() => load(failedId)}> + retry + </button> + </> + ); + } + + return ( + <main style={{ display: "flex", gap: "2rem", padding: "1rem", fontFamily: "system-ui, sans-serif", flexWrap: "wrap" }}> + <section style={{ flex: "1 1 260px" }}> + <label> + Search{" "} + <input data-testid="query" type="text" value={query} onChange={e => onQuery(e.target.value)} /> + </label> + <div data-testid="results">{resultsBody}</div> + </section> + <section data-testid="detail" style={{ flex: "1 1 260px" }}> + {detailBody} + </section> + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-react-3/runlog.jsonl new file mode 100644 index 0000000..a7e42e8 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:39:40.631Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:39:47.853Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/small/search-rx-1/main.rx new file mode 100644 index 0000000..2d2e3b9 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-1/main.rx @@ -0,0 +1,74 @@ +import pure { len } from "./prelude" +import pure { search, fetchItem } from "./api" + +export default component App() { + source query := "" + source selected: number | null := null + + // The hits for the current query: nothing is asked under two characters. + hits = len(query) < 2 ? [] : await search(query) + + // The selected hit's item; null means "no such item". + item = selected == null ? null : await fetchItem(selected) + + action retry() { + refresh item + } + + <main> + <input + data-testid="query" + value={query} + onInput={(e) => { query := e.target.value }} + /> + + <div data-testid="results"> + {if len(query) == 1 { + <p>too short: {query}</p> + } else { + {if len(query) > 1 { + {errored fallback(e) { + <p>search failed: {e.message}</p> + } { + {loading fallback { + <p>searching…</p> + } { + {if pending? hits { + <p>searching…</p> + } else { + <p>{len(hits)} results for {query}</p> + }} + <ul> + {for h in hits { + <li data-testid="hit" onClick={() => { selected := h.id }}>{h.id == selected ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + </div> + + <section data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + {loading on(selected) fallback { + <p>loading…</p> + } { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { retry() }}>retry</button> + } { + {if item == null { + <p>item {selected} not found</p> + } else { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + }} + }} + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-rx-1/runlog.jsonl new file mode 100644 index 0000000..67182f0 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-1/runlog.jsonl @@ -0,0 +1,7 @@ +{"at":"2026-10-27T20:36:11.758Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:20.285Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:36:28.193Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:32.530Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:36:54.406Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:58.533Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:37:17.562Z","cmd":"test","ok":true,"build":true,"passed":3,"total":3} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/small/search-rx-2/main.rx new file mode 100644 index 0000000..2f2d2a2 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-2/main.rx @@ -0,0 +1,70 @@ +import pure { len } from "./prelude" +import pure { search, fetchItem } from "./api" + +export default component App() { + source query := "" + source selected: number | null := null + + // Two or more characters ask the server; shorter queries ask nothing. + // The answer carries the query it answers, so the header names the + // query that produced the hits on screen. + found = query.length < 2 ? null : { q: query, hits: await search(query) } + + // One question per selection; a new selection shows `loading` again. + item = selected == null ? null : await fetchItem(selected) + + action retry() { + refresh item + } + + <main> + <input + data-testid="query" + value={query} + onInput={(e) => { query := e.target.value }} + /> + + <section data-testid="results"> + {if query.length == 1 { + <p>too short: {query}</p> + }} + {if query.length >= 2 { + errored fallback(e) { <p>search failed: {e.message}</p> } { + <p>{pending? found ? "searching…" : ""}</p> + {loading fallback { <span></span> } { + {if found { + <p>{len(found.hits)} results for {found.q}</p> + <ul> + {for h in found.hits { + <li data-testid="hit" onClick={() => { selected := h.id }}> + {selected == h.id ? "> " : ""}{h.name} + </li> + }} + </ul> + }} + }} + } + }} + </section> + + <section data-testid="detail"> + {if selected == null { + <p>pick a result</p> + } else { + loading on(selected) fallback { <p>loading…</p> } { + errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { retry() }}>retry</button> + } { + {if item == null { + <p>item {selected} not found</p> + } else { + <h2>{item.name}</h2> + <p>{item.blurb}</p> + }} + } + } + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-rx-2/runlog.jsonl new file mode 100644 index 0000000..f6eb527 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-2/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:40:14.757Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:21.520Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:38.320Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:42.922Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/small/search-rx-3/main.rx new file mode 100644 index 0000000..cdb1b36 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-3/main.rx @@ -0,0 +1,63 @@ +import pure { search, fetchItem } from "./api" + +export default component App() { + source query := "" + source picked: number | null := null + + // asked only for two characters or more; a shorter query asks nothing + found = query.length < 2 ? null : { q: query, list: await search(query) } + + // asked only once a hit is picked + item = picked == null ? null : await fetchItem(picked) + + <main> + <input data-testid="query" value={query} onInput={(e) => { query := e.target.value }} /> + + <section data-testid="results"> + {if query.length == 1 { + <p>too short: {query}</p> + }} + {if query.length >= 2 { + {loading fallback { <p>searching…</p> } { + {errored fallback(e) { + <p>search failed: {e.message}</p> + } { + {if pending? found { + <p>searching…</p> + }} + {if found != null { + <p>{found.list.length} results for {found.q}</p> + <ul> + {for h in found.list { + <li data-testid="hit" onClick={() => { picked := h.id }}>{picked == h.id ? "> " : ""}{h.name}</li> + }} + </ul> + }} + }} + }} + }} + </section> + + <section data-testid="detail"> + {if picked == null { + <p>pick a result</p> + } else { + {if pending? item { + <p>loading…</p> + } else { + {errored fallback(e) { + <p>failed: {e.message}</p> + <button data-testid="retry" onClick={() => { refresh item }}>retry</button> + } { + {if item == null { + <p>item {picked} not found</p> + } else { + <h2>{item?.name}</h2> + <p>{item?.blurb}</p> + }} + }} + }} + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/search-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/search-rx-3/runlog.jsonl new file mode 100644 index 0000000..bfc318f --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/search-rx-3/runlog.jsonl @@ -0,0 +1,7 @@ +{"at":"2026-10-27T20:39:48.189Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:39:54.843Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:40:04.541Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:09.648Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:27.297Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:40:43.982Z","cmd":"test","ok":true,"build":true,"passed":1,"total":1} +{"at":"2026-10-27T20:40:53.974Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/small/settings-react-1/App.tsx new file mode 100644 index 0000000..80b9a68 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-1/App.tsx @@ -0,0 +1,180 @@ +import { useEffect, useRef, useState, type FormEvent } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Density, Settings } from "./api.d.ts"; + +const MAX_NAME = 20; + +type LoadState = + | { status: "loading" } + | { status: "failed"; message: string } + | { status: "ready" }; + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +/** The problem with a display name, or null when it is valid. */ +function nameProblem(name: string): string | null { + if (name.trim() === "") return "Display name is required"; + if (name.length > MAX_NAME) { + return `Display name must be at most ${MAX_NAME} characters, not ${name.length}`; + } + return null; +} + +function describe(s: Settings): string { + return `${s.name} · ${s.theme} · ${s.density}`; +} + +export default function App() { + const [load, setLoad] = useState<LoadState>({ status: "loading" }); + const [attempt, setAttempt] = useState(0); + /** the last settings the backend confirmed (loaded or saved) */ + const [saved, setSaved] = useState<Settings | null>(null); + /** the user's editable copy; filled once loaded, then owned by the user */ + const [draft, setDraft] = useState<Settings | null>(null); + /** the values of the save in flight, if any */ + const [saving, setSaving] = useState<Settings | null>(null); + const [saveError, setSaveError] = useState<string | null>(null); + const latestSave = useRef(0); + + useEffect(() => { + let current = true; + loadSettings().then( + s => { + if (!current) return; + setSaved(s); + setDraft(s); + setLoad({ status: "ready" }); + }, + e => { + if (!current) return; + setLoad({ status: "failed", message: errorMessage(e) }); + }, + ); + return () => { + current = false; + }; + }, [attempt]); + + function retryLoad() { + setLoad({ status: "loading" }); + setAttempt(n => n + 1); + } + + function patchDraft(patch: Partial<Settings>) { + setDraft(d => (d ? { ...d, ...patch } : d)); + } + + function onSave(e: FormEvent) { + e.preventDefault(); + if (!draft || nameProblem(draft.name) !== null) return; + const values: Settings = { name: draft.name, theme: draft.theme, density: draft.density }; + const seq = ++latestSave.current; + setSaving(values); + setSaveError(null); + saveSettings(values).then( + s => { + if (seq !== latestSave.current) return; + setSaved(s); + setSaving(null); + }, + e => { + if (seq !== latestSave.current) return; + setSaving(null); + setSaveError(errorMessage(e)); + }, + ); + } + + const shown = saving ?? saved; + const problem = draft ? nameProblem(draft.name) : null; + + return ( + <main style={{ maxWidth: 480, margin: "0 auto", padding: 16, fontFamily: "system-ui, sans-serif" }}> + <h1>Settings</h1> + + {load.status === "loading" && <p data-testid="load">Loading settings…</p>} + {load.status === "failed" && ( + <> + <p data-testid="load">Could not load settings: {load.message}</p> + <button type="button" data-testid="retry-load" onClick={retryLoad}> + Retry + </button> + </> + )} + + {shown && draft && ( + <> + <p data-testid="saved"> + {`Saved: ${describe(shown)}`} + {saving ? " (saving…)" : ""} + </p> + + <form onSubmit={onSave} noValidate> + <div> + <label> + Display name{" "} + <input + name="name" + data-testid="name" + type="text" + value={draft.name} + onChange={e => patchDraft({ name: e.target.value })} + /> + </label>{" "} + <span data-testid="name-count">{`${draft.name.length}/${MAX_NAME}`}</span> + {problem && ( + <p data-testid="name-error" role="alert"> + {problem} + </p> + )} + </div> + + <div> + <span>Theme </span> + <button + type="button" + data-testid="theme-light" + aria-pressed={draft.theme === "light"} + onClick={() => patchDraft({ theme: "light" })} + > + Light + </button>{" "} + <button + type="button" + data-testid="theme-dark" + aria-pressed={draft.theme === "dark"} + onClick={() => patchDraft({ theme: "dark" })} + > + Dark + </button> + </div> + + <div> + <button + type="button" + data-testid="density-toggle" + onClick={() => + patchDraft({ + density: (draft.density === "comfortable" ? "compact" : "comfortable") as Density, + }) + } + > + {`Density: ${draft.density}`} + </button> + </div> + + <p data-testid="preview">{`Preview: ${draft.theme} / ${draft.density}`}</p> + + <button type="submit" data-testid="save"> + Save + </button> + </form> + + {saveError !== null && <p data-testid="save-error">{`Save failed: ${saveError}`}</p>} + </> + )} + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-react-1/runlog.jsonl new file mode 100644 index 0000000..a595e29 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:35:40.865Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:35:47.666Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/small/settings-react-2/App.tsx new file mode 100644 index 0000000..c8eebd4 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-2/App.tsx @@ -0,0 +1,152 @@ +import { QueryClient, QueryClientProvider, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Density, Settings, Theme } from "./api.js"; + +const SETTINGS_KEY = ["settings"] as const; +const MAX_NAME = 20; + +const queryClient = new QueryClient(); + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <SettingsPage /> + </QueryClientProvider> + ); +} + +function SettingsPage() { + const client = useQueryClient(); + const query = useQuery({ + queryKey: SETTINGS_KEY, + queryFn: () => loadSettings(), + retry: false, + }); + const save = useMutation({ + mutationFn: (s: Settings) => saveSettings(s), + // Only a successful save changes what counts as "saved". + onSuccess: (s) => client.setQueryData(SETTINGS_KEY, s), + }); + + const saved = query.data; + const loading = saved === undefined && query.isFetching; + const loadFailed = saved === undefined && !query.isFetching && query.isError; + + return ( + <main> + <h1>Settings</h1> + + {loading && <p data-testid="load">Loading settings…</p>} + {loadFailed && ( + <p data-testid="load"> + Could not load settings: {query.error instanceof Error ? query.error.message : String(query.error)}{" "} + <button data-testid="retry-load" onClick={() => void query.refetch()}> + Retry + </button> + </p> + )} + + {saved !== undefined && ( + <> + <p data-testid="saved"> + {save.isPending && save.variables + ? `Saved: ${describe(save.variables)} (saving…)` + : `Saved: ${describe(saved)}`} + </p> + {save.isError && ( + <p data-testid="save-error" role="alert"> + Save failed: {save.error instanceof Error ? save.error.message : String(save.error)} + </p> + )} + <Draft + initial={saved} + onSave={(draft) => { + if (draft.name.length === 0 || draft.name.length > MAX_NAME) return; + save.mutate(draft); + }} + /> + </> + )} + </main> + ); +} + +function describe(s: Settings): string { + return `${s.name} · ${s.theme} · ${s.density}`; +} + +function Draft({ initial, onSave }: { initial: Settings; onSave: (s: Settings) => void }) { + const [name, setName] = useState(initial.name); + const [theme, setTheme] = useState<Theme>(initial.theme); + const [density, setDensity] = useState<Density>(initial.density); + + const length = name.length; + const nameError = + length === 0 + ? "Display name is required" + : length > MAX_NAME + ? `Display name must be at most ${MAX_NAME} characters, not ${length}` + : null; + + return ( + <form + onSubmit={(e) => e.preventDefault()} + style={{ display: "grid", gap: "0.75rem", maxWidth: "24rem" }} + > + <label style={{ display: "grid", gap: "0.25rem" }}> + Display name + <input + data-testid="name" + value={name} + onChange={(e) => setName(e.target.value)} + aria-invalid={nameError !== null} + aria-describedby={nameError ? "name-error" : undefined} + /> + </label> + <span data-testid="name-count">{`${length}/${MAX_NAME}`}</span> + {nameError && ( + <span data-testid="name-error" id="name-error" role="alert"> + {nameError} + </span> + )} + + <div role="group" aria-label="Theme" style={{ display: "flex", gap: "0.5rem" }}> + <button + type="button" + data-testid="theme-light" + aria-pressed={theme === "light"} + onClick={() => setTheme("light")} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={theme === "dark"} + onClick={() => setTheme("dark")} + > + Dark + </button> + </div> + + <button + type="button" + data-testid="density-toggle" + onClick={() => setDensity((d) => (d === "comfortable" ? "compact" : "comfortable"))} + > + Density: {density} + </button> + + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + + <button + type="button" + data-testid="save" + onClick={() => onSave({ name, theme, density })} + > + Save + </button> + </form> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-react-2/runlog.jsonl new file mode 100644 index 0000000..be44ef6 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:36:46.853Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:55.009Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/App.tsx new file mode 100644 index 0000000..0af85cc --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/App.tsx @@ -0,0 +1,20 @@ +import { useState } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import SettingsPage from "./SettingsPage"; + +export default function App() { + const [client] = useState( + () => + new QueryClient({ + defaultOptions: { + queries: { retry: false, refetchOnWindowFocus: false, staleTime: Infinity }, + mutations: { retry: false }, + }, + }), + ); + return ( + <QueryClientProvider client={client}> + <SettingsPage /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsForm.tsx b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsForm.tsx new file mode 100644 index 0000000..d96eb28 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsForm.tsx @@ -0,0 +1,79 @@ +import { useState } from "react"; +import type { Density, Settings, Theme } from "./api.js"; + +const NAME_MAX = 20; + +export function nameError(name: string): string | null { + if (name.length === 0) return "Display name is required"; + if (name.length > NAME_MAX) { + return `Display name must be at most ${NAME_MAX} characters, not ${name.length}`; + } + return null; +} + +type Props = { + initial: Settings; + onSave: (s: Settings) => void; +}; + +export default function SettingsForm({ initial, onSave }: Props) { + // The draft is seeded once from the loaded settings and then belongs to the user. + const [name, setName] = useState<string>(initial.name); + const [theme, setTheme] = useState<Theme>(initial.theme); + const [density, setDensity] = useState<Density>(initial.density); + const error = nameError(name); + + return ( + <form + onSubmit={(e) => { + e.preventDefault(); + if (nameError(name) !== null) return; + onSave({ name, theme, density }); + }} + > + <label> + Display name + <input + data-testid="name" + value={name} + onChange={(e) => setName(e.target.value)} + /> + </label> + <span data-testid="name-count">{`${name.length}/${NAME_MAX}`}</span> + {error !== null && <p data-testid="name-error">{error}</p>} + + <div> + <button + type="button" + data-testid="theme-light" + aria-pressed={theme === "light"} + onClick={() => setTheme("light")} + > + Light + </button> + <button + type="button" + data-testid="theme-dark" + aria-pressed={theme === "dark"} + onClick={() => setTheme("dark")} + > + Dark + </button> + </div> + + <button + type="button" + data-testid="density-toggle" + onClick={() => setDensity((d) => (d === "comfortable" ? "compact" : "comfortable"))} + > + Density: {density} + </button> + + <p data-testid="preview">{`Preview: ${theme} / ${density}`}</p> + + <button type="submit" data-testid="save"> + Save + </button> + </form> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsPage.tsx b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsPage.tsx new file mode 100644 index 0000000..32673f5 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/SettingsPage.tsx @@ -0,0 +1,57 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { loadSettings, saveSettings } from "./api.js"; +import type { Settings } from "./api.js"; +import SettingsForm from "./SettingsForm"; + +const SETTINGS_KEY = ["settings"] as const; + +function describeSettings(s: Settings): string { + return `${s.name} · ${s.theme} · ${s.density}`; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export default function SettingsPage() { + const queryClient = useQueryClient(); + const query = useQuery({ + queryKey: SETTINGS_KEY, + queryFn: () => loadSettings(), + }); + const mutation = useMutation({ + mutationFn: (s: Settings) => saveSettings(s), + onSuccess: (saved) => { + queryClient.setQueryData(SETTINGS_KEY, saved); + }, + }); + + if (!query.data) { + if (query.isFetching || query.isPending) { + return <p data-testid="load">Loading settings…</p>; + } + return ( + <div> + <p data-testid="load">Could not load settings: {messageOf(query.error)}</p> + <button type="button" data-testid="retry-load" onClick={() => void query.refetch()}> + Retry + </button> + </div> + ); + } + + const savedText = + mutation.isPending && mutation.variables + ? `Saved: ${describeSettings(mutation.variables)} (saving…)` + : `Saved: ${describeSettings(query.data)}`; + + return ( + <main> + <p data-testid="saved">{savedText}</p> + <SettingsForm initial={query.data} onSave={(s) => mutation.mutate(s)} /> + {mutation.isError && ( + <p data-testid="save-error">Save failed: {messageOf(mutation.error)}</p> + )} + </main> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/runlog.jsonl new file mode 100644 index 0000000..733963a --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:39:35.511Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:39:44.044Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/main.rx new file mode 100644 index 0000000..5ce8793 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/main.rx @@ -0,0 +1,61 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +type Settings = { name: string; theme: string; density: string } + +source saved: Settings = await loadSettings() +source saveError = "" + +action retry() { refresh saved } + +action save(next: Settings) { + optimistic saved := next + optimistic saveError := "" + affects saved + try { + let answer = await saveSettings(next) + saved := answer + } catch (e) { + saveError := e.message + } +} + +component Editor(props: { initial: Settings }) { + source draftName := peek(props.initial.name) + source draftTheme := peek(props.initial.theme) + source draftDensity := peek(props.initial.density) + nameError = len(draftName) == 0 ? "Display name is required" : len(draftName) > 20 ? `Display name must be at most 20 characters, not ${len(draftName)}` : "" + + <div> + <label>Display name <input data-testid="name" value={draftName} onInput={(e) => { draftName := e.target.value }} /></label> + <span data-testid="name-count">{`${len(draftName)}/20`}</span> + {if nameError != "" { + <p data-testid="name-error">{nameError}</p> + }} + <div> + <button data-testid="theme-light" onClick={() => { draftTheme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { draftTheme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { draftDensity := draftDensity == "compact" ? "comfortable" : "compact" }}>Density</button> + </div> + <p data-testid="preview">{`Preview: ${draftTheme} / ${draftDensity}`}</p> + <button data-testid="save" onClick={() => { if (nameError == "") { save({ name: draftName, theme: draftTheme, density: draftDensity }) } }}>Save</button> + </div> +} + +export default component App() { + <main> + {errored fallback(e) { + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { retry() }}>Retry</button> + } { + {loading fallback { <p data-testid="load">Loading settings…</p> } { + <p data-testid="saved">{`Saved: ${saved.name} · ${saved.theme} · ${saved.density}${(pending? saved) ? " (saving…)" : ""}`}</p> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + <Editor initial={saved} /> + }} + }} + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/runlog.jsonl new file mode 100644 index 0000000..2489c05 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-1/runlog.jsonl @@ -0,0 +1,4 @@ +{"at":"2026-10-27T20:36:00.986Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:12.907Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:36:32.565Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:36:36.732Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/main.rx new file mode 100644 index 0000000..1439dd5 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/main.rx @@ -0,0 +1,67 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" + +type Settings = { name: string; theme: string; density: string } + +export default component App() { + // attempt is the retry key: a retry re-asks the question and shows Loading again. + source attempt := 0 + // Both arms ask the same question; reading attempt makes a retry ask it again. + source saved: Settings = attempt >= 0 ? await loadSettings() : await loadSettings() + source saveError: string := "" + + action save(next: Settings) { + optimistic saved := next + onFlush(() => { saveError := "" }) + affects saved + try { + let answer = await saveSettings(next) + saved := answer + } catch (e) { + saveError := e.message + } + } + + <main> + {loading on(attempt) fallback { <p data-testid="load">Loading settings…</p> } { + {errored fallback(e) { + <div> + <p data-testid="load">{`Could not load settings: ${e.message}`}</p> + <button data-testid="retry-load" onClick={() => { attempt := attempt + 1 }}>Retry</button> + </div> + } { + source draftName := peek(saved.name) + source draftTheme := peek(saved.theme) + source draftDensity := peek(saved.density) + nameError = draftName.length == 0 ? "Display name is required" : draftName.length > 20 ? `Display name must be at most 20 characters, not ${draftName.length}` : "" + + <p data-testid="saved">{`Saved: ${saved.name} · ${saved.theme} · ${saved.density}${pending? saved ? " (saving…)" : ""}`}</p> + + <div> + <label> + Display name + <input data-testid="name" value={draftName} onInput={(e) => { draftName := e.target.value }} /> + </label> + <span data-testid="name-count">{`${draftName.length}/20`}</span> + {if nameError != "" { + <p data-testid="name-error">{nameError}</p> + }} + <div> + <button data-testid="theme-light" onClick={() => { draftTheme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { draftTheme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { draftDensity := draftDensity == "compact" ? "comfortable" : "compact" }}>Toggle density</button> + </div> + <p data-testid="preview">{`Preview: ${draftTheme} / ${draftDensity}`}</p> + <button data-testid="save" onClick={() => { + if (nameError == "") { + save({ name: draftName, theme: draftTheme, density: draftDensity }) + } + }}>Save</button> + </div> + {if saveError != "" { + <p data-testid="save-error">{`Save failed: ${saveError}`}</p> + }} + }} + }} + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/runlog.jsonl new file mode 100644 index 0000000..68f4d75 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-2/runlog.jsonl @@ -0,0 +1,6 @@ +{"at":"2026-10-27T20:40:29.372Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/overlap-small/settings-rx-2/src/main.rx: line 9: expected '{' near 'source draftTheme :='"} +{"at":"2026-10-27T20:40:49.576Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/overlap-small/settings-rx-2/src/main.rx: line 33: W-2: write in reactive scope (no statement position) near '\n source draf'"} +{"at":"2026-10-27T20:40:59.963Z","cmd":"check","ok":false,"out":"/home/claude/runs/overlap-small/settings-rx-2/src/main.rx:9: error UNBOUND [SCOPE] Error is not declared here (a component sees its declaration site, not its caller — RG-7a)"} +{"at":"2026-10-27T20:41:06.292Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:41:14.529Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:41:35.156Z","cmd":"test","ok":true,"build":true,"passed":3,"total":3} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/main.rx new file mode 100644 index 0000000..c3e36f7 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/main.rx @@ -0,0 +1,82 @@ +import pure { loadSettings } from "./api" +import { saveSettings } from "./api" +import pure { len } from "./prelude" + +type Settings = { name: string; theme: string; density: string } + +// Bumped by retry: each bump asks the load again. +source attempt := 0 + +// The settings as last saved (or loaded). A question: asked again when `attempt` changes. +source saved = attempt >= 0 ? await loadSettings() : null + +source saveError: string = "" + +action retry() { + attempt := attempt + 1 +} + +action save(s: Settings) { + optimistic saved := s + affects saved + optimistic saveError := "" + try { + let answer = await saveSettings(s) + saved := answer + } catch (e) { + saveError := e.message + } +} + +// The draft: seeded once from the loaded settings, then it belongs to the user. +component Editor(props: { initial: Settings }) { + source dName := peek(props.initial.name) + source dTheme := peek(props.initial.theme) + source dDensity := peek(props.initial.density) + nameOk = len(dName) > 0 && len(dName) <= 20 + + <section> + <p data-testid="saved">Saved: {saved.name} · {saved.theme} · {saved.density}{(pending? saved) ? " (saving…)" : ""}</p> + + <label> + Name + <input data-testid="name" value={dName} onInput={(e) => { dName := e.target.value }} /> + </label> + <span data-testid="name-count">{len(dName)}/20</span> + {if len(dName) == 0 { + <p data-testid="name-error">Display name is required</p> + }} + {if len(dName) > 20 { + <p data-testid="name-error">Display name must be at most 20 characters, not {len(dName)}</p> + }} + + <div> + <button data-testid="theme-light" onClick={() => { dTheme := "light" }}>Light</button> + <button data-testid="theme-dark" onClick={() => { dTheme := "dark" }}>Dark</button> + <button data-testid="density-toggle" onClick={() => { dDensity := dDensity == "compact" ? "comfortable" : "compact" }}>Density</button> + </div> + <p data-testid="preview">Preview: {dTheme} / {dDensity}</p> + + <button data-testid="save" onClick={() => { + if (nameOk) { save({ name: dName, theme: dTheme, density: dDensity }) } + }}>Save</button> + {if saveError != "" { + <p data-testid="save-error">Save failed: {saveError}</p> + }} + </section> +} + +export default component App() { + loading on(attempt) fallback { + <p data-testid="load">Loading settings…</p> + } { + errored fallback(e) { + <div> + <p data-testid="load">Could not load settings: {e.message}</p> + <button data-testid="retry-load" onClick={() => { retry() }}>Retry</button> + </div> + } { + <Editor initial={saved} /> + } + } +} diff --git a/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/runlog.jsonl new file mode 100644 index 0000000..0266aef --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/settings-rx-3/runlog.jsonl @@ -0,0 +1,7 @@ +{"at":"2026-10-27T20:40:09.961Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:13.138Z","cmd":"test","ok":false,"build":false,"passed":0,"total":4} +{"at":"2026-10-27T20:40:27.125Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:32.476Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:40:49.395Z","cmd":"test","ok":false,"build":true,"passed":3,"total":5} +{"at":"2026-10-27T20:41:02.873Z","cmd":"test","ok":true,"build":true,"passed":2,"total":2} +{"at":"2026-10-27T20:41:10.324Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/App.tsx b/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/App.tsx new file mode 100644 index 0000000..43cd887 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/App.tsx @@ -0,0 +1,195 @@ +import { useState } from "react"; +import { + QueryClient, + QueryClientProvider, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import type { QueryClient as QC } from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; + +const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +/** Writes an assignee into every cached issue list, wherever the issue appears. */ +function setAssignee(qc: QC, id: number, assignee: string) { + qc.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(issue => (issue.id === id ? { ...issue, assignee } : issue)), + ); +} + +export default function App() { + const [client] = useState( + () => + new QueryClient({ + defaultOptions: { + queries: { retry: false, refetchOnWindowFocus: false }, + mutations: { retry: false }, + }, + }), + ); + return ( + <QueryClientProvider client={client}> + <Tracker /> + </QueryClientProvider> + ); +} + +function Tracker() { + const qc = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + + const list = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + retry: false, + }); + + const assignMutation = useMutation({ + mutationFn: ({ id, user }: { id: number; user: string }) => assign(id, user), + onMutate: ({ id, user }) => { + const snapshot = qc.getQueriesData<Issue[]>({ queryKey: ["issues"] }); + setAssignee(qc, id, user); + return { snapshot }; + }, + onError: (_error, _vars, context) => { + for (const [key, data] of context?.snapshot ?? []) qc.setQueryData(key, data); + }, + onSuccess: res => setAssignee(qc, res.id, res.assignee), + }); + + const select = (id: number) => { + setSelected(id); + assignMutation.reset(); + }; + + return ( + <div style={{ fontFamily: "system-ui, sans-serif", padding: 16, display: "flex", gap: 24, alignItems: "flex-start" }}> + <div style={{ flex: "1 1 320px", minWidth: 0 }}> + <label> + Current user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div style={{ margin: "12px 0", display: "flex", gap: 8 }}> + <button + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list"> + {list.isPending ? ( + <p>{`loading ${filter} issues`}</p> + ) : list.isError ? ( + <p>{`list error: ${errorMessage(list.error)}`}</p> + ) : ( + <ul style={{ listStyle: "none", padding: 0, margin: 0 }}> + {list.data.map(issue => ( + <li key={issue.id}> + <button + data-testid="issue" + aria-pressed={selected === issue.id} + onClick={() => select(issue.id)} + style={{ width: "100%", textAlign: "left", padding: "6px 8px", marginBottom: 4 }} + > + #{issue.id} {issue.title} ({issue.assignee ?? "unassigned"}) + </button> + </li> + ))} + </ul> + )} + </div> + </div> + + <div data-testid="detail" style={{ flex: "2 1 360px", minWidth: 0 }}> + {selected === null ? ( + <p>select an issue</p> + ) : ( + <Detail + key={selected} + id={selected} + user={user} + assignError={assignMutation.error} + onAssign={() => assignMutation.mutate({ id: selected, user })} + /> + )} + </div> + </div> + ); +} + +function Detail({ + id, + user, + assignError, + onAssign, +}: { + id: number; + user: string; + assignError: Error | null; + onAssign: () => void; +}) { + const q = useQuery({ + queryKey: ["issue", id], + queryFn: () => fetchIssue(id), + retry: false, + }); + + let content; + if (q.isPending || (q.isFetching && q.isError)) { + content = <p>{`loading #${id}`}</p>; + } else if (q.isError) { + content = ( + <> + <p>{`detail error: ${errorMessage(q.error)}`}</p> + <button data-testid="retry" onClick={() => void q.refetch()}> + retry + </button> + </> + ); + } else if (q.data === null) { + content = ( + <> + <p>{`detail error: issue #${id} not found`}</p> + <button data-testid="retry" onClick={() => void q.refetch()}> + retry + </button> + </> + ); + } else { + content = ( + <> + <h2 style={{ margin: "0 0 8px" }}> + #{q.data.id} {q.data.title} + </h2> + <p>{q.data.body}</p> + </> + ); + } + + return ( + <div> + {content} + <div style={{ marginTop: 12 }}> + <button data-testid="assign" onClick={onAssign}> + {`assign to ${user}`} + </button> + {assignError && <p>{`assign error: ${errorMessage(assignError)}`}</p>} + </div> + </div> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/runlog.jsonl new file mode 100644 index 0000000..354d340 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-1/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:36:07.844Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:36:17.808Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/App.tsx b/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/App.tsx new file mode 100644 index 0000000..6cbafbd --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/App.tsx @@ -0,0 +1,180 @@ +import { useState } from "react"; +import type { ReactNode } from "react"; +import { QueryClient, QueryClientProvider, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { QueryKey } from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; + +const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selectedId, setSelectedId] = useState<number | null>(null); + + const issues = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + const selected = selectedId; + const detail = useQuery({ + queryKey: ["issue", selected], + queryFn: () => (selected === null ? Promise.resolve(null) : fetchIssue(selected)), + enabled: selected !== null, + }); + + const assignIssue = useMutation({ + mutationFn: (vars: { id: number; who: string }) => assign(vars.id, vars.who), + onMutate: async ({ id, who }) => { + // Remember each cached list's current assignee for this row, then show the new one at once. + const prior = queryClient.getQueriesData<Issue[]>({ queryKey: ["issues"] }); + const restore: [QueryKey, string | null][] = []; + for (const [key, rows] of prior) { + const row = rows?.find(r => r.id === id); + if (row) restore.push([key, row.assignee]); + } + queryClient.setQueriesData<Issue[]>( + { queryKey: ["issues"] }, + rows => rows?.map(r => (r.id === id ? { ...r, assignee: who } : r)), + ); + return { id, restore }; + }, + onError: (_error, _vars, ctx) => { + if (!ctx) return; + for (const [key, assignee] of ctx.restore) { + queryClient.setQueryData<Issue[]>(key, rows => + rows?.map(r => (r.id === ctx.id ? { ...r, assignee } : r)), + ); + } + }, + onSuccess: result => { + queryClient.setQueriesData<Issue[]>( + { queryKey: ["issues"] }, + rows => rows?.map(r => (r.id === result.id ? { ...r, assignee: result.assignee } : r)), + ); + }, + }); + + function select(id: number) { + setSelectedId(id); + assignIssue.reset(); + } + + let detailContent: ReactNode; + if (selectedId === null) { + detailContent = <p>select an issue</p>; + } else if (detail.isPending) { + detailContent = <p>loading #{selectedId}</p>; + } else if (detail.isError) { + detailContent = <p>detail error: {errorMessage(detail.error)}</p>; + } else if (detail.data === null || detail.data === undefined) { + detailContent = <p>detail error: issue #{selectedId} not found</p>; + } else { + detailContent = ( + <> + <h2>#{detail.data.id} {detail.data.title}</h2> + <p>{detail.data.body}</p> + </> + ); + } + const showRetry = selectedId !== null && (detail.isError || detail.data === null); + + let listContent: ReactNode; + if (issues.isPending) { + listContent = <p>loading {filter} issues</p>; + } else if (issues.isError) { + listContent = <p>list error: {errorMessage(issues.error)}</p>; + } else { + listContent = ( + <ul> + {issues.data.map(issue => ( + <li key={issue.id}> + <button + type="button" + data-testid="issue" + aria-current={issue.id === selectedId ? "true" : undefined} + onClick={() => select(issue.id)} + > + {`#${issue.id} ${issue.title} (${issue.assignee ?? "unassigned"})`} + </button> + </li> + ))} + </ul> + ); + } + + return ( + <main style={{ fontFamily: "system-ui, sans-serif", padding: 16 }}> + <div style={{ marginBottom: 12 }}> + <label> + Current user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + </div> + <div style={{ display: "flex", gap: 24, alignItems: "flex-start" }}> + <div style={{ flex: 1, minWidth: 0 }}> + <div style={{ marginBottom: 8 }}> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button>{" "} + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <section data-testid="list">{listContent}</section> + </div> + <section data-testid="detail" style={{ flex: 1, minWidth: 0 }}> + {detailContent} + {showRetry && ( + <button type="button" data-testid="retry" onClick={() => void detail.refetch()}> + retry + </button> + )} + {selectedId !== null && ( + <div> + <button + type="button" + data-testid="assign" + onClick={() => assignIssue.mutate({ id: selectedId, who: user })} + > + assign to {user} + </button> + {assignIssue.isError && <p>detail error: {errorMessage(assignIssue.error)}</p>} + </div> + )} + </section> + </div> + </main> + ); +} + +export default function App() { + const [client] = useState( + () => + new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false }, + mutations: { retry: false }, + }, + }), + ); + return ( + <QueryClientProvider client={client}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/runlog.jsonl new file mode 100644 index 0000000..4e49967 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-2/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:37:50.324Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:37:59.888Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/App.tsx b/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/App.tsx new file mode 100644 index 0000000..07e6f6e --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/App.tsx @@ -0,0 +1,224 @@ +import { useState } from "react"; +import { + QueryClient, + QueryClientProvider, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type AssignError = { id: number; message: string }; + +const errorMessage = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +export default function App() { + const [queryClient] = useState( + () => + new QueryClient({ + defaultOptions: { + queries: { retry: false, refetchOnWindowFocus: false }, + mutations: { retry: false }, + }, + }), + ); + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + const [assignError, setAssignError] = useState<AssignError | null>(null); + + const select = (id: number) => { + setSelected(id); + setAssignError(null); + }; + + const assignMutation = useMutation({ + mutationFn: ({ id, user }: { id: number; user: string }) => assign(id, user), + onMutate: async ({ id, user }) => { + await queryClient.cancelQueries({ queryKey: ["issues"] }); + const snapshot = queryClient.getQueriesData<Issue[]>({ queryKey: ["issues"] }); + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, list => + list?.map(i => (i.id === id ? { ...i, assignee: user } : i)), + ); + return { snapshot }; + }, + onSuccess: ({ id, assignee }) => { + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, list => + list?.map(i => (i.id === id ? { ...i, assignee } : i)), + ); + }, + onError: (err, { id }, context) => { + for (const [key, data] of context?.snapshot ?? []) { + const before = data?.find(i => i.id === id); + if (!before) continue; + queryClient.setQueryData<Issue[]>(key, list => + list?.map(i => (i.id === id ? { ...i, assignee: before.assignee } : i)), + ); + } + setAssignError({ id, message: errorMessage(err) }); + }, + }); + + const onAssign = () => { + if (selected === null) return; + setAssignError(null); + assignMutation.mutate({ id: selected, user }); + }; + + return ( + <main style={{ fontFamily: "system-ui, sans-serif", padding: 16, maxWidth: 960, margin: "0 auto" }}> + <h1 style={{ fontSize: 22 }}>Issue tracker</h1> + <label style={{ display: "block", marginBottom: 12 }}> + Current user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div style={{ display: "flex", gap: 8, marginBottom: 12 }}> + <button + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div style={{ display: "flex", gap: 24, alignItems: "flex-start", flexWrap: "wrap" }}> + <IssueList filter={filter} selected={selected} onSelect={select} /> + <section data-testid="detail" style={{ flex: "1 1 320px", minWidth: 0 }}> + {selected === null ? ( + <p>select an issue</p> + ) : ( + <IssueDetail + key={selected} + id={selected} + user={user} + assignError={assignError?.id === selected ? assignError.message : null} + onAssign={onAssign} + /> + )} + </section> + </div> + </main> + ); +} + +function IssueList({ + filter, + selected, + onSelect, +}: { + filter: Filter; + selected: number | null; + onSelect: (id: number) => void; +}) { + const query = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + let content; + if (query.isFetching) { + content = <p>{`loading ${filter} issues`}</p>; + } else if (query.isError) { + content = <p>{`list error: ${errorMessage(query.error)}`}</p>; + } else { + content = ( + <ul style={{ listStyle: "none", padding: 0, margin: 0 }}> + {(query.data ?? []).map(issue => ( + <li key={issue.id} style={{ marginBottom: 4 }}> + <button + data-testid="issue" + aria-current={selected === issue.id ? "true" : undefined} + onClick={() => onSelect(issue.id)} + style={{ fontWeight: selected === issue.id ? 700 : 400 }} + > + {`#${issue.id} ${issue.title} (${issue.assignee ?? "unassigned"})`} + </button> + </li> + ))} + </ul> + ); + } + + return ( + <section data-testid="list" style={{ flex: "1 1 320px", minWidth: 0 }}> + {content} + </section> + ); +} + +function IssueDetail({ + id, + user, + assignError, + onAssign, +}: { + id: number; + user: string; + assignError: string | null; + onAssign: () => void; +}) { + const query = useQuery({ + queryKey: ["issue", id], + queryFn: () => fetchIssue(id), + }); + + let body; + if (query.isFetching) { + body = <p>{`loading #${id}`}</p>; + } else if (query.isError) { + body = ( + <> + <p>{`detail error: ${errorMessage(query.error)}`}</p> + <button data-testid="retry" onClick={() => void query.refetch()}> + retry + </button> + </> + ); + } else if (query.data === null || query.data === undefined) { + body = ( + <> + <p>{`detail error: issue #${id} not found`}</p> + <button data-testid="retry" onClick={() => void query.refetch()}> + retry + </button> + </> + ); + } else { + body = ( + <> + <h2 style={{ fontSize: 18, margin: "0 0 8px" }}>{`#${query.data.id} ${query.data.title}`}</h2> + <p style={{ whiteSpace: "pre-wrap" }}>{query.data.body}</p> + </> + ); + } + + return ( + <div> + {body} + <div style={{ marginTop: 12 }}> + <button data-testid="assign" onClick={onAssign}> + {`assign to ${user}`} + </button> + </div> + {assignError !== null && <p>{`detail error: ${assignError}`}</p>} + </div> + ); +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/runlog.jsonl new file mode 100644 index 0000000..69f04c5 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-react-3/runlog.jsonl @@ -0,0 +1,2 @@ +{"at":"2026-10-27T20:40:03.687Z","cmd":"check","ok":true,"out":"ok"} +{"at":"2026-10-27T20:40:12.159Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/main.rx b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/main.rx new file mode 100644 index 0000000..ca88d58 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/main.rx @@ -0,0 +1,83 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign, IssueDetail } from "./api" + +error IssueNotFound(id: number) => `issue #${id} not found` + +fn found(it: IssueDetail | null, id: number) => it == null ? throw IssueNotFound(id) : it + +// a guessed or accepted assignee for one issue, kept as a list of overrides +fn withOverride(list: { id: number; assignee: string | null }[], id: number, who: string | null) => + [...list.filter(o => o.id != id), { id: id, assignee: who }] + +fn assigned(list: { id: number; assignee: string | null }[], id: number, own: string | null) => + list.filter(o => o.id == id).length > 0 ? list.filter(o => o.id == id)[0].assignee : own + +export default component App() { + source user: string := "guest" + source status: string := "open" + source selected: number | null := null + + // the rows for the current filter, from the server + issues = await fetchIssues(status) + + // the detail of the selected issue; `null` selects nothing + detail = selected == null ? null : found(await fetchIssue(selected), selected) + + // assignments made in this session: the guess until the server answers, then the server's answer + source overrides: { id: number; assignee: string | null }[] = [] + source assignError: string = "" + + action assignIssue(id: number, who: string) { + try { + optimistic assignError := "" + optimistic overrides := withOverride(overrides, id, who) + let answer = await assign(id, who) + overrides := withOverride(overrides, id, answer.assignee) + } catch (e) { + assignError := e.message + } + } + + <main> + <h1>Issue tracker</h1> + <label>user <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /></label> + <div> + <button data-testid="filter-open" onClick={() => { status := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { status := "closed" }}>closed</button> + </div> + <div class="panes"> + <section data-testid="list"> + {loading on(status) fallback { <p>loading {status} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for i in issues { + <li data-testid="issue" onClick={() => { selected := i.id; assignError := "" }}> + #{i.id} {i.title} ({assigned(overrides, i.id, i.assignee) ?? "unassigned"}) + </li> + }} + </ul> + }} + }} + </section> + <section data-testid="detail"> + {if selected == null { + <p>select an issue</p> + } else { + <div> + {loading on(selected) fallback { <p>loading #{selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + {if detail != null { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + <button data-testid="assign" onClick={() => { assignIssue(selected, user) }}>assign to {user}</button> + <p>{assignError}</p> + </div> + }} + </section> + </div> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl new file mode 100644 index 0000000..5ac054b --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-1/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T20:36:27.173Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/overlap-small/tracker-rx-1/src/main.rx: line 26: field near '[id]: who }\n le'"} +{"at":"2026-10-27T20:36:35.475Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:36:44.393Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:36:57.475Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:37:06.452Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/main.rx b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/main.rx new file mode 100644 index 0000000..d0c6eeb --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/main.rx @@ -0,0 +1,73 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign, Issue, IssueDetail } from "./api" +import pure { indexOf } from "./prelude" + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it: IssueDetail | null, id: number): IssueDetail => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source status: string := "open" + source issues: Issue[] key id = await fetchIssues(status) + source selectedId: number | null := null + source assignError: string := "" + + detail = selectedId == null ? null : found(await fetchIssue(selectedId ?? 0), selectedId ?? 0) + + action assignIssue(id: number, who: string) { + assignError := "" + try { + let i = indexOf(issues, t => t.id == id) + if (i >= 0) { + optimistic issues[i].assignee := who + } + await assign(id, who) + refresh issues + } catch (e) { + assignError := e.message + } + } + + <main> + <label> + User <input data-testid="user" value={user} onInput={e => { user := e.target.value }} /> + </label> + <div> + <button data-testid="filter-open" onClick={() => { status := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { status := "closed" }}>closed</button> + </div> + + <section data-testid="list"> + {errored fallback(e) { <p>{`list error: ${e.message}`}</p> } { + {loading on(status) fallback { <p>{`loading ${status} issues`}</p> } { + <ul> + {for issue in issues { + <li data-testid="issue" onClick={() => { selectedId := issue.id; assignError := "" }}> + {`#${issue.id} ${issue.title} (${issue.assignee ?? "unassigned"})`} + </li> + }} + </ul> + }} + }} + </section> + + <section data-testid="detail"> + {if selectedId == null { + <p>select an issue</p> + } else { + {errored fallback(e) { <p>{`detail error: ${e.message}`}</p> } { + {loading on(selectedId) fallback { <p>{`loading #${selectedId}`}</p> } { + <h2>{`#${detail?.id} ${detail?.title}`}</h2> + <p>{detail?.body}</p> + }} + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + <button data-testid="assign" onClick={() => { assignIssue(selectedId ?? 0, user) }}>{`assign to ${user}`}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl new file mode 100644 index 0000000..5130a65 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-2/runlog.jsonl @@ -0,0 +1,5 @@ +{"at":"2026-10-27T20:40:31.442Z","cmd":"check","ok":false,"out":"src/main.rx: parse error: /home/claude/runs/overlap-small/tracker-rx-2/src/main.rx: line 34: expected ')' near ': any) => { user := '"} +{"at":"2026-10-27T20:40:35.401Z","cmd":"check","ok":false,"out":"/home/claude/runs/overlap-small/tracker-rx-2/src/main.rx:18: error OPTIMISTIC_NO_QUESTION [AC-3] assignError is declared with := — a guess needs a question to confirm it (source assignError = …, ST-5c)"} +{"at":"2026-10-27T20:40:39.630Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:49.520Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} +{"at":"2026-10-27T20:41:03.597Z","cmd":"test","ok":true,"build":true,"passed":3,"total":3} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/main.rx b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/main.rx new file mode 100644 index 0000000..e4eb244 --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/main.rx @@ -0,0 +1,81 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign } from "./api" + +type Issue = { id: number; title: string; assignee: string | null } +type IssueDetail = { id: number; title: string; body: string } + +error NotFound(id: number) => `issue #${id} not found` + +fn found(it: IssueDetail | null, id: number): IssueDetail => it == null ? throw NotFound(id) : it + +export default component App() { + source user := "guest" + source filterName := "open" + source issues: Issue[] key id = await fetchIssues(filterName) + source selectedId: number | null := null + source assignError: string := "" + source overlay: { id: number; user: string } | null := null + + detailIssue = selectedId == null ? null : found(await fetchIssue(selectedId), selectedId) + + action assignSelected(id: number, who: string) { + try { + await assign(id, who) + } catch (e) { + if (overlay != null && overlay.id == id) { overlay := null } + assignError := e.message + } + } + + <main> + <label>current user <input data-testid="user" value={user} onInput={(e) => { user := e.target.value }} /></label> + + <div> + <button data-testid="filter-open" onClick={() => { filterName := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filterName := "closed" }}>closed</button> + </div> + + <div data-testid="list"> + {loading on(filterName) fallback { <p>loading {filterName} issues</p> } { + errored fallback(e) { <p>list error: {e.message}</p> } { + {for i in issues { + <button data-testid="issue" onClick={() => { + selectedId := i.id + onFlush(() => { assignError := "" }) + }}>#{i.id} {i.title} ({(overlay != null && overlay.id == i.id ? overlay.user : i.assignee) ?? "unassigned"})</button> + }} + } + }} + </div> + + <section data-testid="detail"> + {if selectedId == null { + <p>select an issue</p> + } else { + <div> + {loading on(selectedId) fallback { <p>loading #{selectedId}</p> } { + errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detailIssue.id} {detailIssue.title}</h2> + <p>{detailIssue.body}</p> + } + }} + <button data-testid="retry" onClick={() => { refresh detailIssue }}>retry</button> + <button data-testid="assign" onClick={() => { + let id = selectedId + let who = user + if (id != null) { + onFlush(() => { + assignError := "" + overlay := { id: id, user: who } + }) + assignSelected(id, who) + } + }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + </div> + }} + </section> + </main> +} diff --git a/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl new file mode 100644 index 0000000..242920d --- /dev/null +++ b/eval/results/overlap-2026-10-27/runs/small/tracker-rx-3/runlog.jsonl @@ -0,0 +1,15 @@ +{"at":"2026-10-27T20:40:25.448Z","cmd":"check","ok":false,"out":"/home/claude/runs/overlap-small/tracker-rx-3/src/main.rx:21: error UNCAUGHT_ACTION_FAILURE [R-f-6] action assignSelected can fail with Error and does not catch it: a handler cannot (it does not await the action) — `try { … } catch (e) { … }` in the body"} +{"at":"2026-10-27T20:40:29.930Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:38.208Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:40:46.313Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:40:51.842Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:40:59.195Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:41:05.291Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:41:11.786Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:41:17.613Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:41:28.764Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:41:34.581Z","cmd":"test","ok":false,"build":true,"passed":2,"total":4} +{"at":"2026-10-27T20:41:55.403Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:42:00.605Z","cmd":"test","ok":false,"build":true,"passed":3,"total":4} +{"at":"2026-10-27T20:43:00.710Z","cmd":"check","ok":true,"out":"src/main.rx: ok"} +{"at":"2026-10-27T20:43:06.101Z","cmd":"test","ok":true,"build":true,"passed":4,"total":4} diff --git a/eval/results/overlap-2026-10-27/usage.txt b/eval/results/overlap-2026-10-27/usage.txt new file mode 100644 index 0000000..2d5875e --- /dev/null +++ b/eval/results/overlap-2026-10-27/usage.txt @@ -0,0 +1,36 @@ +big search-react-1 tokens=44680 +big settings-react-1 tokens=46282 +big settings-rx-1 tokens=58777 +big search-rx-1 tokens=59411 +big tracker-react-1 tokens=48450 +small settings-react-1 tokens=50436 +big settings-react-2 tokens=45022 +small tracker-react-1 tokens=55300 +big search-react-2 tokens=46336 +small search-react-1 tokens=54508 +small settings-rx-1 tokens=75902 +big search-react-3 tokens=46566 +big tracker-rx-1 tokens=74690 +small settings-react-2 tokens=49441 +big search-rx-2 tokens=66153 +big settings-rx-2 tokens=59469 +big settings-react-3 tokens=45106 +small tracker-rx-1 tokens=81095 +big tracker-react-2 tokens=49435 +small search-rx-1 tokens=85195 +small tracker-react-2 tokens=59055 +big tracker-rx-2 tokens=70147 +big tracker-react-3 tokens=47135 +small settings-react-3 tokens=51797 +small search-react-3 tokens=52520 +big settings-rx-3 tokens=66702 +small search-react-2 tokens=53587 +big tracker-rx-3 tokens=74251 +small tracker-react-3 tokens=58601 (report on resume) +big search-rx-3 tokens=61865 +small search-rx-2 tokens=78533 +small search-rx-3 tokens=82162 +small settings-rx-3 tokens=95706 (report on resume) +small tracker-rx-2 tokens=83638 +small settings-rx-2 tokens=92497 +small tracker-rx-3 tokens=111053 From 4ac4b1f7cac183adb1fbc0df7539e2003f6c4dcd Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 21:38:26 +0000 Subject: [PATCH 07/11] eval: round 4 draft (change an existing app): rename, close, comments Three change requests on round 3's frontier tracker apps, with plain briefs, visible tests, hidden tests (the base app's as regressions) and overlap tests. References in both arms: React passes everything; .rx passes everything but the list-reload cases (solidjs/solid#4329). scaffold --base copies an existing app into the run and gives the agent its own .scratch folder. Not pre-registered yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/harness/prompt-change.md | 17 ++ eval/harness/scaffold.mjs | 20 ++- eval/reference/close/react/App.tsx | 205 +++++++++++++++++++++ eval/reference/close/rx/main.rx | 100 +++++++++++ eval/reference/comments/react/App.tsx | 236 +++++++++++++++++++++++++ eval/reference/comments/rx/main.rx | 122 +++++++++++++ eval/reference/rename/react/App.tsx | 229 ++++++++++++++++++++++++ eval/reference/rename/rx/main.rx | 101 +++++++++++ eval/tasks/close/BASE.md | 39 ++++ eval/tasks/close/api.d.rx | 7 + eval/tasks/close/api.d.ts | 6 + eval/tasks/close/api.js | 6 + eval/tasks/close/brief-plain.md | 24 +++ eval/tasks/close/fixtures.mjs | 12 ++ eval/tasks/close/hidden-overlap.mjs | 55 ++++++ eval/tasks/close/hidden.mjs | 41 +++++ eval/tasks/close/visible.mjs | 48 +++++ eval/tasks/comments/BASE.md | 39 ++++ eval/tasks/comments/api.d.rx | 9 + eval/tasks/comments/api.d.ts | 8 + eval/tasks/comments/api.js | 7 + eval/tasks/comments/brief-plain.md | 31 ++++ eval/tasks/comments/fixtures.mjs | 15 ++ eval/tasks/comments/hidden-overlap.mjs | 52 ++++++ eval/tasks/comments/hidden.mjs | 61 +++++++ eval/tasks/comments/visible.mjs | 52 ++++++ eval/tasks/rename/BASE.md | 39 ++++ eval/tasks/rename/api.d.rx | 8 + eval/tasks/rename/api.d.ts | 6 + eval/tasks/rename/api.js | 6 + eval/tasks/rename/brief-plain.md | 25 +++ eval/tasks/rename/fixtures.mjs | 11 ++ eval/tasks/rename/hidden-overlap.mjs | 61 +++++++ eval/tasks/rename/hidden.mjs | 48 +++++ eval/tasks/rename/visible.mjs | 49 +++++ 35 files changed, 1789 insertions(+), 6 deletions(-) create mode 100644 eval/harness/prompt-change.md create mode 100644 eval/reference/close/react/App.tsx create mode 100644 eval/reference/close/rx/main.rx create mode 100644 eval/reference/comments/react/App.tsx create mode 100644 eval/reference/comments/rx/main.rx create mode 100644 eval/reference/rename/react/App.tsx create mode 100644 eval/reference/rename/rx/main.rx create mode 100644 eval/tasks/close/BASE.md create mode 100644 eval/tasks/close/api.d.rx create mode 100644 eval/tasks/close/api.d.ts create mode 100644 eval/tasks/close/api.js create mode 100644 eval/tasks/close/brief-plain.md create mode 100644 eval/tasks/close/fixtures.mjs create mode 100644 eval/tasks/close/hidden-overlap.mjs create mode 100644 eval/tasks/close/hidden.mjs create mode 100644 eval/tasks/close/visible.mjs create mode 100644 eval/tasks/comments/BASE.md create mode 100644 eval/tasks/comments/api.d.rx create mode 100644 eval/tasks/comments/api.d.ts create mode 100644 eval/tasks/comments/api.js create mode 100644 eval/tasks/comments/brief-plain.md create mode 100644 eval/tasks/comments/fixtures.mjs create mode 100644 eval/tasks/comments/hidden-overlap.mjs create mode 100644 eval/tasks/comments/hidden.mjs create mode 100644 eval/tasks/comments/visible.mjs create mode 100644 eval/tasks/rename/BASE.md create mode 100644 eval/tasks/rename/api.d.rx create mode 100644 eval/tasks/rename/api.d.ts create mode 100644 eval/tasks/rename/api.js create mode 100644 eval/tasks/rename/brief-plain.md create mode 100644 eval/tasks/rename/fixtures.mjs create mode 100644 eval/tasks/rename/hidden-overlap.mjs create mode 100644 eval/tasks/rename/hidden.mjs create mode 100644 eval/tasks/rename/visible.mjs diff --git a/eval/harness/prompt-change.md b/eval/harness/prompt-change.md new file mode 100644 index 0000000..11ef703 --- /dev/null +++ b/eval/harness/prompt-change.md @@ -0,0 +1,17 @@ +You are a developer changing a small web app that already works. Work only inside the directory {{DIR}}: read and +write files there and nowhere else{{DOCS_RULE}}. Keep any scratch files in {{DIR}}/.scratch, and use no other +temporary folder. Do not look for other solutions, tests or examples anywhere on this machine. + +Read {{DIR}}/BRIEF.md: it is the change you are asked to make, and the whole requirement for it. {{DIR}}/BASE.md is +the brief the app was built to; the app must keep doing everything it says. The app is written in {{LANG_RULE}} + +`src/api.js` is the backend's API and `src/flights.js` is its plumbing; do not change either. Their types are in +{{API_TYPES}}. + +Two commands, run from {{DIR}}: +- `./check` runs {{CHECKER}}. +- `./test` builds the app and runs the visible tests in a browser against a fake backend{{BACKEND_NOTE}} + (see `tests/`). The visible tests cover only part of the briefs: a run is judged by both briefs, with more tests. + +You may run `./check` and `./test` at most 15 times in total. When you believe the app does everything both briefs +say, stop and reply with two or three sentences on what you changed and anything you were unsure about. diff --git a/eval/harness/scaffold.mjs b/eval/harness/scaffold.mjs index fa5efe2..e9dab01 100644 --- a/eval/harness/scaffold.mjs +++ b/eval/harness/scaffold.mjs @@ -2,7 +2,9 @@ // agent may run. With --reference, also copies in the reference solution (eval/reference/<task>/<arm>). // --brief <name> uses tasks/<task>/brief-<name>.md (e.g. `plain`) instead of brief.md. Writes the agent's prompt to // <dir>/.prompt. -// Usage: node harness/scaffold.mjs <task> <rx|react> <dir> [--reference] [--brief <name>] +// --base <dir> makes a change task (round 4): it copies an existing app's source files from <dir> into src/, and the +// task's BASE.md (the brief that app was built to) next to BRIEF.md, and the prompt asks for a change, not a new app. +// Usage: node harness/scaffold.mjs <task> <rx|react> <dir> [--reference] [--brief <name>] [--base <dir>] import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -10,7 +12,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const here = path.dirname(fileURLToPath(import.meta.url)); const evalDir = path.resolve(here, ".."); -export function scaffold(task, arm, dir, { reference = false, brief = null } = {}) { +export function scaffold(task, arm, dir, { reference = false, brief = null, base = null } = {}) { const t = path.join(evalDir, "tasks", task); dir = path.resolve(dir); mkdirSync(path.join(dir, "src"), { recursive: true }); mkdirSync(path.join(dir, "tests"), { recursive: true }); @@ -37,14 +39,19 @@ export function scaffold(task, arm, dir, { reference = false, brief = null } = { cpSync(path.join(evalDir, "../guide"), path.join(dir, "docs/guide"), { recursive: true }); } if (reference) cpSync(path.join(evalDir, "reference", task, arm), path.join(dir, "src"), { recursive: true }); - writeFileSync(path.join(dir, ".prompt"), prompt(arm, dir, { plain: brief === "plain" })); + if (base) { + cpSync(path.join(t, "BASE.md"), path.join(dir, "BASE.md")); + for (const f of readdirSync(base)) if (/\.(rx|tsx?)$/.test(f)) cpSync(path.join(base, f), path.join(dir, "src", f)); + mkdirSync(path.join(dir, ".scratch")); + } + writeFileSync(path.join(dir, ".prompt"), prompt(arm, dir, { plain: brief === "plain", change: !!base })); return dir; } -export function prompt(arm, dir, { plain = false } = {}) { +export function prompt(arm, dir, { plain = false, change = false } = {}) { const fill = { ...JSON.parse(readFileSync(path.join(here, "prompt-arms.json"), "utf8"))[arm] }; fill.BACKEND_NOTE = plain ? "" : " whose answers the tests control"; - let text = readFileSync(path.join(here, "prompt.md"), "utf8"); + let text = readFileSync(path.join(here, change ? "prompt-change.md" : "prompt.md"), "utf8"); for (let i = 0; i < 2; i++) for (const k of [...Object.keys(fill), "DIR"]) text = text.replaceAll(`{{${k}}}`, k === "DIR" ? dir : fill[k]); return text; } @@ -52,5 +59,6 @@ export function prompt(arm, dir, { plain = false } = {}) { if (import.meta.url === pathToFileURL(process.argv[1]).href) { const [task, arm, dir] = process.argv.slice(2); const b = process.argv.indexOf("--brief"); - console.log(scaffold(task, arm, dir, { reference: process.argv.includes("--reference"), brief: b > 0 ? process.argv[b + 1] : null })); + const base = process.argv.indexOf("--base"); + console.log(scaffold(task, arm, dir, { reference: process.argv.includes("--reference"), brief: b > 0 ? process.argv[b + 1] : null, base: base > 0 ? path.resolve(process.argv[base + 1]) : null })); } diff --git a/eval/reference/close/react/App.tsx b/eval/reference/close/react/App.tsx new file mode 100644 index 0000000..51bc183 --- /dev/null +++ b/eval/reference/close/react/App.tsx @@ -0,0 +1,205 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from "@tanstack/react-query"; +import { assign, closeIssue, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type Pending = { seq: number; user: string }; +type AssignError = { issueId: number; message: string }; + +const errorMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + // optimistic assignments still in the air, per issue, oldest first + const [pending, setPending] = useState<Record<number, Pending[]>>({}); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seqRef = useRef(0); + const confirmedSeqRef = useRef<Record<number, number>>({}); + // closes still in the air, by issue id, with how many are out + const [closing, setClosing] = useState<Record<number, number>>({}); + const [closeError, setCloseError] = useState<AssignError | null>(null); + const closeSeqRef = useRef(0); + + const list = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + const detail = useQuery({ + queryKey: ["issue", selected], + queryFn: async () => { + const id = selected as number; + const d = await fetchIssue(id); + if (d === null) throw new Error(`issue #${id} not found`); + return d; + }, + enabled: selected !== null, + }); + + const assigneeOf = (issue: Issue) => { + const p = pending[issue.id]; + if (p && p.length > 0) return p[p.length - 1].user; + return issue.assignee; + }; + + const select = (id: number) => { + setSelected(id); + setAssignError(null); + setCloseError(null); + }; + + const doClose = async (id: number) => { + const seq = ++closeSeqRef.current; + setCloseError(null); + setClosing(prev => ({ ...prev, [id]: (prev[id] ?? 0) + 1 })); + const done = () => + setClosing(prev => { + const next = { ...prev }; + if ((next[id] ?? 0) > 1) next[id] -= 1; + else delete next[id]; + return next; + }); + try { + await closeIssue(id); + queryClient.setQueryData<Issue[]>(["issues", "open"], old => old?.filter(i => i.id !== id)); + done(); + } catch (e) { + done(); + if (closeSeqRef.current === seq) setCloseError({ issueId: id, message: errorMessage(e) }); + } + }; + + const doAssign = async (id: number, who: string) => { + const seq = ++seqRef.current; + setAssignError(null); + setPending(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { seq, user: who }] })); + const done = () => + setPending(prev => { + const rest = (prev[id] ?? []).filter(p => p.seq !== seq); + const next = { ...prev }; + if (rest.length) next[id] = rest; + else delete next[id]; + return next; + }); + try { + const res = await assign(id, who); + if ((confirmedSeqRef.current[id] ?? 0) < seq) { + confirmedSeqRef.current[id] = seq; + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === id ? { ...i, assignee: res.assignee } : i)), + ); + } + done(); + } catch (e) { + done(); + // only report the failure if no later assignment has started since + if (seqRef.current === seq) setAssignError({ issueId: id, message: errorMessage(e) }); + } + }; + + let listContent; + if (list.isPending) listContent = <p>loading {filter} issues</p>; + else if (list.isError) listContent = <p>list error: {errorMessage(list.error)}</p>; + else + listContent = ( + <ul> + {list.data.filter(issue => !(filter === "open" && closing[issue.id])).map(issue => ( + <li key={issue.id}> + <button + type="button" + data-testid="issue" + aria-pressed={issue.id === selected} + onClick={() => select(issue.id)} + > + #{issue.id} {issue.title} ({assigneeOf(issue) ?? "unassigned"}) + </button> + </li> + ))} + </ul> + ); + + let detailContent; + if (selected === null) detailContent = <p>select an issue</p>; + else { + let body; + if (detail.isFetching || detail.isPending) body = <p>loading #{selected}</p>; + else if (detail.isError) body = <p>detail error: {errorMessage(detail.error)}</p>; + else + body = ( + <> + <h2> + #{detail.data.id} {detail.data.title} + </h2> + <p>{detail.data.body}</p> + </> + ); + const id = selected; + detailContent = ( + <> + {body} + <button type="button" data-testid="retry" onClick={() => void detail.refetch()}> + retry + </button> + <button type="button" data-testid="assign" onClick={() => void doAssign(id, user)}> + assign to {user} + </button> + {assignError && assignError.issueId === selected && ( + <p data-testid="assign-error">{assignError.message}</p> + )} + <button type="button" data-testid="close" onClick={() => void doClose(id)}> + close + </button> + {closeError && closeError.issueId === selected && <p>close error: {closeError.message}</p>} + </> + ); + } + + return ( + <div style={{ display: "flex", gap: "2rem" }}> + <section> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + <section data-testid="detail">{detailContent}</section> + </div> + ); +} + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false, refetchOnReconnect: false }, + }, +}); + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/reference/close/rx/main.rx b/eval/reference/close/rx/main.rx new file mode 100644 index 0000000..08d0fa6 --- /dev/null +++ b/eval/reference/close/rx/main.rx @@ -0,0 +1,100 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign, closeIssue } from "./api" +import pure { indexOf, map } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: string = "" + source closeError: string = "" + // one flag per listed issue: a close guesses its own row's flag + source closing key id = map(issues, t => { id: t.id, on: false }) + + action closeOne(id: number) { + optimistic closeError := "" + try { + let i = indexOf(closing, c => c.id == id) + if (i >= 0) { + optimistic closing[i].on := true + } + await closeIssue(id) + refresh issues + } catch (e) { + closeError := e.message + } + } + + fn isClosing(id: number) => filter == "open" && indexOf(closing, c => c.id == id && c.on) >= 0 + + action assignTo(id: number, who: string) { + optimistic assignError := "" + try { + let i = indexOf(issues, t => t.id == id) + optimistic issues[i].assignee := who + await assign(id, who) + refresh issues + } catch (e) { + assignError := e.message + } + } + + <main> + <div> + <label> + user{" "} + <input data-testid="user" value={user} onInput={e => { user := e.target.value }} /> + </label> + </div> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div data-testid="list"> + {loading on(filter) fallback { <p>loading {latest filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for it in issues { + {if !isClosing(it.id) { + <li data-testid="issue" onClick={() => { + selected := it.id + onFlush(() => { + assignError := "" + closeError := "" + }) + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + }} + </ul> + }} + }} + </div> + <div data-testid="detail"> + {if selected != null { + detail = found(await fetchIssue(selected), selected) + {loading on(selected) fallback { <p>loading #{latest selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + }} + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + <button data-testid="close" onClick={() => { closeOne(selected) }}>close</button> + {if closeError != "" { + <p>close error: {closeError}</p> + }} + } else { + <p>select an issue</p> + }} + </div> + </main> +} diff --git a/eval/reference/comments/react/App.tsx b/eval/reference/comments/react/App.tsx new file mode 100644 index 0000000..17ee082 --- /dev/null +++ b/eval/reference/comments/react/App.tsx @@ -0,0 +1,236 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from "@tanstack/react-query"; +import { addComment, assign, fetchComments, fetchIssue, fetchIssues } from "./api.js"; +import type { Issue } from "./api.js"; + +type Filter = "open" | "closed"; +type Pending = { seq: number; user: string }; +type AssignError = { issueId: number; message: string }; +type Sending = { seq: number; issueId: number; author: string; text: string }; + +const errorMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + // optimistic assignments still in the air, per issue, oldest first + const [pending, setPending] = useState<Record<number, Pending[]>>({}); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seqRef = useRef(0); + const confirmedSeqRef = useRef<Record<number, number>>({}); + // posts still in the air, oldest first + const [sending, setSending] = useState<Sending[]>([]); + const [postError, setPostError] = useState<AssignError | null>(null); + const [commentInput, setCommentInput] = useState(""); + const postSeqRef = useRef(0); + + const list = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + const detail = useQuery({ + queryKey: ["issue", selected], + queryFn: async () => { + const id = selected as number; + const d = await fetchIssue(id); + if (d === null) throw new Error(`issue #${id} not found`); + return d; + }, + enabled: selected !== null, + }); + + const comments = useQuery({ + queryKey: ["comments", selected], + queryFn: () => fetchComments(selected as number), + enabled: selected !== null, + }); + + const doPost = async (id: number, author: string, text: string) => { + const seq = ++postSeqRef.current; + setPostError(null); + setCommentInput(""); + setSending(prev => [...prev, { seq, issueId: id, author, text }]); + try { + const res = await addComment(id, author, text); + queryClient.setQueryData<{ author: string; text: string }[]>(["comments", id], old => (old ? [...old, res] : old)); + } catch (e) { + if (postSeqRef.current === seq) setPostError({ issueId: id, message: errorMessage(e) }); + } finally { + setSending(prev => prev.filter(p => p.seq !== seq)); + } + }; + + const assigneeOf = (issue: Issue) => { + const p = pending[issue.id]; + if (p && p.length > 0) return p[p.length - 1].user; + return issue.assignee; + }; + + const select = (id: number) => { + setSelected(id); + setAssignError(null); + setPostError(null); + }; + + const doAssign = async (id: number, who: string) => { + const seq = ++seqRef.current; + setAssignError(null); + setPending(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { seq, user: who }] })); + const done = () => + setPending(prev => { + const rest = (prev[id] ?? []).filter(p => p.seq !== seq); + const next = { ...prev }; + if (rest.length) next[id] = rest; + else delete next[id]; + return next; + }); + try { + const res = await assign(id, who); + if ((confirmedSeqRef.current[id] ?? 0) < seq) { + confirmedSeqRef.current[id] = seq; + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === id ? { ...i, assignee: res.assignee } : i)), + ); + } + done(); + } catch (e) { + done(); + // only report the failure if no later assignment has started since + if (seqRef.current === seq) setAssignError({ issueId: id, message: errorMessage(e) }); + } + }; + + let listContent; + if (list.isPending) listContent = <p>loading {filter} issues</p>; + else if (list.isError) listContent = <p>list error: {errorMessage(list.error)}</p>; + else + listContent = ( + <ul> + {list.data.map(issue => ( + <li key={issue.id}> + <button + type="button" + data-testid="issue" + aria-pressed={issue.id === selected} + onClick={() => select(issue.id)} + > + #{issue.id} {issue.title} ({assigneeOf(issue) ?? "unassigned"}) + </button> + </li> + ))} + </ul> + ); + + let detailContent; + if (selected === null) detailContent = <p>select an issue</p>; + else { + let body; + if (detail.isFetching || detail.isPending) body = <p>loading #{selected}</p>; + else if (detail.isError) body = <p>detail error: {errorMessage(detail.error)}</p>; + else + body = ( + <> + <h2> + #{detail.data.id} {detail.data.title} + </h2> + <p>{detail.data.body}</p> + </> + ); + const id = selected; + detailContent = ( + <> + {body} + <button type="button" data-testid="retry" onClick={() => void detail.refetch()}> + retry + </button> + <button type="button" data-testid="assign" onClick={() => void doAssign(id, user)}> + assign to {user} + </button> + {assignError && assignError.issueId === selected && ( + <p data-testid="assign-error">{assignError.message}</p> + )} + <div data-testid="comments"> + {comments.isPending ? ( + <p>loading comments</p> + ) : comments.isError ? ( + <p>comments error: {errorMessage(comments.error)}</p> + ) : ( + <ul> + {comments.data.map((c, i) => ( + <li key={i} data-testid="comment"> + {c.author}: {c.text} + </li> + ))} + {sending + .filter(p => p.issueId === id) + .map(p => ( + <li key={`s${p.seq}`} data-testid="comment"> + {p.author}: {p.text} (sending…) + </li> + ))} + </ul> + )} + </div> + <input data-testid="comment-input" value={commentInput} onChange={e => setCommentInput(e.target.value)} /> + <button + type="button" + data-testid="post" + onClick={() => { + if (commentInput !== "") void doPost(id, user, commentInput); + }} + > + post + </button> + {postError && postError.issueId === selected && <p>post error: {postError.message}</p>} + </> + ); + } + + return ( + <div style={{ display: "flex", gap: "2rem" }}> + <section> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + <section data-testid="detail">{detailContent}</section> + </div> + ); +} + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false, refetchOnReconnect: false }, + }, +}); + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/reference/comments/rx/main.rx b/eval/reference/comments/rx/main.rx new file mode 100644 index 0000000..177b3f3 --- /dev/null +++ b/eval/reference/comments/rx/main.rx @@ -0,0 +1,122 @@ +import pure { fetchIssues, fetchIssue, fetchComments } from "./api" +import { assign, addComment } from "./api" +import pure { indexOf, filter as keep } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: string = "" + source postError: string = "" + source commentInput: string := "" + source comments = selected == null ? [] : await fetchComments(selected) + + // posts in the air: shown under the selected issue's comments until their action ends + source sending: { id: number; author: string; text: string }[] := [] + + action postComment(id: number, author: string, text: string) { + optimistic postError := "" + try { + await addComment(id, author, text) + refresh comments + } catch (e) { + postError := e.message + } + sending := keep(sending, p => !(p.id == id && p.author == author && p.text == text)) + } + + action assignTo(id: number, who: string) { + optimistic assignError := "" + try { + let i = indexOf(issues, t => t.id == id) + optimistic issues[i].assignee := who + await assign(id, who) + refresh issues + } catch (e) { + assignError := e.message + } + } + + <main> + <div> + <label> + user{" "} + <input data-testid="user" value={user} onInput={e => { user := e.target.value }} /> + </label> + </div> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div data-testid="list"> + {loading on(filter) fallback { <p>loading {latest filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for it in issues { + <li data-testid="issue" onClick={() => { + selected := it.id + onFlush(() => { + assignError := "" + postError := "" + }) + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + </ul> + }} + }} + </div> + <div data-testid="detail"> + {if selected != null { + detail = found(await fetchIssue(selected), selected) + {loading on(selected) fallback { <p>loading #{latest selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + }} + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + <div data-testid="comments"> + {loading on(selected) fallback { <p>loading comments</p> } { + {errored fallback(e) { <p>comments error: {e.message}</p> } { + <ul> + {for c in comments { + <li data-testid="comment">{c.author}: {c.text}</li> + }} + {for p in sending { + {if p.id == selected { + <li data-testid="comment">{p.author}: {p.text} (sending…)</li> + }} + }} + </ul> + }} + }} + </div> + <input data-testid="comment-input" value={commentInput} onInput={e => { commentInput := e.target.value }} /> + <button data-testid="post" onClick={() => { + if (commentInput != "") { + let p = { id: selected, author: user, text: commentInput } + onFlush(() => { + sending := [...sending, p] + commentInput := "" + }) + postComment(p.id, p.author, p.text) + } + }}>post</button> + {if postError != "" { + <p>post error: {postError}</p> + }} + } else { + <p>select an issue</p> + }} + </div> + </main> +} diff --git a/eval/reference/rename/react/App.tsx b/eval/reference/rename/react/App.tsx new file mode 100644 index 0000000..30aa704 --- /dev/null +++ b/eval/reference/rename/react/App.tsx @@ -0,0 +1,229 @@ +import { useRef, useState } from "react"; +import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from "@tanstack/react-query"; +import { assign, fetchIssue, fetchIssues, renameIssue } from "./api.js"; +import type { Issue, IssueDetail } from "./api.js"; + +type Filter = "open" | "closed"; +type Pending = { seq: number; user: string }; +type AssignError = { issueId: number; message: string }; +type PendingTitle = { seq: number; title: string }; + +const errorMessage = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +function Tracker() { + const queryClient = useQueryClient(); + const [user, setUser] = useState("guest"); + const [filter, setFilter] = useState<Filter>("open"); + const [selected, setSelected] = useState<number | null>(null); + // optimistic assignments still in the air, per issue, oldest first + const [pending, setPending] = useState<Record<number, Pending[]>>({}); + const [assignError, setAssignError] = useState<AssignError | null>(null); + const seqRef = useRef(0); + const confirmedSeqRef = useRef<Record<number, number>>({}); + // optimistic renames still in the air, per issue, oldest first + const [pendingTitles, setPendingTitles] = useState<Record<number, PendingTitle[]>>({}); + const [renameError, setRenameError] = useState<AssignError | null>(null); + const [titleInput, setTitleInput] = useState(""); + const renameSeqRef = useRef(0); + const confirmedTitleSeqRef = useRef<Record<number, number>>({}); + + const list = useQuery({ + queryKey: ["issues", filter], + queryFn: () => fetchIssues(filter), + }); + + const detail = useQuery({ + queryKey: ["issue", selected], + queryFn: async () => { + const id = selected as number; + const d = await fetchIssue(id); + if (d === null) throw new Error(`issue #${id} not found`); + return d; + }, + enabled: selected !== null, + }); + + const assigneeOf = (issue: Issue) => { + const p = pending[issue.id]; + if (p && p.length > 0) return p[p.length - 1].user; + return issue.assignee; + }; + + const titleOf = (id: number, title: string) => { + const p = pendingTitles[id]; + if (p && p.length > 0) return p[p.length - 1].title; + return title; + }; + + const select = (id: number) => { + setSelected(id); + setAssignError(null); + setRenameError(null); + setTitleInput(""); + }; + + const doRename = async (id: number, title: string) => { + const seq = ++renameSeqRef.current; + setRenameError(null); + setPendingTitles(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { seq, title }] })); + const done = () => + setPendingTitles(prev => { + const rest = (prev[id] ?? []).filter(p => p.seq !== seq); + const next = { ...prev }; + if (rest.length) next[id] = rest; + else delete next[id]; + return next; + }); + try { + const res = await renameIssue(id, title); + if ((confirmedTitleSeqRef.current[id] ?? 0) < seq) { + confirmedTitleSeqRef.current[id] = seq; + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === id ? { ...i, title: res.title } : i)), + ); + queryClient.setQueryData<IssueDetail>(["issue", id], old => (old ? { ...old, title: res.title } : old)); + } + done(); + } catch (e) { + done(); + if (renameSeqRef.current === seq) setRenameError({ issueId: id, message: errorMessage(e) }); + } + }; + + const doAssign = async (id: number, who: string) => { + const seq = ++seqRef.current; + setAssignError(null); + setPending(prev => ({ ...prev, [id]: [...(prev[id] ?? []), { seq, user: who }] })); + const done = () => + setPending(prev => { + const rest = (prev[id] ?? []).filter(p => p.seq !== seq); + const next = { ...prev }; + if (rest.length) next[id] = rest; + else delete next[id]; + return next; + }); + try { + const res = await assign(id, who); + if ((confirmedSeqRef.current[id] ?? 0) < seq) { + confirmedSeqRef.current[id] = seq; + queryClient.setQueriesData<Issue[]>({ queryKey: ["issues"] }, old => + old?.map(i => (i.id === id ? { ...i, assignee: res.assignee } : i)), + ); + } + done(); + } catch (e) { + done(); + // only report the failure if no later assignment has started since + if (seqRef.current === seq) setAssignError({ issueId: id, message: errorMessage(e) }); + } + }; + + let listContent; + if (list.isPending) listContent = <p>loading {filter} issues</p>; + else if (list.isError) listContent = <p>list error: {errorMessage(list.error)}</p>; + else + listContent = ( + <ul> + {list.data.map(issue => ( + <li key={issue.id}> + <button + type="button" + data-testid="issue" + aria-pressed={issue.id === selected} + onClick={() => select(issue.id)} + > + #{issue.id} {titleOf(issue.id, issue.title)} ({assigneeOf(issue) ?? "unassigned"}) + </button> + </li> + ))} + </ul> + ); + + let detailContent; + if (selected === null) detailContent = <p>select an issue</p>; + else { + let body; + if (detail.isFetching || detail.isPending) body = <p>loading #{selected}</p>; + else if (detail.isError) body = <p>detail error: {errorMessage(detail.error)}</p>; + else + body = ( + <> + <h2> + #{detail.data.id} {titleOf(detail.data.id, detail.data.title)} + </h2> + <p>{detail.data.body}</p> + </> + ); + const id = selected; + detailContent = ( + <> + {body} + <button type="button" data-testid="retry" onClick={() => void detail.refetch()}> + retry + </button> + <button type="button" data-testid="assign" onClick={() => void doAssign(id, user)}> + assign to {user} + </button> + {assignError && assignError.issueId === selected && ( + <p data-testid="assign-error">{assignError.message}</p> + )} + <input data-testid="title-input" value={titleInput} onChange={e => setTitleInput(e.target.value)} /> + <button + type="button" + data-testid="rename" + onClick={() => { + if (titleInput !== "") void doRename(id, titleInput); + }} + > + rename + </button> + {renameError && renameError.issueId === selected && <p>rename error: {renameError.message}</p>} + </> + ); + } + + return ( + <div style={{ display: "flex", gap: "2rem" }}> + <section> + <label> + user{" "} + <input data-testid="user" value={user} onChange={e => setUser(e.target.value)} /> + </label> + <div> + <button + type="button" + data-testid="filter-open" + aria-pressed={filter === "open"} + onClick={() => setFilter("open")} + > + open + </button> + <button + type="button" + data-testid="filter-closed" + aria-pressed={filter === "closed"} + onClick={() => setFilter("closed")} + > + closed + </button> + </div> + <div data-testid="list">{listContent}</div> + </section> + <section data-testid="detail">{detailContent}</section> + </div> + ); +} + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0, refetchOnWindowFocus: false, refetchOnReconnect: false }, + }, +}); + +export default function App() { + return ( + <QueryClientProvider client={queryClient}> + <Tracker /> + </QueryClientProvider> + ); +} diff --git a/eval/reference/rename/rx/main.rx b/eval/reference/rename/rx/main.rx new file mode 100644 index 0000000..a3f53de --- /dev/null +++ b/eval/reference/rename/rx/main.rx @@ -0,0 +1,101 @@ +import pure { fetchIssues, fetchIssue } from "./api" +import { assign, renameIssue } from "./api" +import pure { indexOf } from "./prelude" + +export error NotFound(id: number) => `issue #${id} not found` + +fn found(it, id: number) => it == null ? throw NotFound(id) : it + +export default component App() { + source user: string := "guest" + source filter: string := "open" + source selected: number | null := null + source issues key id = await fetchIssues(filter) + source assignError: string = "" + source renameError: string = "" + source titleInput: string := "" + source detail = selected == null ? null : found(await fetchIssue(selected), selected) + + action renameTo(id: number, title: string) { + optimistic renameError := "" + try { + let i = indexOf(issues, t => t.id == id) + if (i >= 0) { + optimistic issues[i].title := title + } + if (detail != null && detail.id == id) { + optimistic detail.title := title + } + await renameIssue(id, title) + refresh issues + refresh detail + } catch (e) { + renameError := e.message + } + } + + action assignTo(id: number, who: string) { + optimistic assignError := "" + try { + let i = indexOf(issues, t => t.id == id) + optimistic issues[i].assignee := who + await assign(id, who) + refresh issues + } catch (e) { + assignError := e.message + } + } + + <main> + <div> + <label> + user{" "} + <input data-testid="user" value={user} onInput={e => { user := e.target.value }} /> + </label> + </div> + <div> + <button data-testid="filter-open" onClick={() => { filter := "open" }}>open</button> + <button data-testid="filter-closed" onClick={() => { filter := "closed" }}>closed</button> + </div> + <div data-testid="list"> + {loading on(filter) fallback { <p>loading {latest filter} issues</p> } { + {errored fallback(e) { <p>list error: {e.message}</p> } { + <ul> + {for it in issues { + <li data-testid="issue" onClick={() => { + selected := it.id + titleInput := "" + onFlush(() => { + assignError := "" + renameError := "" + }) + }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> + }} + </ul> + }} + }} + </div> + <div data-testid="detail"> + {if selected != null { + {loading on(selected) fallback { <p>loading #{latest selected}</p> } { + {errored fallback(e) { <p>detail error: {e.message}</p> } { + <h2>#{detail.id} {detail.title}</h2> + <p>{detail.body}</p> + }} + <button data-testid="retry" onClick={() => { refresh detail }}>retry</button> + }} + <button data-testid="assign" onClick={() => { assignTo(selected, user) }}>assign to {user}</button> + {if assignError != "" { + <p data-testid="assign-error">{assignError}</p> + }} + <input data-testid="title-input" value={titleInput} onInput={e => { titleInput := e.target.value }} /> + <button data-testid="rename" onClick={() => { if (titleInput != "") { renameTo(selected, titleInput) } }}>rename</button> + {if renameError != "" { + <p>rename error: {renameError}</p> + }} + } else { + <p>select an issue</p> + }} + </div> + </main> +} diff --git a/eval/tasks/close/BASE.md b/eval/tasks/close/BASE.md new file mode 100644 index 0000000..981ca16 --- /dev/null +++ b/eval/tasks/close/BASE.md @@ -0,0 +1,39 @@ +# Issue tracker + +Build an issue tracker: a filtered list of issues beside the selected issue's detail, and a button that assigns the +selected issue to the current user. + +## The API (`src/api.js`, given; do not change it) + +- `fetchIssues(filter: "open" | "closed"): Promise<{ id: number; title: string; assignee: string | null }[]>`. May + reject with an `Error`. +- `fetchIssue(id: number): Promise<{ id: number; title: string; body: string } | null>`: `null` when there is no such + issue. May reject with an `Error`. +- `assign(id: number, user: string): Promise<{ id: number; assignee: string }>`: assigns the issue on the server. May + reject with an `Error` (the server declined). + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Current user.** A text input, `user`, holding the current user's name. It starts as `guest`. +2. **Filter.** Two buttons, `filter-open` and `filter-closed`. The filter starts as `open`. +3. **List**, `list`, of the issues for the current filter, from `fetchIssues(filter)`. + - While they load (when the page opens or the filter changes): `loading <filter> issues` (`loading closed issues`). + - Then one row per issue, `data-testid="issue"`, reading `#<id> <title> (<assignee>)`, or `(unassigned)` when the + assignee is `null`: `#3 Crash on save (unassigned)`. + - If it fails: `list error: <message>`. +4. **Detail**, `detail`. + - Nothing selected: `select an issue`. Clicking a row selects its issue. + - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #<id>`. + - An issue: `#<id> <title>` and its body. + - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. + - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. +5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading + `assign to <current user>`. Clicking it: + - shows the current user as the issue's assignee in its list row straight away; + - calls `assign(id, user)`; + - if the server declines, the row goes back to the assignee it had before, and the detail shows the server's + message (`declined`); + - if the server accepts, the row keeps the new assignee. + - An assignment error is cleared when a new assignment starts or another issue is selected. diff --git a/eval/tasks/close/api.d.rx b/eval/tasks/close/api.d.rx new file mode 100644 index 0000000..3842216 --- /dev/null +++ b/eval/tasks/close/api.d.rx @@ -0,0 +1,7 @@ +// src/api.js, as .rx sees it. fetchIssues and fetchIssue are loaders (safe to ask from an equation); assign and closeIssue write. +export type Issue = { id: number; title: string; assignee: string | null } +export type IssueDetail = { id: number; title: string; body: string } +export pure fn fetchIssues(filter: string): Promise<Issue[]> +export pure fn fetchIssue(id: number): Promise<IssueDetail | null> +export fn assign(id: number, user: string): Promise<{ id: number; assignee: string }> +export fn closeIssue(id: number): Promise<{ id: number }> diff --git a/eval/tasks/close/api.d.ts b/eval/tasks/close/api.d.ts new file mode 100644 index 0000000..69204bb --- /dev/null +++ b/eval/tasks/close/api.d.ts @@ -0,0 +1,6 @@ +export type Issue = { id: number; title: string; assignee: string | null }; +export type IssueDetail = { id: number; title: string; body: string }; +export function fetchIssues(filter: "open" | "closed"): Promise<Issue[]>; +export function fetchIssue(id: number): Promise<IssueDetail | null>; +export function assign(id: number, user: string): Promise<{ id: number; assignee: string }>; +export function closeIssue(id: number): Promise<{ id: number }>; diff --git a/eval/tasks/close/api.js b/eval/tasks/close/api.js new file mode 100644 index 0000000..ae27530 --- /dev/null +++ b/eval/tasks/close/api.js @@ -0,0 +1,6 @@ +import { flight } from "./flights.js"; + +export const fetchIssues = filter => flight("fetchIssues", [filter]); +export const fetchIssue = id => flight("fetchIssue", [id]); +export const assign = (id, user) => flight("assign", [id, user]); +export const closeIssue = id => flight("closeIssue", [id]); diff --git a/eval/tasks/close/brief-plain.md b/eval/tasks/close/brief-plain.md new file mode 100644 index 0000000..0c62109 --- /dev/null +++ b/eval/tasks/close/brief-plain.md @@ -0,0 +1,24 @@ +# Change request: close an issue + +The app in `src/` is an issue tracker that already works. `BASE.md` is the brief it was built to. Keep everything it +does, and add closing an issue. + +## The API (`src/api.js`, given; do not change it) + +Everything in `BASE.md`, and one new function: + +- `closeIssue(id: number): Promise<{ id: number }>`: closes the issue on the server. May reject with an `Error` (the + server declined). + +## What to add + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. While an issue is selected, the detail has a button `close` (`data-testid="close"`). +2. Clicking `close`: + - removes the issue's row from the list of open issues straight away (the detail keeps showing the issue); + - calls `closeIssue(id)`; + - if the server declines, the row comes back where it was, and the detail shows `close error: <message>` + (`close error: declined`); + - if the server accepts, the row stays out of the open issues. +3. A close error is cleared when a new close starts or another issue is selected. diff --git a/eval/tasks/close/fixtures.mjs b/eval/tasks/close/fixtures.mjs new file mode 100644 index 0000000..d2f1f38 --- /dev/null +++ b/eval/tasks/close/fixtures.mjs @@ -0,0 +1,12 @@ +export const OPEN = [{ id: 1, title: "Login fails", assignee: null }, { id: 3, title: "Crash on save", assignee: null }, { id: 4, title: "Typo in footer", assignee: "dana" }]; +export const CLOSED = [{ id: 2, title: "Old bug", assignee: "eve" }]; +export const OPEN_ROWS = ["#1 Login fails (unassigned)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]; +export const D1 = { id: 1, title: "Login fails", body: "on Safari only" }; +export const D3 = { id: 3, title: "Crash on save", body: "stack trace attached" }; +/** the open issues loaded */ +export async function loaded(app) { await app.land("fetchIssues:open", OPEN); } +/** the open issues loaded and issue `id` selected with its detail */ +export async function selected(app, id = 3, detail = D3) { await loaded(app); await app.clickText("issue", `#${id} `); await app.land(`fetchIssue:${id}`, detail); } +/** the open issues without `id` */ +export const openWithout = id => OPEN.filter(i => i.id !== id); +export const rowsWithout = id => OPEN_ROWS.filter(r => !r.startsWith(`#${id} `)); diff --git a/eval/tasks/close/hidden-overlap.mjs b/eval/tasks/close/hidden-overlap.mjs new file mode 100644 index 0000000..2d49ef9 --- /dev/null +++ b/eval/tasks/close/hidden-overlap.mjs @@ -0,0 +1,55 @@ +// Round 4 hidden tests: the change's writes overlapping the base app's, and the base app's overlap tests as +// regressions. Never shown to the agent. +import base from "../tracker/hidden-overlap.mjs"; +import { D1, OPEN, OPEN_ROWS, selected, openWithout, rowsWithout } from "./fixtures.mjs"; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "a declined close keeps an assignment in flight on another issue", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("close"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.click("assign"); + await app.fail("closeIssue:3", "declined"); + await app.rows("issue", ["#1 Login fails (ann)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + await app.land("assign:1,ann", { id: 1, assignee: "ann" }); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 1 ? { ...i, assignee: "ann" } : i)); + await app.rows("issue", ["#1 Login fails (ann)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + } }, + { name: "a declined assignment keeps a close in flight on another issue", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.click("close"); + await app.fail("assign:3,ann", "declined"); + await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + await app.land("closeIssue:1", { id: 1 }); + await app.landIfAsked("fetchIssues:open", openWithout(1)); + await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + } }, + { name: "a list that loads while a close is out keeps the row out", async run(app) { + await selected(app); + await app.click("close"); + await app.click("filter-closed"); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:closed", []); + await app.landIfAsked("fetchIssues:open", OPEN); + await app.rows("issue", rowsWithout(3)); + await app.land("closeIssue:3", { id: 3 }); + await app.landIfAsked("fetchIssues:open", openWithout(3)); + await app.rows("issue", rowsWithout(3)); + } }, + { name: "two closes at once: declining one brings back only that row", async run(app) { + await selected(app); + await app.click("close"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.click("close"); + await app.fail("closeIssue:3", "declined"); + await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + await app.land("closeIssue:1", { id: 1 }); + await app.landIfAsked("fetchIssues:open", openWithout(1)); + await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + } }, +]; diff --git a/eval/tasks/close/hidden.mjs b/eval/tasks/close/hidden.mjs new file mode 100644 index 0000000..a023fc5 --- /dev/null +++ b/eval/tasks/close/hidden.mjs @@ -0,0 +1,41 @@ +// Round 4 hidden tests: the change's own behaviour, and the base app's hidden tests as regressions. Never shown to +// the agent. +import base from "../tracker/hidden.mjs"; +import { CLOSED, OPEN_ROWS, selected, openWithout, rowsWithout } from "./fixtures.mjs"; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "a declined close brings the row back in place and shows the message", async run(app) { + await selected(app); + await app.click("close"); + await app.fail("closeIssue:3", "declined"); + await app.rows("issue", OPEN_ROWS); + await app.see("detail", "close error: declined"); + } }, + { name: "a new close clears the previous error at once", async run(app) { + await selected(app); + await app.click("close"); + await app.fail("closeIssue:3", "declined"); + await app.click("close"); + await app.notSee("detail", "close error"); + await app.rows("issue", rowsWithout(3)); + } }, + { name: "selecting another issue clears the close error", async run(app) { + await selected(app); + await app.click("close"); + await app.fail("closeIssue:3", "declined"); + await app.clickText("issue", "#1 "); + await app.notSee("detail", "close error"); + } }, + { name: "an accepted close survives switching filters and back", async run(app) { + await selected(app); + await app.click("close"); + await app.land("closeIssue:3", { id: 3 }); + await app.landIfAsked("fetchIssues:open", openWithout(3)); + await app.click("filter-closed"); + await app.landIfAsked("fetchIssues:closed", [...CLOSED, { id: 3, title: "Crash on save", assignee: null }]); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:open", openWithout(3)); + await app.rows("issue", rowsWithout(3)); + } }, +]; diff --git a/eval/tasks/close/visible.mjs b/eval/tasks/close/visible.mjs new file mode 100644 index 0000000..d292f68 --- /dev/null +++ b/eval/tasks/close/visible.mjs @@ -0,0 +1,48 @@ +// Visible tests: the agent sees and runs these. +import { CLOSED, D3, OPEN_ROWS, loaded, selected, rowsWithout } from "./fixtures.mjs"; + +export default [ + { name: "on load: the open issues load", async run(app) { + await app.see("list", "loading open issues"); + await app.see("detail", "select an issue"); + await loaded(app); + await app.rows("issue", OPEN_ROWS); + } }, + { name: "switching the filter loads the closed issues", async run(app) { + await loaded(app); + await app.click("filter-closed"); + await app.see("list", "loading closed issues"); + await app.land("fetchIssues:closed", CLOSED); + await app.rows("issue", ["#2 Old bug (eve)"]); + } }, + { name: "selecting an issue loads its detail", async run(app) { + await loaded(app); + await app.clickText("issue", "#3 "); + await app.see("detail", "loading #3"); + await app.land("fetchIssue:3", D3); + await app.see("detail", "#3 Crash on save", "stack trace attached"); + } }, + { name: "assigning shows the new assignee at once", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.see("assign", "assign to ann"); + await app.click("assign"); + await app.inAir("assign:3,ann"); + await app.see("list", "#3 Crash on save (ann)"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.see("list", "#3 Crash on save (ann)"); + } }, + { name: "closing removes the row at once and keeps the detail", async run(app) { + await selected(app); + await app.click("close"); + await app.inAir("closeIssue:3"); + await app.rows("issue", rowsWithout(3)); + await app.see("detail", "#3 Crash on save"); + } }, + { name: "an accepted close keeps the row out", async run(app) { + await selected(app); + await app.click("close"); + await app.land("closeIssue:3", { id: 3 }); + await app.rows("issue", rowsWithout(3)); + } }, +]; diff --git a/eval/tasks/comments/BASE.md b/eval/tasks/comments/BASE.md new file mode 100644 index 0000000..981ca16 --- /dev/null +++ b/eval/tasks/comments/BASE.md @@ -0,0 +1,39 @@ +# Issue tracker + +Build an issue tracker: a filtered list of issues beside the selected issue's detail, and a button that assigns the +selected issue to the current user. + +## The API (`src/api.js`, given; do not change it) + +- `fetchIssues(filter: "open" | "closed"): Promise<{ id: number; title: string; assignee: string | null }[]>`. May + reject with an `Error`. +- `fetchIssue(id: number): Promise<{ id: number; title: string; body: string } | null>`: `null` when there is no such + issue. May reject with an `Error`. +- `assign(id: number, user: string): Promise<{ id: number; assignee: string }>`: assigns the issue on the server. May + reject with an `Error` (the server declined). + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Current user.** A text input, `user`, holding the current user's name. It starts as `guest`. +2. **Filter.** Two buttons, `filter-open` and `filter-closed`. The filter starts as `open`. +3. **List**, `list`, of the issues for the current filter, from `fetchIssues(filter)`. + - While they load (when the page opens or the filter changes): `loading <filter> issues` (`loading closed issues`). + - Then one row per issue, `data-testid="issue"`, reading `#<id> <title> (<assignee>)`, or `(unassigned)` when the + assignee is `null`: `#3 Crash on save (unassigned)`. + - If it fails: `list error: <message>`. +4. **Detail**, `detail`. + - Nothing selected: `select an issue`. Clicking a row selects its issue. + - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #<id>`. + - An issue: `#<id> <title>` and its body. + - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. + - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. +5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading + `assign to <current user>`. Clicking it: + - shows the current user as the issue's assignee in its list row straight away; + - calls `assign(id, user)`; + - if the server declines, the row goes back to the assignee it had before, and the detail shows the server's + message (`declined`); + - if the server accepts, the row keeps the new assignee. + - An assignment error is cleared when a new assignment starts or another issue is selected. diff --git a/eval/tasks/comments/api.d.rx b/eval/tasks/comments/api.d.rx new file mode 100644 index 0000000..b33d40e --- /dev/null +++ b/eval/tasks/comments/api.d.rx @@ -0,0 +1,9 @@ +// src/api.js, as .rx sees it. fetchIssues, fetchIssue and fetchComments are loaders (safe to ask from an equation); assign and addComment write. +export type Issue = { id: number; title: string; assignee: string | null } +export type IssueDetail = { id: number; title: string; body: string } +export pure fn fetchIssues(filter: string): Promise<Issue[]> +export pure fn fetchIssue(id: number): Promise<IssueDetail | null> +export fn assign(id: number, user: string): Promise<{ id: number; assignee: string }> +export type Comment = { author: string; text: string } +export pure fn fetchComments(id: number): Promise<Comment[]> +export fn addComment(id: number, author: string, text: string): Promise<Comment> diff --git a/eval/tasks/comments/api.d.ts b/eval/tasks/comments/api.d.ts new file mode 100644 index 0000000..695d716 --- /dev/null +++ b/eval/tasks/comments/api.d.ts @@ -0,0 +1,8 @@ +export type Issue = { id: number; title: string; assignee: string | null }; +export type IssueDetail = { id: number; title: string; body: string }; +export function fetchIssues(filter: "open" | "closed"): Promise<Issue[]>; +export function fetchIssue(id: number): Promise<IssueDetail | null>; +export function assign(id: number, user: string): Promise<{ id: number; assignee: string }>; +export type Comment = { author: string; text: string }; +export function fetchComments(id: number): Promise<Comment[]>; +export function addComment(id: number, author: string, text: string): Promise<Comment>; diff --git a/eval/tasks/comments/api.js b/eval/tasks/comments/api.js new file mode 100644 index 0000000..4993d2f --- /dev/null +++ b/eval/tasks/comments/api.js @@ -0,0 +1,7 @@ +import { flight } from "./flights.js"; + +export const fetchIssues = filter => flight("fetchIssues", [filter]); +export const fetchIssue = id => flight("fetchIssue", [id]); +export const assign = (id, user) => flight("assign", [id, user]); +export const fetchComments = id => flight("fetchComments", [id]); +export const addComment = (id, author, text) => flight("addComment", [id, author, text]); diff --git a/eval/tasks/comments/brief-plain.md b/eval/tasks/comments/brief-plain.md new file mode 100644 index 0000000..8bc47df --- /dev/null +++ b/eval/tasks/comments/brief-plain.md @@ -0,0 +1,31 @@ +# Change request: comments + +The app in `src/` is an issue tracker that already works. `BASE.md` is the brief it was built to. Keep everything it +does, and add comments to the selected issue. + +## The API (`src/api.js`, given; do not change it) + +Everything in `BASE.md`, and two new functions: + +- `fetchComments(id: number): Promise<{ author: string; text: string }[]>`: the issue's comments, oldest first. May + reject with an `Error`. +- `addComment(id: number, author: string, text: string): Promise<{ author: string; text: string }>`: adds a comment + on the server. May reject with an `Error` (the server declined). + +## What to add + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Comments.** While an issue is selected, the detail has an element `comments` with that issue's comments, from + `fetchComments(id)`. + - While they load: `loading comments`. + - Then one row per comment, `data-testid="comment"`, reading `<author>: <text>` (`dana: seen it too`). + - If it fails: `comments error: <message>`. +2. **Posting.** The detail has a text input `comment-input` and a button `post` (`data-testid="post"`). Clicking + `post` while `comment-input` holds text: + - adds the comment as the last row straight away, reading `<current user>: <text> (sending…)`, and empties the + input; + - calls `addComment(id, user, text)`; + - if the server declines, the row goes away, and the detail shows `post error: <message>` (`post error: declined`); + - if the server accepts, the row stays and reads `<current user>: <text>`. +3. A post error is cleared when a new post starts or another issue is selected. diff --git a/eval/tasks/comments/fixtures.mjs b/eval/tasks/comments/fixtures.mjs new file mode 100644 index 0000000..e250fb8 --- /dev/null +++ b/eval/tasks/comments/fixtures.mjs @@ -0,0 +1,15 @@ +export const OPEN = [{ id: 1, title: "Login fails", assignee: null }, { id: 3, title: "Crash on save", assignee: null }, { id: 4, title: "Typo in footer", assignee: "dana" }]; +export const CLOSED = [{ id: 2, title: "Old bug", assignee: "eve" }]; +export const OPEN_ROWS = ["#1 Login fails (unassigned)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]; +export const D1 = { id: 1, title: "Login fails", body: "on Safari only" }; +export const D3 = { id: 3, title: "Crash on save", body: "stack trace attached" }; +/** the open issues loaded */ +export async function loaded(app) { await app.land("fetchIssues:open", OPEN); } +/** the open issues loaded and issue `id` selected with its detail */ +export async function selected(app, id = 3, detail = D3) { await loaded(app); await app.clickText("issue", `#${id} `); await app.land(`fetchIssue:${id}`, detail); } +export const C3 = [{ author: "dana", text: "seen it too" }]; +export const C1 = []; +/** issue 3 selected with its detail and its comments */ +export async function withComments(app) { await selected(app); await app.land("fetchComments:3", C3); } +/** type `text` into the comment input and click post */ +export async function post(app, text) { await app.type("comment-input", text); await app.click("post"); } diff --git a/eval/tasks/comments/hidden-overlap.mjs b/eval/tasks/comments/hidden-overlap.mjs new file mode 100644 index 0000000..a134a41 --- /dev/null +++ b/eval/tasks/comments/hidden-overlap.mjs @@ -0,0 +1,52 @@ +// Round 4 hidden tests: the change's writes overlapping the base app's, and the base app's overlap tests as +// regressions. Never shown to the agent. +import base from "../tracker/hidden-overlap.mjs"; +import { C1, C3, D1, D3, withComments, post } from "./fixtures.mjs"; + +const after = (...more) => [...C3, ...more]; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "two posts at once: declining the first keeps the second", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "first"); + await post(app, "second"); + await app.fail("addComment:3,ann,first", "declined"); + await app.rows("comment", ["dana: seen it too", "ann: second (sending…)"]); + await app.land("addComment:3,ann,second", { author: "ann", text: "second" }); + await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "second" })); + await app.rows("comment", ["dana: seen it too", "ann: second"]); + } }, + { name: "a post on one issue does not show on another", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.land("fetchComments:1", C1); + await app.notSee("comments", "on it"); + await app.land("addComment:3,ann,on it", { author: "ann", text: "on it" }); + await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "on it" })); + await app.notSee("comments", "on it"); + } }, + { name: "comments that load while a post is out keep the post", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.land("fetchComments:1", C1); + await app.clickText("issue", "#3 "); await app.landIfAsked("fetchIssue:3", D3); + await app.landIfAsked("fetchComments:3", C3); + await app.rows("comment", ["dana: seen it too", "ann: on it (sending…)"]); + await app.land("addComment:3,ann,on it", { author: "ann", text: "on it" }); + await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "on it" })); + await app.rows("comment", ["dana: seen it too", "ann: on it"]); + } }, + { name: "a post declined while an assignment is out leaves the assignment", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await app.click("assign"); + await post(app, "on it"); + await app.fail("addComment:3,ann,on it", "declined"); + await app.see("list", "#3 Crash on save (ann)"); + await app.see("detail", "post error: declined"); + } }, +]; diff --git a/eval/tasks/comments/hidden.mjs b/eval/tasks/comments/hidden.mjs new file mode 100644 index 0000000..38b3827 --- /dev/null +++ b/eval/tasks/comments/hidden.mjs @@ -0,0 +1,61 @@ +// Round 4 hidden tests: the change's own behaviour, and the base app's hidden tests as regressions. Never shown to +// the agent. +import base from "../tracker/hidden.mjs"; +import { C1, C3, D1, D3, loaded, selected, withComments, post } from "./fixtures.mjs"; + +const after = (...more) => [...C3, ...more]; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "a declined post removes the row and shows the message", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.fail("addComment:3,ann,on it", "declined"); + await app.rows("comment", ["dana: seen it too"]); + await app.see("detail", "post error: declined"); + } }, + { name: "a new post clears the previous error at once", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.fail("addComment:3,ann,on it", "declined"); + await post(app, "really"); + await app.notSee("detail", "post error"); + await app.rows("comment", ["dana: seen it too", "ann: really (sending…)"]); + } }, + { name: "selecting another issue clears the post error", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.fail("addComment:3,ann,on it", "declined"); + await app.clickText("issue", "#1 "); + await app.notSee("detail", "post error"); + } }, + { name: "comments landing out of order show the selected issue's", async run(app) { + await loaded(app); + await app.clickText("issue", "#3 "); await app.clickText("issue", "#1 "); + await app.land("fetchIssue:1", D1); + await app.land("fetchComments:1", C1); + await app.landIfAsked("fetchIssue:3", D3); + await app.landIfAsked("fetchComments:3", C3); + await app.notSee("comments", "seen it too"); + } }, + { name: "failing comments show their error and leave the issue", async run(app) { + await selected(app); + await app.fail("fetchComments:3", "timeout"); + await app.see("comments", "comments error: timeout"); + await app.see("detail", "#3 Crash on save", "stack trace attached"); + } }, + { name: "an accepted post stays after switching issues and back", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.land("addComment:3,ann,on it", { author: "ann", text: "on it" }); + await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "on it" })); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.landIfAsked("fetchComments:1", C1); + await app.clickText("issue", "#3 "); await app.landIfAsked("fetchIssue:3", D3); + await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "on it" })); + await app.rows("comment", ["dana: seen it too", "ann: on it"]); + } }, +]; diff --git a/eval/tasks/comments/visible.mjs b/eval/tasks/comments/visible.mjs new file mode 100644 index 0000000..b07c5ab --- /dev/null +++ b/eval/tasks/comments/visible.mjs @@ -0,0 +1,52 @@ +// Visible tests: the agent sees and runs these. +import { CLOSED, D3, OPEN_ROWS, loaded, selected, withComments, post } from "./fixtures.mjs"; + +export default [ + { name: "on load: the open issues load", async run(app) { + await app.see("list", "loading open issues"); + await app.see("detail", "select an issue"); + await loaded(app); + await app.rows("issue", OPEN_ROWS); + } }, + { name: "switching the filter loads the closed issues", async run(app) { + await loaded(app); + await app.click("filter-closed"); + await app.see("list", "loading closed issues"); + await app.land("fetchIssues:closed", CLOSED); + await app.rows("issue", ["#2 Old bug (eve)"]); + } }, + { name: "selecting an issue loads its detail", async run(app) { + await loaded(app); + await app.clickText("issue", "#3 "); + await app.see("detail", "loading #3"); + await app.land("fetchIssue:3", D3); + await app.see("detail", "#3 Crash on save", "stack trace attached"); + } }, + { name: "assigning shows the new assignee at once", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.see("assign", "assign to ann"); + await app.click("assign"); + await app.inAir("assign:3,ann"); + await app.see("list", "#3 Crash on save (ann)"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.see("list", "#3 Crash on save (ann)"); + } }, + { name: "selecting an issue loads its comments", async run(app) { + await selected(app); + await app.see("comments", "loading comments"); + await app.land("fetchComments:3", [{ author: "dana", text: "seen it too" }]); + await app.rows("comment", ["dana: seen it too"]); + } }, + { name: "posting shows the comment at once and keeps it when accepted", async run(app) { + await withComments(app); + await app.type("user", "ann"); + await post(app, "on it"); + await app.inAir("addComment:3,ann,on it"); + await app.rows("comment", ["dana: seen it too", "ann: on it (sending…)"]); + await app.hasValue("comment-input", ""); + await app.land("addComment:3,ann,on it", { author: "ann", text: "on it" }); + await app.landIfAsked("fetchComments:3", [{ author: "dana", text: "seen it too" }, { author: "ann", text: "on it" }]); + await app.rows("comment", ["dana: seen it too", "ann: on it"]); + } }, +]; diff --git a/eval/tasks/rename/BASE.md b/eval/tasks/rename/BASE.md new file mode 100644 index 0000000..981ca16 --- /dev/null +++ b/eval/tasks/rename/BASE.md @@ -0,0 +1,39 @@ +# Issue tracker + +Build an issue tracker: a filtered list of issues beside the selected issue's detail, and a button that assigns the +selected issue to the current user. + +## The API (`src/api.js`, given; do not change it) + +- `fetchIssues(filter: "open" | "closed"): Promise<{ id: number; title: string; assignee: string | null }[]>`. May + reject with an `Error`. +- `fetchIssue(id: number): Promise<{ id: number; title: string; body: string } | null>`: `null` when there is no such + issue. May reject with an `Error`. +- `assign(id: number, user: string): Promise<{ id: number; assignee: string }>`: assigns the issue on the server. May + reject with an `Error` (the server declined). + +## What the page must do + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. **Current user.** A text input, `user`, holding the current user's name. It starts as `guest`. +2. **Filter.** Two buttons, `filter-open` and `filter-closed`. The filter starts as `open`. +3. **List**, `list`, of the issues for the current filter, from `fetchIssues(filter)`. + - While they load (when the page opens or the filter changes): `loading <filter> issues` (`loading closed issues`). + - Then one row per issue, `data-testid="issue"`, reading `#<id> <title> (<assignee>)`, or `(unassigned)` when the + assignee is `null`: `#3 Crash on save (unassigned)`. + - If it fails: `list error: <message>`. +4. **Detail**, `detail`. + - Nothing selected: `select an issue`. Clicking a row selects its issue. + - The selected issue comes from `fetchIssue(id)`. While it loads: `loading #<id>`. + - An issue: `#<id> <title>` and its body. + - `null`: `detail error: issue #<id> not found`. A failure: `detail error: <message>`. + - A button `retry` (`data-testid="retry"`) calls `fetchIssue(id)` again and shows its answer. +5. **Assign.** While an issue is selected, the detail has a button `assign` (`data-testid="assign"`) reading + `assign to <current user>`. Clicking it: + - shows the current user as the issue's assignee in its list row straight away; + - calls `assign(id, user)`; + - if the server declines, the row goes back to the assignee it had before, and the detail shows the server's + message (`declined`); + - if the server accepts, the row keeps the new assignee. + - An assignment error is cleared when a new assignment starts or another issue is selected. diff --git a/eval/tasks/rename/api.d.rx b/eval/tasks/rename/api.d.rx new file mode 100644 index 0000000..7e53e9a --- /dev/null +++ b/eval/tasks/rename/api.d.rx @@ -0,0 +1,8 @@ +// src/api.js, as .rx sees it. fetchIssues and fetchIssue are loaders (safe to ask from an equation); assign and +// renameIssue write. +export type Issue = { id: number; title: string; assignee: string | null } +export type IssueDetail = { id: number; title: string; body: string } +export pure fn fetchIssues(filter: string): Promise<Issue[]> +export pure fn fetchIssue(id: number): Promise<IssueDetail | null> +export fn assign(id: number, user: string): Promise<{ id: number; assignee: string }> +export fn renameIssue(id: number, title: string): Promise<{ id: number; title: string }> diff --git a/eval/tasks/rename/api.d.ts b/eval/tasks/rename/api.d.ts new file mode 100644 index 0000000..6aff21f --- /dev/null +++ b/eval/tasks/rename/api.d.ts @@ -0,0 +1,6 @@ +export type Issue = { id: number; title: string; assignee: string | null }; +export type IssueDetail = { id: number; title: string; body: string }; +export function fetchIssues(filter: "open" | "closed"): Promise<Issue[]>; +export function fetchIssue(id: number): Promise<IssueDetail | null>; +export function assign(id: number, user: string): Promise<{ id: number; assignee: string }>; +export function renameIssue(id: number, title: string): Promise<{ id: number; title: string }>; diff --git a/eval/tasks/rename/api.js b/eval/tasks/rename/api.js new file mode 100644 index 0000000..c5bcdf3 --- /dev/null +++ b/eval/tasks/rename/api.js @@ -0,0 +1,6 @@ +import { flight } from "./flights.js"; + +export const fetchIssues = filter => flight("fetchIssues", [filter]); +export const fetchIssue = id => flight("fetchIssue", [id]); +export const assign = (id, user) => flight("assign", [id, user]); +export const renameIssue = (id, title) => flight("renameIssue", [id, title]); diff --git a/eval/tasks/rename/brief-plain.md b/eval/tasks/rename/brief-plain.md new file mode 100644 index 0000000..bfad64a --- /dev/null +++ b/eval/tasks/rename/brief-plain.md @@ -0,0 +1,25 @@ +# Change request: rename an issue + +The app in `src/` is an issue tracker that already works. `BASE.md` is the brief it was built to. Keep everything it +does, and add renaming an issue. + +## The API (`src/api.js`, given; do not change it) + +Everything in `BASE.md`, and one new function: + +- `renameIssue(id: number, title: string): Promise<{ id: number; title: string }>`: renames the issue on the server. + May reject with an `Error` (the server declined). + +## What to add + +Elements the tests look for carry these `data-testid`s. Text in quotes must appear exactly as written. + +1. While an issue is selected, the detail has a text input `title-input`, empty when the issue is selected, and a + button `rename` (`data-testid="rename"`). +2. Clicking `rename` while `title-input` holds a title: + - shows the new title in the issue's list row and in the detail's heading (`#<id> <title>`) straight away; + - calls `renameIssue(id, title)`; + - if the server declines, the row and the heading go back to the title they had before, and the detail shows + `rename error: <message>` (`rename error: declined`); + - if the server accepts, the row and the heading keep the new title. +3. A rename error is cleared when a new rename starts or another issue is selected. diff --git a/eval/tasks/rename/fixtures.mjs b/eval/tasks/rename/fixtures.mjs new file mode 100644 index 0000000..b0f6ea3 --- /dev/null +++ b/eval/tasks/rename/fixtures.mjs @@ -0,0 +1,11 @@ +export const OPEN = [{ id: 1, title: "Login fails", assignee: null }, { id: 3, title: "Crash on save", assignee: null }, { id: 4, title: "Typo in footer", assignee: "dana" }]; +export const CLOSED = [{ id: 2, title: "Old bug", assignee: "eve" }]; +export const OPEN_ROWS = ["#1 Login fails (unassigned)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]; +export const D1 = { id: 1, title: "Login fails", body: "on Safari only" }; +export const D3 = { id: 3, title: "Crash on save", body: "stack trace attached" }; +/** the open issues loaded */ +export async function loaded(app) { await app.land("fetchIssues:open", OPEN); } +/** the open issues loaded and issue `id` selected with its detail */ +export async function selected(app, id = 3, detail = D3) { await loaded(app); await app.clickText("issue", `#${id} `); await app.land(`fetchIssue:${id}`, detail); } +/** type `title` into the detail's title input and click rename */ +export async function rename(app, title) { await app.type("title-input", title); await app.click("rename"); } diff --git a/eval/tasks/rename/hidden-overlap.mjs b/eval/tasks/rename/hidden-overlap.mjs new file mode 100644 index 0000000..0feb2ce --- /dev/null +++ b/eval/tasks/rename/hidden-overlap.mjs @@ -0,0 +1,61 @@ +// Round 4 hidden tests: the change's writes overlapping the base app's, and the base app's overlap tests as +// regressions. Never shown to the agent. +import base from "../tracker/hidden-overlap.mjs"; +import { D3, OPEN, selected, rename } from "./fixtures.mjs"; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "a declined assignment keeps a rename in flight on the same issue", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await rename(app, "Crash at boot"); + await app.fail("assign:3,ann", "declined"); + await app.see("list", "#3 Crash at boot (unassigned)"); + await app.land("renameIssue:3,Crash at boot", { id: 3, title: "Crash at boot" }); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 3 ? { ...i, title: "Crash at boot" } : i)); + await app.see("list", "#3 Crash at boot (unassigned)"); + } }, + { name: "a declined rename keeps an assignment in flight on the same issue", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.click("assign"); + await rename(app, "Crash at boot"); + await app.fail("renameIssue:3,Crash at boot", "declined"); + await app.see("list", "#3 Crash on save (ann)"); + await app.see("detail", "rename error: declined"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 3 ? { ...i, assignee: "ann" } : i)); + await app.see("list", "#3 Crash on save (ann)"); + } }, + { name: "a list that loads while a rename is out keeps the new title", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.click("filter-closed"); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:closed", []); + await app.landIfAsked("fetchIssues:open", OPEN); + await app.see("list", "#3 Crash at boot (unassigned)"); + await app.land("renameIssue:3,Crash at boot", { id: 3, title: "Crash at boot" }); + await app.see("list", "#3 Crash at boot (unassigned)"); + } }, + { name: "a detail that loads while a rename is out keeps the new title", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.click("retry"); + await app.land("fetchIssue:3", D3); + await app.see("detail", "#3 Crash at boot"); + await app.land("renameIssue:3,Crash at boot", { id: 3, title: "Crash at boot" }); + await app.see("detail", "#3 Crash at boot"); + } }, + { name: "an earlier rename declined after a later one was accepted keeps the later title", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await rename(app, "Crash on load"); + await app.land("renameIssue:3,Crash on load", { id: 3, title: "Crash on load" }); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 3 ? { ...i, title: "Crash on load" } : i)); + await app.fail("renameIssue:3,Crash at boot", "declined"); + await app.see("list", "#3 Crash on load (unassigned)"); + await app.see("detail", "#3 Crash on load"); + } }, +]; diff --git a/eval/tasks/rename/hidden.mjs b/eval/tasks/rename/hidden.mjs new file mode 100644 index 0000000..557b12b --- /dev/null +++ b/eval/tasks/rename/hidden.mjs @@ -0,0 +1,48 @@ +// Round 4 hidden tests: the change's own behaviour, and the base app's hidden tests as regressions. Never shown to +// the agent. +import base from "../tracker/hidden.mjs"; +import { D1, D3, OPEN, CLOSED, loaded, selected, rename } from "./fixtures.mjs"; + +export default [ + ...base.map(t => ({ ...t, name: `base: ${t.name}` })), + { name: "a declined rename reverts the row and the heading and shows the message", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.fail("renameIssue:3,Crash at boot", "declined"); + await app.see("list", "#3 Crash on save (unassigned)"); + await app.see("detail", "#3 Crash on save", "rename error: declined"); + await app.notSee("detail", "Crash at boot"); + } }, + { name: "a new rename clears the previous error at once", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.fail("renameIssue:3,Crash at boot", "declined"); + await rename(app, "Crash on load"); + await app.notSee("detail", "rename error"); + await app.see("list", "#3 Crash on load (unassigned)"); + } }, + { name: "selecting another issue clears the rename error", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.fail("renameIssue:3,Crash at boot", "declined"); + await app.clickText("issue", "#1 "); + await app.notSee("detail", "rename error"); + } }, + { name: "the title input is empty when another issue is selected", async run(app) { + await selected(app); + await app.type("title-input", "half typed"); + await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); + await app.hasValue("title-input", ""); + } }, + { name: "an accepted rename survives switching filters and back", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.land("renameIssue:3,Crash at boot", { id: 3, title: "Crash at boot" }); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 3 ? { ...i, title: "Crash at boot" } : i)); + await app.click("filter-closed"); + await app.land("fetchIssues:closed", CLOSED); + await app.click("filter-open"); + await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 3 ? { ...i, title: "Crash at boot" } : i)); + await app.see("list", "#3 Crash at boot (unassigned)"); + } }, +]; diff --git a/eval/tasks/rename/visible.mjs b/eval/tasks/rename/visible.mjs new file mode 100644 index 0000000..4732931 --- /dev/null +++ b/eval/tasks/rename/visible.mjs @@ -0,0 +1,49 @@ +// Visible tests: the agent sees and runs these. +import { CLOSED, D3, OPEN_ROWS, loaded, selected, rename } from "./fixtures.mjs"; + +export default [ + { name: "on load: the open issues load", async run(app) { + await app.see("list", "loading open issues"); + await app.see("detail", "select an issue"); + await loaded(app); + await app.rows("issue", OPEN_ROWS); + } }, + { name: "switching the filter loads the closed issues", async run(app) { + await loaded(app); + await app.click("filter-closed"); + await app.see("list", "loading closed issues"); + await app.land("fetchIssues:closed", CLOSED); + await app.rows("issue", ["#2 Old bug (eve)"]); + } }, + { name: "selecting an issue loads its detail", async run(app) { + await loaded(app); + await app.clickText("issue", "#3 "); + await app.see("detail", "loading #3"); + await app.land("fetchIssue:3", D3); + await app.see("detail", "#3 Crash on save", "stack trace attached"); + } }, + { name: "assigning shows the new assignee at once", async run(app) { + await selected(app); + await app.type("user", "ann"); + await app.see("assign", "assign to ann"); + await app.click("assign"); + await app.inAir("assign:3,ann"); + await app.see("list", "#3 Crash on save (ann)"); + await app.land("assign:3,ann", { id: 3, assignee: "ann" }); + await app.see("list", "#3 Crash on save (ann)"); + } }, + { name: "renaming shows the new title at once in the row and the heading", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.inAir("renameIssue:3,Crash at boot"); + await app.see("list", "#3 Crash at boot (unassigned)"); + await app.see("detail", "#3 Crash at boot"); + } }, + { name: "an accepted rename keeps the new title", async run(app) { + await selected(app); + await rename(app, "Crash at boot"); + await app.land("renameIssue:3,Crash at boot", { id: 3, title: "Crash at boot" }); + await app.see("list", "#3 Crash at boot (unassigned)"); + await app.see("detail", "#3 Crash at boot"); + } }, +]; From 587609bb2bdaf81e32512953e0031e8240915f44 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 21:41:26 +0000 Subject: [PATCH 08/11] checker: SHADOWED_IMPORT (MO-1d, taken by default) A declaration in a nested block may not shadow a name an import bound. In the round 4 eval draft, every .rx base app has `source filter`, which hid the prelude's `filter`: a call to it passed rxc check and failed at run time. Both checkers agree (287/287 crate programs). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- compiler/src/check.rs | 32 ++++++++++++++++++++++++-------- ref/src/check.ts | 13 ++++++++++--- scenarios/S-module-1.md | 15 ++++++++++++++- spec/09-modules.md | 7 +++++++ spec/99-diagnostics.md | 1 + 5 files changed, 56 insertions(+), 12 deletions(-) diff --git a/compiler/src/check.rs b/compiler/src/check.rs index eb2c39b..f8afe9d 100644 --- a/compiler/src/check.rs +++ b/compiler/src/check.rs @@ -748,6 +748,8 @@ pub struct Checker<'a> { apps: Vec<AppRec<'a>>, actions: Vec<ActionRec<'a>>, provisions: Vec<Provision<'a>>, + /// MO-1d: the symbols import declarations bound, which no declaration in a nested block may shadow. + import_syms: HashSet<SId>, later: Vec<Later<'a>>, comp_syms: HashMap<RId, SId>, rows_busy: HashSet<String>, @@ -819,6 +821,7 @@ impl<'a> Checker<'a> { apps: vec![], actions: vec![], provisions: vec![], + import_syms: HashSet::new(), later: vec![], comp_syms: HashMap::new(), rows_busy: HashSet::new(), @@ -982,13 +985,20 @@ impl<'a> Checker<'a> { // 1. names (E-7: one declaration per name in a block) for d in decls { self.line = d.line() as i64; - if let Decl::Import { resolved: Some(_), .. } = d { - self.import_rx(d, env); - continue; - } - if let Decl::Import { declared: Some(_), .. } = d { - self.import_declared(d, env); - continue; + if let Decl::Import { names, .. } = d { + if matches!(d, Decl::Import { resolved: Some(_), .. }) { + self.import_rx(d, env); + } else if matches!(d, Decl::Import { declared: Some(_), .. }) { + self.import_declared(d, env); + } + if matches!(d, Decl::Import { resolved: Some(_), .. } | Decl::Import { declared: Some(_), .. }) { + for n in names { + if let Some(&s) = self.envs[env].vars.get(&n.alias) { + self.import_syms.insert(s); + } + } + continue; + } } match d { Decl::Import { names, pure, fails, side, line, .. } => { @@ -997,7 +1007,8 @@ impl<'a> Checker<'a> { s.io = !pure; s.rejects = Some(fails.clone().unwrap_or_else(|| vec!["Error".into()]).into_iter().collect()); s.side = *side; - self.bind(env, &n.alias, s); + let s = self.bind(env, &n.alias, s); + self.import_syms.insert(s); } continue; } @@ -1034,6 +1045,11 @@ impl<'a> Checker<'a> { self.err("DUPLICATE_DECLARATION", "E-7", format!("{} is declared twice in one block", name)); } } + if let Some(shadowed) = self.envs[env].parent.and_then(|p| self.lookup(p, name)) { + if self.import_syms.contains(&shadowed) { + self.err("SHADOWED_IMPORT", "MO-1d", format!("{} shadows the imported {}: rename one (`import {{ {} as … }}`)", name, name, name)); + } + } let kind = match d { Decl::Fn { .. } => SymKind::Fn, Decl::Component { .. } => SymKind::Comp, diff --git a/ref/src/check.ts b/ref/src/check.ts index 1299359..7f965f6 100644 --- a/ref/src/check.ts +++ b/ref/src/check.ts @@ -207,6 +207,7 @@ class Checker { private apps: { comp: Region; site: Region; line: number; tag: string; args: { field: string; e: Expr; env: Env }[] }[] = []; private actions: { name: string; body: Stmt[]; env: Env; line: number }[] = []; private provisions: { key: string; e: Expr; env: Env }[] = []; + private importSyms = new Set<Sym>(); // MO-1d: what imports bound; a nested block may not shadow it private later: (() => void)[] = []; private compSyms = new Map<Region, Sym>(); private rowsBusy = new Set<string>(); @@ -236,9 +237,13 @@ class Checker { // 1. names (E-7: one declaration per name in a block) for (const d of decls) { this.line = d.line; - if (d.k === "import" && d.resolved) { this.importRx(d, env); continue; } // MO-1/MO-3: an .rx module's signature - if (d.k === "import" && d.declared) { this.importDeclared(d, env); continue; } // MO-8: a JS module's declaration file - if (d.k === "import") { for (const { alias } of d.names) this.bind(env, alias, { kind: "import", name: alias, region: env.region, line: d.line, io: !d.pure, rejects: new Set(d.fails ?? ["Error"]), side: d.side }); continue; } // R-io-4, R-f-8, SF-5 + if (d.k === "import") { + if (d.resolved) this.importRx(d, env); // MO-1/MO-3: an .rx module's signature + else if (d.declared) this.importDeclared(d, env); // MO-8: a JS module's declaration file + else for (const { alias } of d.names) this.bind(env, alias, { kind: "import", name: alias, region: env.region, line: d.line, io: !d.pure, rejects: new Set(d.fails ?? ["Error"]), side: d.side }); // R-io-4, R-f-8, SF-5 + for (const { alias } of d.names) { const s = env.vars.get(alias); if (s) this.importSyms.add(s); } + continue; + } if (d.k === "effect" || d.k === "provide" || d.k === "typedecl") continue; if (d.k === "context") { // RG-7: a declared context — its identity is the declaration, not its spelling if (env.vars.has(d.name)) this.err("DUPLICATE_DECLARATION", "E-7", `${d.name} is declared twice in one block`); @@ -249,6 +254,8 @@ class Checker { continue; } if (env.vars.has(d.name) && env.vars.get(d.name)!.kind !== "import") this.err("DUPLICATE_DECLARATION", "E-7", `${d.name} is declared twice in one block`); + const shadowed = env.parent?.lookup(d.name); + if (shadowed && this.importSyms.has(shadowed)) this.err("SHADOWED_IMPORT", "MO-1d", `${d.name} shadows the imported ${d.name}: rename one (\`import { ${d.name} as … }\`)`); const sym: Sym = { kind: d.k === "fn" ? "fn" : d.k === "component" ? "comp" : d.k === "action" ? "action" : d.k === "server" ? "server" : d.k === "source" ? "source" : d.k === "error" ? "errclass" : "eq", name: d.name, region: env.region, line: d.line }; if (d.k === "server") sym.params = d.params.map(name => ({ name, ref: false })); if (d.k === "source") sym.question = !!d.question; diff --git a/scenarios/S-module-1.md b/scenarios/S-module-1.md index 2cd06b3..3d5ce11 100644 --- a/scenarios/S-module-1.md +++ b/scenarios/S-module-1.md @@ -1,6 +1,6 @@ # S-module-1 — three modules: shared module state, read-only across the boundary, rows and provisions through signatures -Rules: MO-1, MO-2, MO-3, MO-4, MO-5, MO-6, R-f-5, R-f-6, RG-7b. +Rules: MO-1, MO-1d, MO-2, MO-3, MO-4, MO-5, MO-6, R-f-5, R-f-6, RG-7b. ## Modules @@ -185,3 +185,16 @@ export default component App() { `Save`'s action reads `props.v`: an argument that can fail is refused at the application, whatever boundary is above it (R-f-6). + +``` +-- reject: SHADOWED_IMPORT +import { initials } from "./store.rx" + +export default component App() { + source initials := "AB" + <p>{initials(initials)}</p> +} +``` + +MO-1d: the component's `initials` would hide the imported function, and the +call would call a string. Rename one: `import { initials as short }`. diff --git a/spec/09-modules.md b/spec/09-modules.md index 70f73d7..be40af3 100644 --- a/spec/09-modules.md +++ b/spec/09-modules.md @@ -30,6 +30,13 @@ see the list at the end. component, `export default component App() { … }`: the runtime mounts it, as Solid's `render(() => <App />, el)`. A module imported for its exports may have a default component too; it is a component like any other. +- **MO-1d.** [taken by default 2026-10-27] A declaration in a nested block + may not shadow a name an import bound (`SHADOWED_IMPORT`): rename one, + `import { filter as keep }`. A declaration that hides an imported + function turns every call of it in that block into a call of a + value, which nothing checks before run time (no type checker in the + loop, 11). Found in round 4 of the AI eval: a component's + `source filter` hid the prelude's `filter`. ## MO-2 — The value boundary diff --git a/spec/99-diagnostics.md b/spec/99-diagnostics.md index 6d5a504..e1d22d7 100644 --- a/spec/99-diagnostics.md +++ b/spec/99-diagnostics.md @@ -84,6 +84,7 @@ named. | Code | Bin | Rule | Scenario | | --------------------------------- | --- | ----- | -------- | | `DUPLICATE_DECLARATION` | S | E-7 | S-check-1 | +| `SHADOWED_IMPORT` (new) | S | MO-1d | S-module-1 | | `CYCLE` | S | C-1 (CN-3a: union of branches; only an equation's own `pre` breaks it) | S-check-1, S-core-1 | | `PRE_OF_NON_REACTIVE` | S | C-2 | S-check-1 | | `UNBOUND` | S | lexical scope (a component sees its declaration site, RG-7a) | S-check-1 | From 0238421229bcbe69768efd0bcb5a170c3fbe4bc6 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Tue, 27 Oct 2026 22:32:12 +0000 Subject: [PATCH 09/11] runtime: refresh re-asks the question under a whole-value guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source guessed as a whole (`optimistic list := …`) lowers to a createSignal truth with a createOptimistic layer over it. `refresh list` refreshed the layer, whose compute only reads the truth, so nothing was re-asked: the guess was corrected at the body's end and the accepted row vanished until something else re-asked. Refresh now re-asks the truth (rt-web, rt and the oracle). New S-solid-35 variant; compiled, oracle and ssr suites pass, and no round 3 score changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- ref/compile/rt-web.ts | 4 +++- ref/compile/rt.ts | 4 +++- ref/oracle/lower.ts | 6 +++--- scenarios/S-solid-35.md | 27 +++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ref/compile/rt-web.ts b/ref/compile/rt-web.ts index c9cd021..2db9769 100644 --- a/ref/compile/rt-web.ts +++ b/ref/compile/rt-web.ts @@ -89,6 +89,7 @@ export class Flight { export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) => unknown> }) { const flights = new Map<string, Flight>(); const lastKey = new Map<string, string>(); + const truthOf = new Map<Acc, Acc>(); // an optimistic signal-form source: its guess layer -> its question const registry = new Map<string, Entry[]>(); const contexts = new Map<string, ReturnType<typeof createContext<Entry | undefined>>>(); const storeAccs = new WeakSet<object>(); @@ -163,6 +164,7 @@ export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); const [get, setGuess] = createOptimistic(() => truth()); + truthOf.set(get as Acc, truth as Acc); // AC-4: `refresh x` re-asks the truth, not the guess layer over it return [get, { set: setTruth as any, path: () => { throw new Error("optimistic path writes are not lowered yet"); }, guess: setGuess as any }]; } if (mode.structured) { @@ -236,7 +238,7 @@ export function createApp(opts: { root: any; api?: Record<string, (...a: any[]) until: (p: () => unknown) => until(p), // AC-6 onSettled: (f: () => void) => onSettled(f), // EF-7 onFlush: (f: () => void) => onFlush(f), // EF-8 - refresh(name: string, get: Acc) { const key = lastKey.get(name); if (key) flights.delete(key); for (const h of hoistedBy.get(name) ?? []) refresh(h as any); refresh((storeAccs.has(get) ? get() : get) as any); }, + refresh(name: string, get: Acc) { const key = lastKey.get(name); if (key) flights.delete(key); for (const h of hoistedBy.get(name) ?? []) refresh(h as any); const t = truthOf.get(get) ?? get; refresh((storeAccs.has(t) ? t() : t) as any); }, affects: (get: Acc) => affects(get as any), preOr: (v: unknown, init: () => unknown) => v === undefined ? init() : v, diff --git a/ref/compile/rt.ts b/ref/compile/rt.ts index cb79ec2..d2dd861 100644 --- a/ref/compile/rt.ts +++ b/ref/compile/rt.ts @@ -99,6 +99,7 @@ export function makePortal<N>(R: { createElement: (t: string) => N; insertNode: export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> } = {}) { const flights = new Map<string, Flight>(); const lastKey = new Map<string, string>(); + const truthOf = new Map<Acc, Acc>(); // an optimistic signal-form source: its guess layer -> its question const registry = new Map<string, Entry[]>(); const contexts = new Map<string, ReturnType<typeof createContext<Entry | undefined>>>(); const storeAccs = new WeakSet<object>(); @@ -173,6 +174,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> if (mode.optimistic) { const [truth, setTruth] = createSignal(fn as any, opts as any); const [get, setGuess] = createOptimistic(() => truth()); + truthOf.set(get as Acc, truth as Acc); // AC-4: `refresh x` re-asks the truth, not the guess layer over it return [get, { set: setTruth as any, path: () => { throw new Error("optimistic path writes are not lowered yet"); }, guess: setGuess as any }]; } if (mode.structured) { @@ -244,7 +246,7 @@ export function createApp(opts: { api?: Record<string, (...a: any[]) => unknown> until: (p: () => unknown) => until(p), // AC-6 onSettled: (f: () => void) => onSettled(f), // EF-7 onFlush: (f: () => void) => onFlush(f), // EF-8 - refresh(name: string, get: Acc) { const key = lastKey.get(name); if (key) flights.delete(key); for (const h of hoistedBy.get(name) ?? []) refresh(h as any); refresh((storeAccs.has(get) ? get() : get) as any); }, + refresh(name: string, get: Acc) { const key = lastKey.get(name); if (key) flights.delete(key); for (const h of hoistedBy.get(name) ?? []) refresh(h as any); const t = truthOf.get(get) ?? get; refresh((storeAccs.has(t) ? t() : t) as any); }, affects: (get: Acc) => affects(get as any), preOr: (v: unknown, init: () => unknown) => v === undefined ? init() : v, diff --git a/ref/oracle/lower.ts b/ref/oracle/lower.ts index 803122f..2f6e981 100644 --- a/ref/oracle/lower.ts +++ b/ref/oracle/lower.ts @@ -80,7 +80,7 @@ export const fmt = (v: unknown): string => v === ERR ? "#error" : v === PENDING_ // ---------------------------------------------------------------- scopes type Binding = - | { k: "sig"; get: () => unknown; set: (v: unknown) => void; question: boolean; optimistic: boolean; setGuess?: (v: unknown) => void } + | { k: "sig"; get: () => unknown; set: (v: unknown) => void; question: boolean; optimistic: boolean; setGuess?: (v: unknown) => void; truth?: () => unknown } | { k: "store"; get: () => any; set: (v: unknown) => void; setPath: (f: (d: any) => any) => void; question: boolean; setGuess?: (f: (d: any) => any) => void } | { k: "memo"; get: () => unknown } | { k: "const"; value: unknown } @@ -245,7 +245,7 @@ export class Oracle { else if (this.optimisticTargets.has(d.name)) { // AC-3 over ST-5: the truth is a signal with a question, written plainly; the guess layer is createOptimistic over it const [truth, setTruth] = createSignal(compute as any, this.loadingValue(d.init, scope) as any); const [get, setGuess] = createOptimistic(() => truth()); - scope.vars.set(d.name, { k: "sig", get, set: setTruth as any, setGuess: setGuess as any, question: true, optimistic: true }); + scope.vars.set(d.name, { k: "sig", get, set: setTruth as any, setGuess: setGuess as any, question: true, optimistic: true, truth }); // AC-4: `refresh` re-asks `truth`, not the guess layer } else if (this.isStructuredAwait(d.init, d.name)) { const [store, setStore] = (createStore as any)(async (draft: Record<string, unknown>) => { const r = compute(draft); const v = r instanceof Flight ? await r.first : await r; if (v && typeof v === "object") { for (const k of Object.keys(draft)) if (!(k in (v as object))) delete draft[k]; Object.assign(draft, v); } }, {}); /* ST-5 structured: the derive writes its draft (Solid's createStore(fn, seed) idiom) */ scope.vars.set(d.name, { k: "store", get: () => store, set: () => { throw new Unsupported("root write to a derived store"); }, setPath: setStore, question: true }); } else { const [get, set] = createSignal(compute as any, this.loadingValue(d.init, scope) as any); scope.vars.set(d.name, { k: "sig", get, set: set as any, question: true, optimistic: false }); } @@ -447,7 +447,7 @@ export class Oracle { } case "throw": throw untrack(() => this.ev(s.e, scope, env)); case "action": this.lowerAction([], s.body, scope, env)(); return; - case "refresh": { const b = scope.lookup(s.name); if (!b || !("get" in b)) throw new Error("INVALID_REFRESH_TARGET"); const key = this.lastKey.get(s.name); if (key) this.forget(key); for (const h of this.hoistedByName.get(s.name) ?? []) refresh(h as any); refresh((b.k === "store" ? b.get() : b.get) as any); return; } + case "refresh": { const b = scope.lookup(s.name); if (!b || !("get" in b)) throw new Error("INVALID_REFRESH_TARGET"); const key = this.lastKey.get(s.name); if (key) this.forget(key); for (const h of this.hoistedByName.get(s.name) ?? []) refresh(h as any); refresh((b.k === "store" ? b.get() : (b.k === "sig" && b.truth) || b.get) as any); return; } case "affects": { const b = scope.lookup(s.name); if (!b || !("get" in b)) throw new Error("INVALID_AFFECTS_TARGET"); affects(b.get as any); return; } case "optimistic": case "write": { const b = s.target.ctx ? this.provision(s.target.name, scope) : scope.lookup(s.target.name); diff --git a/scenarios/S-solid-35.md b/scenarios/S-solid-35.md index 3808c57..0e964df 100644 --- a/scenarios/S-solid-35.md +++ b/scenarios/S-solid-35.md @@ -59,3 +59,30 @@ the body's end corrects the guess — `derived` re-asks from the truth (`x:1`), so instant 5's `resolve derived` lands that flight while the obsolete `x:5` (instant 4) is inert; `latest guess` is the truth from instant 3, and untracked `pending? guess` is `true` until the commit. + +## Variant: the guess's own question re-asked holds the change + +``` +export default component App() { + source list = await fetch("list") + action add() { try { optimistic list := list + "+mine"; await fetch("save"); refresh list } catch {} } + + errored fallback(e) { <p>error: {e.message}</p> } { + <p>{list}</p> + } +} +``` + +| instant | action | frame | +| ------- | ------------------------ | ----------- | +| 0 | mount | `` | +| 1 | resolve list "a" | `a` | +| 2 | handler `add()` | `a+mine` | +| 3 | resolve "save" 0 | `a+mine` | +| 4 | resolve list "a,mine" | `a,mine` | + +The other side of D-35, for a guess over a whole value (`createOptimistic`, +not a store): instant 3, `refresh list` re-asks the guess's own question, +so the body's end does not correct the guess; the change commits when the +new answer lands (instant 4). Lowered, `refresh` must re-ask the question +under the guess, not the guess's layer. From 4745ed24d0f66e7caff51500bf574ad5c1beeea0 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 28 Oct 2026 02:30:49 +0000 Subject: [PATCH 10/11] repin Solid to next @ ac28157 (#4329 fixed); round 4 tests allow H-11 lingering - ref/SOLID_NEXT and the crate's solidjs-compiler rev move to ac28157a (rc.19 plus solidjs/solid#4330). All ref suites pass with zero divergences. - eval builds against that checkout's packages. Round 3's four #4329 failures pass on it, as its README now records. - Round 4 overlap tests: when two adds or removes on one list are out, a declined one may stay until the other settles (one change, H-11; Ryan 2026-10-28: desired). The one still out must not go. - .rx references for close and comments use the plain Solid pattern: a whole-list guess, then refresh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- compiler/Cargo.lock | 2 +- compiler/Cargo.toml | 2 +- eval/package-lock.json | 125 ++++++++++++---------- eval/package.json | 6 +- eval/reference/close/rx/main.rx | 14 +-- eval/reference/comments/rx/main.rx | 22 ++-- eval/results/overlap-2026-10-27/README.md | 5 +- eval/tasks/close/hidden-overlap.mjs | 11 +- eval/tasks/comments/hidden-overlap.mjs | 6 +- ref/SOLID_NEXT | 2 +- 10 files changed, 95 insertions(+), 100 deletions(-) diff --git a/compiler/Cargo.lock b/compiler/Cargo.lock index 9c63c55..0d301fb 100644 --- a/compiler/Cargo.lock +++ b/compiler/Cargo.lock @@ -1153,7 +1153,7 @@ checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" [[package]] name = "solidjs-compiler" version = "0.0.0" -source = "git+https://github.com/solidjs/solid?rev=d2323b077c7dda32ebced7843c2ce2b788a91e8d#d2323b077c7dda32ebced7843c2ce2b788a91e8d" +source = "git+https://github.com/solidjs/solid?rev=ac28157aba6d03b806f0043bce70a87d515e953e#ac28157aba6d03b806f0043bce70a87d515e953e" dependencies = [ "html5ever", "htmlize", diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index f3a1c56..2b8a3b8 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -26,7 +26,7 @@ regex = "1" serde_json = { version = "1", features = ["preserve_order"] } # CP-1b: Solid's compiler, at the commit ref/ runs on (ref/SOLID_NEXT), and the Oxc release it is built against (CP-1c: # the program crosses the seam, so the versions must match) -solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "d2323b077c7dda32ebced7843c2ce2b788a91e8d", default-features = false } +solidjs-compiler = { git = "https://github.com/solidjs/solid", rev = "ac28157aba6d03b806f0043bce70a87d515e953e", default-features = false } oxc_allocator = "0.144" oxc_ast = "0.144" oxc_span = "0.144" diff --git a/eval/package-lock.json b/eval/package-lock.json index 7894660..2df2468 100644 --- a/eval/package-lock.json +++ b/eval/package-lock.json @@ -6,8 +6,8 @@ "": { "name": "rx-eval", "dependencies": { - "@solidjs/signals": "2.0.0-rc.19", - "@solidjs/web": "2.0.0-rc.19", + "@solidjs/signals": "file:../../solid-next/packages/signals", + "@solidjs/web": "file:../../solid-next/packages/web", "@tanstack/react-query": "5.90.2", "@types/react": "19.2.2", "@types/react-dom": "19.2.1", @@ -16,12 +16,70 @@ "eslint-plugin-react-hooks": "7.0.0", "react": "19.2.0", "react-dom": "19.2.0", - "solid-js": "2.0.0-rc.19", + "solid-js": "file:../../solid-next/packages/solid", "typescript": "5.9.3", "typescript-eslint": "8.46.0", "vite": "7.1.9" } }, + "../../solid-next/packages/signals": { + "name": "@solidjs/signals", + "version": "2.0.0-rc.19", + "license": "MIT", + "devDependencies": { + "@codspeed/vitest-plugin": "^5.4.0", + "@ianvs/prettier-plugin-sort-imports": "^4.1.1", + "@rollup/plugin-replace": "^6.0.3", + "@rollup/plugin-typescript": "^12.3.0", + "@types/node": "^25.0.8", + "rimraf": "^5.0.1", + "rollup": "^4.53.4", + "rollup-plugin-prettier": "^4.1.2", + "terser": "^5.49.0", + "tslib": "^2.8.1", + "typescript": "^6.0.3", + "vite": "^7.0.0", + "vitest": "^4.1.6" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "../../solid-next/packages/solid": { + "name": "solid-js", + "version": "2.0.0-rc.19", + "license": "MIT", + "dependencies": { + "@solidjs/signals": "^2.0.0-rc.19", + "csstype": "^3.1.0", + "seroval": "1.7.0", + "seroval-plugins": "1.7.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "../../solid-next/packages/web": { + "name": "@solidjs/web", + "version": "2.0.0-rc.19", + "license": "MIT", + "dependencies": { + "seroval": "1.7.0", + "seroval-plugins": "1.7.0" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "^5.4.0", + "@solidjs/signals": "workspace:*", + "fast-check": "3.23.2", + "solid-js": "workspace:*" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "solid-js": "^2.0.0-rc.19" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -1348,29 +1406,12 @@ ] }, "node_modules/@solidjs/signals": { - "version": "2.0.0-rc.19", - "resolved": "https://registry.npmjs.org/@solidjs/signals/-/signals-2.0.0-rc.19.tgz", - "integrity": "sha512-Epg4lyelYZnwG152q1U7lXAXp/2/jaQdaRbzh60rcOelSwmCMf/WN6FQuUEUwbn1p/QNyoJzdGSKPajkVe3zOg==", - "license": "MIT", - "engines": { - "node": ">=22.12.0" - } + "resolved": "../../solid-next/packages/signals", + "link": true }, "node_modules/@solidjs/web": { - "version": "2.0.0-rc.19", - "resolved": "https://registry.npmjs.org/@solidjs/web/-/web-2.0.0-rc.19.tgz", - "integrity": "sha512-FTHxQ8aE+vSHWc1exWw2Y+Brb727I4hCiWi+Y0DURbaqCou8xjCTzn8uiPi7mwC1tqpKSWfkn1Iq1ya/Bl9vYQ==", - "license": "MIT", - "dependencies": { - "seroval": "1.7.0", - "seroval-plugins": "1.7.0" - }, - "engines": { - "node": ">=22.12.0" - }, - "peerDependencies": { - "solid-js": "^2.0.0-rc.19" - } + "resolved": "../../solid-next/packages/web", + "link": true }, "node_modules/@tanstack/query-core": { "version": "5.90.2", @@ -2979,27 +3020,6 @@ "semver": "bin/semver.js" } }, - "node_modules/seroval": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.7.0.tgz", - "integrity": "sha512-FU+LUkmDzWNsJ6oubZFN3dGPKny53DRhPJG6nftEaXDL8MIn19+kEE1Gvc+63Go+qPOwVUlKIdKhsu2HN6tksw==", - "license": "MIT", - "engines": { - "node": ">=20" - } - }, - "node_modules/seroval-plugins": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.7.0.tgz", - "integrity": "sha512-WM4rArfPtLZriTuxC5DBvNulEwAaRcjNe/dkMOEesOcuvXf5nEwKf5Bw3AT92WHohqcc45+57GdtsCuBTI0mjw==", - "license": "MIT", - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "seroval": "^1.0" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -3022,19 +3042,8 @@ } }, "node_modules/solid-js": { - "version": "2.0.0-rc.19", - "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-2.0.0-rc.19.tgz", - "integrity": "sha512-tlA+rZT1n3XkuxW0sMIoRf/QpOa0H9N9ZDrZLQFueUkZp+hElG+EXWBojBYYi4lVBKV8l03RTNgv+lfVCYkp9w==", - "license": "MIT", - "dependencies": { - "@solidjs/signals": "^2.0.0-rc.19", - "csstype": "^3.1.0", - "seroval": "1.7.0", - "seroval-plugins": "1.7.0" - }, - "engines": { - "node": ">=22.12.0" - } + "resolved": "../../solid-next/packages/solid", + "link": true }, "node_modules/source-map-js": { "version": "1.2.2", diff --git a/eval/package.json b/eval/package.json index 94b088c..2ca6344 100644 --- a/eval/package.json +++ b/eval/package.json @@ -4,9 +4,9 @@ "type": "module", "description": "The .rx vs React benchmark harness (eval/README.md)", "dependencies": { - "solid-js": "2.0.0-rc.19", - "@solidjs/web": "2.0.0-rc.19", - "@solidjs/signals": "2.0.0-rc.19", + "solid-js": "file:../../solid-next/packages/solid", + "@solidjs/web": "file:../../solid-next/packages/web", + "@solidjs/signals": "file:../../solid-next/packages/signals", "react": "19.2.0", "react-dom": "19.2.0", "@tanstack/react-query": "5.90.2", diff --git a/eval/reference/close/rx/main.rx b/eval/reference/close/rx/main.rx index 08d0fa6..7768b2c 100644 --- a/eval/reference/close/rx/main.rx +++ b/eval/reference/close/rx/main.rx @@ -1,6 +1,6 @@ import pure { fetchIssues, fetchIssue } from "./api" import { assign, closeIssue } from "./api" -import pure { indexOf, map } from "./prelude" +import pure { indexOf, filter as keep } from "./prelude" export error NotFound(id: number) => `issue #${id} not found` @@ -13,16 +13,10 @@ export default component App() { source issues key id = await fetchIssues(filter) source assignError: string = "" source closeError: string = "" - // one flag per listed issue: a close guesses its own row's flag - source closing key id = map(issues, t => { id: t.id, on: false }) - action closeOne(id: number) { optimistic closeError := "" try { - let i = indexOf(closing, c => c.id == id) - if (i >= 0) { - optimistic closing[i].on := true - } + optimistic issues := keep(issues, t => t.id != id) await closeIssue(id) refresh issues } catch (e) { @@ -30,8 +24,6 @@ export default component App() { } } - fn isClosing(id: number) => filter == "open" && indexOf(closing, c => c.id == id && c.on) >= 0 - action assignTo(id: number, who: string) { optimistic assignError := "" try { @@ -60,7 +52,6 @@ export default component App() { {errored fallback(e) { <p>list error: {e.message}</p> } { <ul> {for it in issues { - {if !isClosing(it.id) { <li data-testid="issue" onClick={() => { selected := it.id onFlush(() => { @@ -68,7 +59,6 @@ export default component App() { closeError := "" }) }}>#{it.id} {it.title} ({it.assignee ?? "unassigned"})</li> - }} }} </ul> }} diff --git a/eval/reference/comments/rx/main.rx b/eval/reference/comments/rx/main.rx index 177b3f3..a48b9b8 100644 --- a/eval/reference/comments/rx/main.rx +++ b/eval/reference/comments/rx/main.rx @@ -1,6 +1,6 @@ import pure { fetchIssues, fetchIssue, fetchComments } from "./api" import { assign, addComment } from "./api" -import pure { indexOf, filter as keep } from "./prelude" +import pure { indexOf } from "./prelude" export error NotFound(id: number) => `issue #${id} not found` @@ -16,20 +16,18 @@ export default component App() { source commentInput: string := "" source comments = selected == null ? [] : await fetchComments(selected) - // posts in the air: shown under the selected issue's comments until their action ends - source sending: { id: number; author: string; text: string }[] := [] - action postComment(id: number, author: string, text: string) { optimistic postError := "" try { + optimistic comments := [...comments, { author: author, text: `${text} (sending…)` }] await addComment(id, author, text) refresh comments } catch (e) { postError := e.message } - sending := keep(sending, p => !(p.id == id && p.author == author && p.text == text)) } + action assignTo(id: number, who: string) { optimistic assignError := "" try { @@ -91,11 +89,6 @@ export default component App() { {for c in comments { <li data-testid="comment">{c.author}: {c.text}</li> }} - {for p in sending { - {if p.id == selected { - <li data-testid="comment">{p.author}: {p.text} (sending…)</li> - }} - }} </ul> }} }} @@ -103,12 +96,9 @@ export default component App() { <input data-testid="comment-input" value={commentInput} onInput={e => { commentInput := e.target.value }} /> <button data-testid="post" onClick={() => { if (commentInput != "") { - let p = { id: selected, author: user, text: commentInput } - onFlush(() => { - sending := [...sending, p] - commentInput := "" - }) - postComment(p.id, p.author, p.text) + let text = commentInput + onFlush(() => { commentInput := "" }) + postComment(selected, user, text) } }}>post</button> {if postError != "" { diff --git a/eval/results/overlap-2026-10-27/README.md b/eval/results/overlap-2026-10-27/README.md index e2bef55..6ba65be 100644 --- a/eval/results/overlap-2026-10-27/README.md +++ b/eval/results/overlap-2026-10-27/README.md @@ -40,8 +40,9 @@ it shipped is the same Solid runtime bug. `optimistic issues[i].assignee := who`, `await assign(…)`, `refresh issues`. By the spec the guess holds until the action ends (AC-3, D-34; Ryan's ruling of 2026-10-27). In rc.19, a list answer that lands while the action is pending drops a per-key guess from the screen (solidjs/solid#4329). The failure message matches the issue's repro -(the row shows the server's old assignee). This is an attribution by mechanism, not by rerunning on a fixed runtime: -no fix exists yet. +(the row shows the server's old assignee). Confirmed on 2026-10-28: with Solid next @ ac28157a (the #4329 fix, +solidjs/solid#4330), all four runs pass that test, and small tracker-2 still fails its two hidden tests. The +pre-registered result stands as above; the recount without #4329 is what the fixed runtime gives. **The same race, both arms.** "A list that loads while an assignment is out" is the one case both arms failed. 3 of 3 small-model React apps failed it in their own code (the refetch overwrote the optimistic cache). 0 of 3 diff --git a/eval/tasks/close/hidden-overlap.mjs b/eval/tasks/close/hidden-overlap.mjs index 2d49ef9..b8f43cd 100644 --- a/eval/tasks/close/hidden-overlap.mjs +++ b/eval/tasks/close/hidden-overlap.mjs @@ -12,7 +12,8 @@ export default [ await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.click("assign"); await app.fail("closeIssue:3", "declined"); - await app.rows("issue", ["#1 Login fails (ann)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + // the declined close may stay until the assignment settles (one change, H-11); the assignment must not go + await app.see("list", "#1 Login fails (ann)"); await app.land("assign:1,ann", { id: 1, assignee: "ann" }); await app.landIfAsked("fetchIssues:open", OPEN.map(i => i.id === 1 ? { ...i, assignee: "ann" } : i)); await app.rows("issue", ["#1 Login fails (ann)", "#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); @@ -24,7 +25,8 @@ export default [ await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.click("close"); await app.fail("assign:3,ann", "declined"); - await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + // the declined assignment may stay until the close settles (one change, H-11); the close must not undo + await app.notSee("list", "#1 Login fails"); await app.land("closeIssue:1", { id: 1 }); await app.landIfAsked("fetchIssues:open", openWithout(1)); await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); @@ -41,13 +43,14 @@ export default [ await app.landIfAsked("fetchIssues:open", openWithout(3)); await app.rows("issue", rowsWithout(3)); } }, - { name: "two closes at once: declining one brings back only that row", async run(app) { + { name: "two closes at once: a declined one does not undo the other", async run(app) { await selected(app); await app.click("close"); await app.clickText("issue", "#1 "); await app.land("fetchIssue:1", D1); await app.click("close"); await app.fail("closeIssue:3", "declined"); - await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); + // the declined close may stay until the other settles (one change, H-11); the other close must not undo + await app.notSee("list", "#1 Login fails"); await app.land("closeIssue:1", { id: 1 }); await app.landIfAsked("fetchIssues:open", openWithout(1)); await app.rows("issue", ["#3 Crash on save (unassigned)", "#4 Typo in footer (dana)"]); diff --git a/eval/tasks/comments/hidden-overlap.mjs b/eval/tasks/comments/hidden-overlap.mjs index a134a41..09ed58c 100644 --- a/eval/tasks/comments/hidden-overlap.mjs +++ b/eval/tasks/comments/hidden-overlap.mjs @@ -7,13 +7,15 @@ const after = (...more) => [...C3, ...more]; export default [ ...base.map(t => ({ ...t, name: `base: ${t.name}` })), - { name: "two posts at once: declining the first keeps the second", async run(app) { + { name: "two posts at once: a declined one does not take the other with it", async run(app) { await withComments(app); await app.type("user", "ann"); await post(app, "first"); await post(app, "second"); await app.fail("addComment:3,ann,first", "declined"); - await app.rows("comment", ["dana: seen it too", "ann: second (sending…)"]); + // the declined post may stay until the other settles (two guesses on one list are one change, H-11); the one + // still out must not go + await app.see("comments", "ann: second (sending…)"); await app.land("addComment:3,ann,second", { author: "ann", text: "second" }); await app.landIfAsked("fetchComments:3", after({ author: "ann", text: "second" })); await app.rows("comment", ["dana: seen it too", "ann: second"]); diff --git a/ref/SOLID_NEXT b/ref/SOLID_NEXT index d6eb76f..15f9aba 100644 --- a/ref/SOLID_NEXT +++ b/ref/SOLID_NEXT @@ -1 +1 @@ -d2323b077c7dda32ebced7843c2ce2b788a91e8d +ac28157aba6d03b806f0043bce70a87d515e953e From 4ba4c7185a7ffa8995bb30611e6aad1a4f533a6c Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 28 Oct 2026 05:24:57 +0000 Subject: [PATCH 11/11] eval: round 4 pre-registration (change an existing app) Freezes the round 4 hidden tests (SHA-256 in eval/PREREG-change.md). The comments test that shows solidjs/solid#4336 is run and reported alongside, outside the defect count (score.mjs 'alongside'). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117BoWKEv4MghW8Ns9LeUUX --- eval/PREREG-change.md | 78 ++++++++++++++++++++++++++++++++++++++++++ eval/harness/score.mjs | 11 ++++-- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 eval/PREREG-change.md diff --git a/eval/PREREG-change.md b/eval/PREREG-change.md new file mode 100644 index 0000000..460a379 --- /dev/null +++ b/eval/PREREG-change.md @@ -0,0 +1,78 @@ +# Round 4: changing an existing app + +Pre-registration. Everything in this file was fixed and committed before the round's first agent run. + +## Why this round + +Rounds 2 and 3 (`results/plain-2026-10-27/`, `results/overlap-2026-10-27/`) had agents write small apps from +scratch. Neither found a defect gap. This round asks the other half of Ryan's question: does an agent keep an app +correct when it **changes** one it did not write, where the new feature adds writes that overlap the app's existing +ones? + +## Design + +- **Bases**: round 3's frontier-model tracker apps, `results/overlap-2026-10-27/runs/big/tracker-<arm>-<n>`. Run n of + each task and arm starts from base n of that arm, for both models. On Solid next @ ac28157a every base passes all + of the tracker's visible, hidden and hidden-overlap tests. +- **Tasks** (`tasks/<task>/brief-plain.md`, the change; `BASE.md`, the tracker's plain brief, which the app must keep + meeting). None names a race. + - rename: rename the selected issue (`renameIssue`), shown in the list and the detail straight away. + - close: close an issue (`closeIssue`); it leaves the list straight away and comes back if declined. + - comments: load the selected issue's comments (`fetchComments`) and post one (`addComment`), shown straight away + with `(sending…)`. +- **Tests**: per task `visible.mjs` (the tracker's 4 plus 2 new), `hidden.mjs` (new tests plus the tracker's hidden + tests as regressions) and `hidden-overlap.mjs` (new overlap tests plus the tracker's overlap tests). rename 16 + 7, + close 15 + 6, comments 17 + 6 hidden and overlap tests. +- **Prompt**: `harness/prompt-change.md`. Each run keeps scratch files in its own `.scratch` folder (round 3's leak). +- **Toolchain**: Solid next @ ac28157a (2.0.0-rc.19 plus the #4329 fix, solidjs/solid#4330) in both arms, through + `eval/package.json`. The .rx compiler and runtime from this branch: MO-1d `SHADOWED_IMPORT` (taken by default + 2026-10-27), and `refresh` re-asking the question under a whole-value guess (`ref/compile/rt-web.ts`). The .rx + arm gets `docs/spec` and `docs/guide/async.md` as in round 3, unchanged. +- 3 tasks × 2 arms × 2 models (frontier, Haiku) × 3 runs = 36 fresh runs, 15 `./check` + `./test` each. + +Hidden test files, SHA-256 at freezing: + +``` +869e02eceb679c5907a67571a26f061157eefd86e8e38a1a7aec8db7d9e65fc0 tasks/rename/hidden.mjs +532e2caf837f3effaf77c5979af2651ed4d0ec2973b496d76846035d45831403 tasks/rename/hidden-overlap.mjs +86aebac10d51c9244504e5da3b368c356e6b8c65dfab5d9dbc35b03adb66604d tasks/close/hidden.mjs +669c606395e539e031ca7f376304a42607d15ffccb823b291b5d616f30ddce42 tasks/close/hidden-overlap.mjs +25d241a2309310943ed24d72d26c2a7e638a6feecbd3397b39123ebf5e956dab tasks/comments/hidden.mjs +3d95d4274efea4cd7d793140d2daab3f6b087bcd22c6b4ff7e5a8492d4d361ec tasks/comments/hidden-overlap.mjs +c02e80f9b64e44dd843b601713ddc67553a5f85835e36ee5d13c35fb4121d09e tasks/tracker/hidden.mjs +5d3e09edb9f0c2845aba82fc43b9689c8fe1066167fefa2dd04cbb93f4711b3e tasks/tracker/hidden-overlap.mjs +``` + +**References** (`reference/<task>/<arm>`, not subjects) were written on the bases. Both arms pass every test, except +the .rx comments reference on "a post on one issue does not show on another" (below). + +**Tests loosened while writing the references** (before freezing). By H-11 two guesses on one slot from two actions +are one change, so a declined guess can stay on screen until the other action settles. Ryan (2026-10-28): that is the +desired behaviour. Four tests were changed to accept it: close's "a declined close keeps an assignment…", "a declined +assignment keeps a close…", "two closes at once: a declined one does not undo the other", and comments' "two posts at +once: a declined one does not take the other with it". Each still checks the end state after both settle. + +## Hypothesis and test + +- **H1**: the React arm ships more defects than the .rx arm. +- **Primary measure**: defects shipped: a failed build or any failed visible, hidden or hidden-overlap test at the + agent's "done" (`node harness/score.mjs <dir> --overlap`), except the one test below. One number per arm, pooled + over tasks and models (18 runs each). +- **Primary test**: Fisher's exact test, two-sided, α = 0.05 (`node harness/compare.mjs`). H1 holds if the React + rate is higher and p < 0.05. **H1 reversed** is reported as plainly. +- **Outside the count, reported alongside**: comments' "a post on one issue does not show on another". The .rx + idiom for it (a whole-value guess on the comments source) shows solidjs/solid#4336 (filed 2026-10-28): the guess + stays on screen after the selected issue changes, until the post settles. `score.mjs` runs it and lists it per run + under `failedAlongside`, for both arms, and does not count it. +- **Other runtime bugs** found during the round count as defects, as in rounds 2 and 3, and are listed separately + with the primary result recomputed without them. +- **Reported alongside**: the same table per model and per task; base regressions (tracker tests failed) apart from + new-feature failures; each failure classified by hand; agent tokens and cycles. +- Published as is, whatever the outcome. + +## Known limits + +- Isolation is still by instruction. +- The bases are frontier-model apps; the small model changes code it would not have written. +- The new tests were written after round 3's results were known, and tuned on the references. +- 18 runs per arm detects a large effect only. diff --git a/eval/harness/score.mjs b/eval/harness/score.mjs index a518ac0..a7d840a 100644 --- a/eval/harness/score.mjs +++ b/eval/harness/score.mjs @@ -1,6 +1,7 @@ // Scores finished runs: for each run directory (named <task>-<arm>-<n>), the arm's check, the visible and the hidden // tests, and the agent's own log of `./check` and `./test` runs. Writes results.json and prints a summary table. // With --overlap, each task's hidden-overlap.mjs (round 3) runs too and counts toward the defect. +// Tests named in `alongside` are run and reported but do not count toward the defect (round 4 pre-registration). // Usage: node harness/score.mjs <runs-dir> [out.json] [--overlap] import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; @@ -13,6 +14,8 @@ const tasksDir = path.resolve(here, "../tasks"); const overlap = process.argv.includes("--overlap"); const [runsArg, outArg] = process.argv.slice(2).filter(a => a !== "--overlap"); const runs = path.resolve(runsArg); +// solidjs/solid#4336: the .rx idiom for this case shows the bug in the runtime, not in the app's code. +const alongside = new Set(["a post on one issue does not show on another"]); const rows = []; for (const name of readdirSync(runs).sort()) { @@ -26,8 +29,9 @@ for (const name of readdirSync(runs).sort()) { const visible = await runFile(dir, path.join(tasksDir, task, "visible.mjs")); const hidden = await runFile(dir, path.join(tasksDir, task, "hidden.mjs")); const extra = overlap ? await runFile(dir, path.join(tasksDir, task, "hidden-overlap.mjs")) : null; - const failedHidden = hidden.results.filter(r => !r.ok); - const failedOverlap = extra ? extra.results.filter(r => !r.ok) : []; + const failedHidden = hidden.results.filter(r => !r.ok && !alongside.has(r.name)); + const failedOverlap = extra ? extra.results.filter(r => !r.ok && !alongside.has(r.name)) : []; + const failedAlongside = [...hidden.results, ...(extra ? extra.results : [])].filter(r => !r.ok && alongside.has(r.name)); rows.push({ run: name, task, arm, n: Number(n), checks: log.filter(e => e.cmd === "check").length, tests: log.filter(e => e.cmd === "test").length, @@ -38,9 +42,10 @@ for (const name of readdirSync(runs).sort()) { ...(extra ? { overlap: `${extra.results.length - failedOverlap.length}/${extra.results.length}`, failedOverlap: failedOverlap.map(r => ({ name: r.name, error: r.error })) } : {}), shippedDefect: !hidden.build.ok || visible.results.some(r => !r.ok) || failedHidden.length > 0 || failedOverlap.length > 0, failedHidden: failedHidden.map(r => ({ name: r.name, error: r.error })), + failedAlongside: failedAlongside.map(r => ({ name: r.name, error: r.error })), failedVisible: visible.results.filter(r => !r.ok).map(r => ({ name: r.name, error: r.error })), }); const r = rows.at(-1); - console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}`); + console.log(`${r.run.padEnd(18)} check ${r.checkOk ? "ok " : "ERR"} visible ${r.visible} hidden ${r.hidden.padEnd(5)}${r.overlap ? ` overlap ${r.overlap}` : ""} cycles ${r.checks + r.tests} (green after ${r.cyclesToGreen ?? "never"})${r.shippedDefect ? " DEFECT" : ""}${r.failedAlongside.length ? " (alongside failed)" : ""}`); } writeFileSync(outArg ? path.resolve(outArg) : path.join(runs, "results.json"), JSON.stringify(rows, null, 2) + "\n");