From 550b8ff569357a5d5dbe8a5757ebc616e28f4aeb Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sat, 11 Jul 2026 17:53:43 +0800 Subject: [PATCH] Add S2 token alignment with safe edit bounds and large-doc gate. --- src/target-go.ts | 55 +++++++++++++++++++++++++-- src/target-rust.ts | 66 +++++++++++++++++++++++++++++--- src/target-ts.ts | 41 ++++++++++++++++++-- test/portable-targets.ts | 81 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 224 insertions(+), 19 deletions(-) diff --git a/src/target-go.ts b/src/target-go.ts index 87c3a7a..5ff9948 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -552,17 +552,65 @@ func parse(t []Tok) int32 { } type Edit struct { Start, End int; Text string } -type Doc struct { text string; root int32 } +type alignMeta struct { Kind string; Off, End int; Nl bool } +type Align struct { +\tOldN int \`json:"oldN"\` +\tNewN int \`json:"newN"\` +\tPrefix int \`json:"prefix"\` +\tSuffix int \`json:"suffix"\` +} +func toMeta(toks []Tok) []alignMeta { +\tm := make([]alignMeta, len(toks)) +\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl} } +\treturn m +} +func computeAlign(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) Align { +\toldN, newN := len(oldToks), len(newToks) +\tprefix := 0 +\tfor prefix < oldN && prefix < newN { +\t\to, n := oldToks[prefix], newToks[prefix] +\t\tif o.Kind != n.Kind || o.Off != n.Off || o.End != n.End || o.Nl != n.Nl { break } +\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } +\t\tprefix++ +\t} +\tdelta := len(newText) - len(oldText) +\tminN := oldN; if newN < minN { minN = newN } +\tsuffix := 0 +\tfor prefix+suffix < minN { +\t\to, n := oldToks[oldN-1-suffix], newToks[newN-1-suffix] +\t\tif o.Kind != n.Kind || o.Nl != n.Nl || n.Off != o.Off+delta || n.End != o.End+delta { break } +\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } +\t\tsuffix++ +\t} +\treturn Align{oldN, newN, prefix, suffix} +} +type Doc struct { text string; root int32; toks []alignMeta; align *Align } func NewDoc(src string) *Doc { \td := &Doc{text: src} +\td.toks = toMeta(tokenize(src)) \td.root = parse(tokenize(src)) \treturn d } func (d *Doc) Text() string { return d.text } func (d *Doc) Root() int32 { return d.root } +func (d *Doc) Align() *Align { return d.align } +func applyEdit(text string, e Edit) string { +\tn := len(text) +\tstart, end := e.Start, e.End +\tif start < 0 { start = 0 } +\tif start > n { start = n } +\tif end < start { end = start } +\tif end > n { end = n } +\treturn text[:start] + e.Text + text[end:] +} func (d *Doc) Edit(edits []Edit) int32 { -\tfor _, e := range edits { d.text = d.text[:e.Start] + e.Text + d.text[e.End:] } -\td.root = parse(tokenize(d.text)) +\toldText, oldToks := d.text, d.toks +\tfor _, e := range edits { d.text = applyEdit(d.text, e) } +\tnewToks := tokenize(d.text) +\td.toks = toMeta(newToks) +\ta := computeAlign(oldText, oldToks, d.text, d.toks) +\td.align = &a +\td.root = parse(newToks) \treturn d.root } `; @@ -610,6 +658,7 @@ func main() { \t\t\t} \t\t\td.Edit(edits) \t\t} +\t\tif a := d.Align(); a != nil { b, _ := json.Marshal(a); os.Stderr.Write(append(b, '\\n')) } \t\troot := d.Root() \t\tif root < 0 || pos != len(toks) { fmt.Fprintf(os.Stderr, "parse error (pos %d/%d)\\n", pos, len(toks)); os.Exit(1) } \t\tvar b strings.Builder diff --git a/src/target-rust.ts b/src/target-rust.ts index 1961568..d5b808b 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -548,14 +548,65 @@ fn parse<'a>(tokens: Tokens<'a>) -> Option<(Parser<'a>, i32)> { } pub struct Edit { pub start: usize, pub end: usize, pub text: String } -pub struct Doc { text: String } +#[derive(Clone)] +struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool } +struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize } +fn to_meta(toks: &[Tok<'_>]) -> Vec { + toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }).collect() +} +fn compute_align(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> Align { + let old_n = old_toks.len(); + let new_n = new_toks.len(); + let mut prefix = 0usize; + while prefix < old_n && prefix < new_n { + let o = &old_toks[prefix]; + let n = &new_toks[prefix]; + if o.kind != n.kind || o.off != n.off || o.end != n.end || o.nl != n.nl { break; } + if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } + prefix += 1; + } + let delta = new_text.len() as isize - old_text.len() as isize; + let min_n = old_n.min(new_n); + let mut suffix = 0usize; + while prefix + suffix < min_n { + let o = &old_toks[old_n - 1 - suffix]; + let n = &new_toks[new_n - 1 - suffix]; + if o.kind != n.kind || o.nl != n.nl { break; } + if n.off != (o.off as isize + delta) as usize || n.end != (o.end as isize + delta) as usize { break; } + if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } + suffix += 1; + } + Align { old_n, new_n, prefix, suffix } +} +fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec> { + meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect() +} +pub struct Doc { text: String, toks: Vec, align: Option } impl Doc { - pub fn new(text: String) -> Doc { Doc { text } } + pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: to_meta(&lex(&text)), align: None } } pub fn text(&self) -> &str { &self.text } + pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() } pub fn edit(&mut self, edits: &[Edit]) { - for e in edits { let s = e.start; let en = e.end; self.text = format!("{}{}{}", &self.text[..s], e.text, &self.text[en..]); } + let old_text = self.text.clone(); + let old_toks = self.toks.clone(); + for e in edits { + let n = self.text.len(); + let start = e.start.min(n); + let end = e.end.max(start).min(n); + self.text = format!("{}{}{}", &self.text[..start], e.text, &self.text[end..]); + } + self.toks = to_meta(&lex(&self.text)); + self.align = Some(compute_align(&old_text, &old_toks, &self.text, &self.toks)); + } + pub fn parse(&self) -> Option<(Parser<'_>, i32)> { + let toks = toks_from_meta(&self.text, &self.toks); + let n = toks.len(); + let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), suppress_cur: Vec::new(), src: &self.text, nodes: Vec::new(), kids: Vec::new(), scratch: Vec::new() }; + match p.parse_${ir.entry}() { + Some(root) if p.pos == n => Some((p, root)), + _ => None, + } } - pub fn parse(&self) -> Option<(Parser<'_>, i32)> { parse(tokenize(&self.text)) } } `; }, @@ -665,8 +716,13 @@ fn main() { let edits: Vec = batch.iter().map(|&(s, e, ref t)| Edit { start: s, end: e, text: t.clone() }).collect(); doc.edit(&edits); } + if let Some(a) = doc.alignment() { + eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix); + } match doc.parse() { - Some((p, root)) => { let mut out = String::new(); write_json(&p, root, &mut out); print!("{}", out); } + Some((p, root)) => { + let mut out = String::new(); write_json(&p, root, &mut out); print!("{}", out); + } None => { eprintln!("parse error"); std::process::exit(1); } } return; diff --git a/src/target-ts.ts b/src/target-ts.ts index 5cb84a1..08a811e 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -473,15 +473,48 @@ export function parse(tokens: Tok[]): Cst | null { } export type Edit = { start: number; end: number; text: string }; -export function createDoc(src: string): { text(): string; root(): Node | null; edit(edits: Edit[]): Node | null } { +type AlignMeta = { kind: string; off: number; end: number; nl: boolean }; +type Align = { oldN: number; newN: number; prefix: number; suffix: number }; +const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl })); +function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Align { + const oldN = oldToks.length, newN = newToks.length; + let prefix = 0; + while (prefix < oldN && prefix < newN) { + const o = oldToks[prefix], n = newToks[prefix]; + if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + prefix++; + } + const delta = newText.length - oldText.length; + const minN = Math.min(oldN, newN); + let suffix = 0; + while (prefix + suffix < minN) { + const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix]; + if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + suffix++; + } + return { oldN, newN, prefix, suffix }; +} +export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } { let text = src; + let prevToks = toMeta(tokenize(src)); + let align: Align | null = null; let root: Node | null = parse(tokenize(src)); return { text(): string { return text; }, root(): Node | null { return root; }, + align(): Align | null { return align; }, edit(edits: Edit[]): Node | null { - for (const e of edits) text = text.slice(0, e.start) + e.text + text.slice(e.end); - root = parse(tokenize(text)); + const oldText = text, oldToks = prevToks; + for (const e of edits) { + const n = text.length, start = Math.max(0, Math.min(e.start, n)), end = Math.max(start, Math.min(e.end, n)); + text = text.slice(0, start) + e.text + text.slice(end); + } + const newToks = tokenize(text); + prevToks = toMeta(newToks); + align = computeAlign(oldText, oldToks, text, prevToks); + root = parse(newToks); return root; }, }; @@ -497,6 +530,8 @@ if (process.argv.includes('edit-session')) { const { init, batches } = JSON.parse(_raw) as { init: string; batches: [number, number, string][][] }; const doc = createDoc(init); for (const batch of batches) doc.edit(batch.map(([start, end, text]) => ({ start, end, text }))); + const a = doc.align(); + if (a) process.stderr.write(JSON.stringify(a) + '\\n'); const root = doc.root(); if (root === null) { process.stderr.write('parse error\\n'); process.exit(1); } process.stdout.write(JSON.stringify(root)); diff --git a/test/portable-targets.ts b/test/portable-targets.ts index 666ab05..a11df1c 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -17,9 +17,10 @@ // // Go/Rust toolchains are optional: a missing `go`/`rustc` is logged and skipped (the TS // rendering, which needs only node, always runs). -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { createParser } from '../src/gen-parser.ts'; +import { createLexer } from '../src/gen-lexer.ts'; import { emitParser, tsTarget, goTarget, rustTarget } from '../src/emit.ts'; import type { CstGrammar } from '../src/types.ts'; @@ -270,21 +271,75 @@ function runProc(cmd: string, args: string[], src: string): Outcome { try { return { ok: true, cst: canon(JSON.parse(execFileSync(cmd, args, { input: src, stdio: ['pipe', 'pipe', 'pipe'] }).toString())) }; } catch { return { ok: false }; } } +type Align = { oldN: number; newN: number; prefix: number; suffix: number }; +type EditOutcome = Outcome & { align?: Align }; +function parseAlignStderr(stderr: string): Align | undefined { + for (const line of stderr.split('\n')) { + const t = line.trim(); + if (!t.startsWith('{')) continue; + try { return JSON.parse(t) as Align; } catch { /* keep scanning */ } + } + return undefined; +} +function runEditSession(cmd: string, args: string[], json: string): EditOutcome { + const r = spawnSync(cmd, args, { input: json, encoding: 'utf8' }); + const align = parseAlignStderr(r.stderr ?? ''); + if (r.status !== 0) return { ok: false, align }; + try { + return { ok: true, cst: canon(JSON.parse(r.stdout)), align }; + } catch { return { ok: false, align }; } +} + +type AlignMeta = { kind: string; off: number; end: number; nl: boolean }; +const oracleToks = (g: CstGrammar, src: string): AlignMeta[] => { + const { tokenize } = createLexer(g); + return tokenize(src).map((t) => ({ kind: t.type, off: t.offset, end: t.offset + t.text.length, nl: t.newlineBefore })); +}; +const computeAlign = (oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Align => { + const oldN = oldToks.length, newN = newToks.length; + let prefix = 0; + while (prefix < oldN && prefix < newN) { + const o = oldToks[prefix], n = newToks[prefix]; + if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + prefix++; + } + const delta = newText.length - oldText.length; + const minN = Math.min(oldN, newN); + let suffix = 0; + while (prefix + suffix < minN) { + const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix]; + if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + suffix++; + } + return { oldN, newN, prefix, suffix }; +}; +const expectAlign = (g: CstGrammar, init: string, batches: EditBatch[]): Align => { + let text = init; + for (let i = 0; i < batches.length - 1; i++) for (const [s, e, r] of batches[i]) text = text.slice(0, s) + r + text.slice(e); + const oldText = text; + const batch = batches[batches.length - 1]; + for (const [s, e, r] of batch) text = text.slice(0, s) + r + text.slice(e); + return computeAlign(oldText, oracleToks(g, oldText), text, oracleToks(g, text)); +}; type EditBatch = [number, number, string][]; -type EditScenario = { init: string; batches: EditBatch[] }; +type EditScenario = { init: string; batches: EditBatch[]; large?: boolean }; const EDIT_SCENARIOS: Record = { calc: [ { init: '1+2*3', batches: [[[3, 3, '4']]] }, { init: '1+2*3', batches: [[[1, 3, '']]] }, { init: '1+2*3', batches: [[[2, 5, '(7-8)']]] }, { init: '1+2*3', batches: [[[0, 0, '9-']], [[7, 8, '']]] }, + { init: '1+2*3+'.repeat(199) + '1+2*3', batches: [[[600, 601, '9']]], large: true }, ], javascript: [ { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]] }, { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]] }, { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]] }, { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]] }, + { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true }, ], }; const applyEdits = (init: string, batches: EditBatch[]): string => { @@ -358,12 +413,10 @@ for (const c of CASES) { const editScenarios = EDIT_SCENARIOS[c.grammar]; if (editScenarios) { - let editOk = 0; - const runEdit = (json: string) => runProc( - r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`, - r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session'], - json, - ); + let editOk = 0, alignOk = 0; + const editCmd = r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`; + const editArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session']; + const runEdit = (json: string) => runEditSession(editCmd, editArgs, json); for (const sc of editScenarios) { const final = applyEdits(sc.init, sc.batches); const a = runEdit(JSON.stringify({ init: sc.init, batches: sc.batches })); @@ -373,8 +426,20 @@ for (const c of CASES) { failures++; console.log(` ${c.grammar}/${r.label}: edit-session mismatch (final=${JSON.stringify(final)}) A ok=${a.ok} B ok=${b.ok}`); } + const want = expectAlign(grammar, sc.init, sc.batches); + if (a.align && a.align.oldN === want.oldN && a.align.newN === want.newN && a.align.prefix === want.prefix && a.align.suffix === want.suffix) { + alignOk++; + if (sc.large && want.prefix + want.suffix < want.oldN - 8) { + failures++; + console.log(` ${c.grammar}/${r.label}: large-doc align too narrow prefix+suffix=${want.prefix + want.suffix} oldN=${want.oldN}`); + } + } else { + failures++; + console.log(` ${c.grammar}/${r.label}: token-align mismatch want=${JSON.stringify(want)} got=${JSON.stringify(a.align)}`); + } } console.log(` ${c.grammar}/${r.label}: ${editOk}/${editScenarios.length} edit-sessions ≡ fresh`); + console.log(` ${c.grammar}/${r.label}: ${alignOk}/${editScenarios.length} token-alignments ≡ oracle`); } } }