From 66811dfb34efcf424ac9121ee81eb2521e586c57 Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sun, 12 Jul 2026 01:37:47 +0800 Subject: [PATCH] Extend windowed relex to newline-mode portable lexers (issue #57 S4). Newline-only grammars extract lexFrom with flowDepth-aware AlignMeta, resync, and streamEq checks while regex/template grammars keep full fallback. Co-authored-by: Cursor --- src/target-go.ts | 192 ++++++++++++++++++++++++++------- src/target-rust.ts | 228 ++++++++++++++++++++++++++++++++------- src/target-ts.ts | 173 +++++++++++++++++++++++------ test/portable-targets.ts | 10 +- 4 files changed, 492 insertions(+), 111 deletions(-) diff --git a/src/target-go.ts b/src/target-go.ts index 7f125d8..04846e3 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -81,7 +81,7 @@ function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, t return `\t\tif ${gate ? gate + 'true' : 'true'} { if e := ${m}(pos); e > pos { ${push('e')}pos = e; continue } }`; } -function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; boundary: string; ws: string; hooks: string } { +function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; stateFrom: string; boundary: string; ws: string; hooks: string } { const commentSkip = nl.comment ? `\t\tif strings.HasPrefix(src[p:], ${J(nl.comment)}) { e := p; for e < n && src[e] != 10 { e++ }; pos = e; continue }\n` : ''; @@ -90,6 +90,10 @@ function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; bounda \t_flowOpen := map[string]bool{${nl.flowOpen.map((x) => `${J(x)}: true`).join(', ')}} \t_flowClose := map[string]bool{${nl.flowClose.map((x) => `${J(x)}: true`).join(', ')}} \tconst _nlTok = ${J(nl.token)} +`, + stateFrom: `\t_flowOpen := map[string]bool{${nl.flowOpen.map((x) => `${J(x)}: true`).join(', ')}} +\t_flowClose := map[string]bool{${nl.flowClose.map((x) => `${J(x)}: true`).join(', ')}} +\tconst _nlTok = ${J(nl.token)} `, boundary: `\t\tif flowDepth == 0 && lineStart { \t\t\tp := pos @@ -117,7 +121,7 @@ ${commentSkip}\t\t\tpos = p ws: `\t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } \t\tif c == 10 || c == 13 { \t\t\tpos++; if c == 13 && pos < n && src[pos] == 10 { pos++ } -\t\t\tif flowDepth == 0 { lineStart = true } else { pendingNl = true } +\t\t\tif flowDepth == 0 { lineStart = true } \t\t\tcontinue \t\t} `, @@ -132,9 +136,10 @@ function lexer(ir: ParserIR): string { const rx = ir.regexCtx; const tpl = ir.tpl; const nl = ir.newlineCfg; - const stateful = !!(rx || tpl || nl); - const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); - const pushPunct = stateful ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`; + const rxOrTpl = !!(rx || tpl); + const newlineOnly = !!(nl && !rx && !tpl); + const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); + const pushPunct = rxOrTpl ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`; const puncts = ir.puncts.map((p) => `\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`).join('\n'); const goMap = (a: string[]) => `map[string]bool{${a.map((x) => `${J(x)}: true`).join(', ')}}`; @@ -181,7 +186,7 @@ function lexer(ir: ParserIR): string { nl ? newlinePartsGo(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true` : ''; - const emitFn = stateful ? `\temit := func(kind, text string, off, end int) { + const emitFn = rxOrTpl ? `\temit := func(kind, text string, off, end int) { ${emitHooks} \t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false${emitTail} \t} @@ -199,28 +204,35 @@ ${emitHooks} \t\t\tpos = e; continue \t\t} ` : ''; - const nlState = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').state : ''; - const nlBoundary = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').boundary : ''; - const nlWs = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) + const nlState = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').state : ''; + const nlStateFrom = nl ? newlinePartsGo(nl, 'pushTok').stateFrom : ''; + const nlBoundary = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').boundary : ''; + const nlWs = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) \t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR \t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } `; - const pushHooks = nl && !stateful ? newlinePartsGo(nl, 'pushTok').hooks : ''; - const pushTokFn = stateful ? '' : nl + const pushHooks = nl && !rxOrTpl ? newlinePartsGo(nl, 'pushTok').hooks : ''; + const pushTokFn = rxOrTpl ? '' : nl ? `\tpushTok := func(kind, text string, off, end int) { ${pushHooks}\t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false \t} \t_ = pushTok ` : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`; - const pushTokAccFn = `\tpushTok := func(kind, text string, off, end int) { *acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false } + const pushTokAccFn = nl && !rxOrTpl + ? `\tpushTok := func(kind, text string, off, end int) { +${pushHooks}\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false +\t} +\t_ = pushTok +` + : `\tpushTok := func(kind, text string, off, end int) { *acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false } \t_ = pushTok `; const loopBody = `${nlBoundary}\t\tc := int(src[pos]) ${nlWs}${tplDispatch}${toks} ${puncts} \t\tpanic(fmt.Sprintf("lex error at %d", pos))`; - if (stateful) { + if (rxOrTpl) { return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok { \ttoks := toks[:0] \tn := len(src) @@ -231,6 +243,21 @@ ${rxState}${tplState}${nlState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\ ${loopBody} \t} \treturn toks +}`; + } + if (newlineOnly) { + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, lineStart bool, emittedContent bool, flowDepth int, acc *[]Tok, limit int) (int, bool, bool, bool, int) { +\tn := len(src) +${nlStateFrom}${pushTokAccFn}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +\tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { +${loopBody} +\t} +\treturn pos, pendingNl, lineStart, emittedContent, flowDepth +} +func lex(src string) []Tok { +\tvar out []Tok +\tlexFrom(src, 0, false, true, false, 0, &out, 0) +\treturn out }`; } return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, acc *[]Tok, limit int) (int, bool) { @@ -393,8 +420,69 @@ ${r.nudSeqs.map((seq) => `\t{ save := pos; sb := len(scratch); nb := len(nodes); } function docEditBlockGo(ir: ParserIR): string { - const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); - const windowHelpers = stateless ? ` + const windowLex = !(ir.regexCtx || ir.tpl); + const hasNewline = !!ir.newlineCfg; + const windowHelpers = windowLex ? (hasNewline ? ` +func findTokAtOffKind(toks []alignMeta, off int, kind string) int { +\tlo, hi := 0, len(toks)-1 +\thit := -1 +\tfor lo <= hi { +\t\tmid := (lo + hi) >> 1 +\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { hit = mid; break } +\t} +\tif hit < 0 { return -1 } +\tstart := hit +\tfor start > 0 && toks[start-1].Off == off { start-- } +\tfor i := start; i < len(toks) && toks[i].Off == off; i++ { +\t\tif toks[i].Kind == kind { return i } +\t} +\treturn -1 +} +func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { +\tdelta := len(ins) - (end - start) +\teditEnd := start + len(ins) +\tmaxIdx := -1 +\tfor i := 0; i < len(oldToks); i++ { +\t\tif oldToks[i].End < start { maxIdx = i } else { break } +\t} +\trb := -1 +\tif maxIdx >= 0 { rb = maxIdx - 1 } +\tvar out []alignMeta +\tif rb >= 0 { out = append(out, oldToks[:rb+1]...) } +\tvar scanOff int +\tvar pendingNl, lineStart, emittedContent bool +\tvar flowDepth int +\tif rb >= 0 { +\t\tscanOff = oldToks[rb].End; pendingNl = false; lineStart = false; emittedContent = true; flowDepth = oldToks[rb].Fd +\t} else { +\t\tscanOff = 0; pendingNl = false; lineStart = true; emittedContent = false; flowDepth = 0 +\t} +\tvar scratch []Tok +\trelexed := 0 +\tfor scanOff < len(newText) { +\t\tbefore := len(scratch) +\t\tscanOff, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, &scratch, 1) +\t\tif len(scratch) == before { break } +\t\tt := scratch[len(scratch)-1] +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth}) +\t\trelexed++ +\t\tif t.Off >= editEnd { +\t\t\toIdx := findTokAtOffKind(oldToks, t.Off-delta, t.Kind) +\t\t\tif oIdx >= 0 { +\t\t\t\to := oldToks[oIdx] +\t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && o.Fd == flowDepth && oldText[o.Off:o.End] == newText[t.Off:t.End] { +\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { +\t\t\t\t\t\tot := oldToks[j] +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd}) +\t\t\t\t\t} +\t\t\t\t\treturn out, relexed +\t\t\t\t} +\t\t\t} +\t\t} +\t} +\treturn out, relexed +} +` : ` func findTokAtOff(toks []alignMeta, off int) int { \tlo, hi := 0, len(toks)-1 \tfor lo <= hi { @@ -424,7 +512,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\tscanOff, pendingNl = lexFrom(newText, scanOff, pendingNl, &scratch, 1) \t\tif len(scratch) == before { break } \t\tt := scratch[len(scratch)-1] -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) @@ -433,7 +521,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { \t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { \t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl}) +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, 0}) \t\t\t\t\t} \t\t\t\t\treturn out, relexed \t\t\t\t} @@ -442,8 +530,8 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t} \treturn out, relexed } -` : ''; - const editBody = stateless +`) : ''; + const editBody = windowLex ? `\tcurText := d.text \tcurToks := d.toks \tfor _, e := range edits { @@ -466,8 +554,52 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \tnewToks := tokenize(d.text) \td.toks = toMeta(newToks) \trelexed = len(d.toks)`; + const toMetaFn = hasNewline ? ` +func scanMeta(src string) []alignMeta { +\tvar toks []Tok +\tvar meta []alignMeta +\tpos, pendingNl, lineStart, emittedContent, flowDepth := 0, false, true, false, 0 +\tfor pos < len(src) { +\t\tbefore := len(toks) +\t\tpos, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, &toks, 1) +\t\tif len(toks) == before { break } +\t\tt := toks[len(toks)-1] +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth}) +\t} +\treturn meta +} +func toMeta(toks []Tok) []alignMeta { panic("use scanMeta for newline") } +` : `func toMeta(toks []Tok) []alignMeta { +\tm := make([]alignMeta, len(toks)) +\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl, 0} } +\treturn m +}`; + const checkStreamEqFn = hasNewline ? ` +func checkStreamEq(text string, meta []alignMeta) bool { +\tfresh := scanMeta(text) +\tif len(fresh) != len(meta) { return false } +\tfor i := range fresh { +\t\tf, m := fresh[i], meta[i] +\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl || f.Fd != m.Fd { return false } +\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t} +\treturn true +} +` : ` +func checkStreamEq(text string, meta []alignMeta) bool { +\tfresh := toMeta(tokenize(text)) +\tif len(fresh) != len(meta) { return false } +\tfor i := range fresh { +\t\tf, m := fresh[i], meta[i] +\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl { return false } +\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t} +\treturn true +} +`; + const initToks = hasNewline ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `type Edit struct { Start, End int; Text string } -type alignMeta struct { Kind string; Off, End int; Nl bool } +type alignMeta struct { Kind string; Off, End int; Nl bool; Fd int } type Align struct { \tOldN int \`json:"oldN"\` \tNewN int \`json:"newN"\` @@ -476,11 +608,7 @@ type Align struct { \tRelexed int \`json:"relexed"\` \tStreamEq bool \`json:"streamEq"\` } -func toMeta(toks []Tok) []alignMeta { -\tm := make([]alignMeta, len(toks)) -\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl} } -\treturn m -} +${toMetaFn} func computeAlignCore(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) (oldN, newN, prefix, suffix int) { \toldN, newN = len(oldToks), len(newToks) \tfor prefix < oldN && prefix < newN { @@ -504,20 +632,10 @@ func toksFromMeta(text string, meta []alignMeta) []Tok { \tfor i, m := range meta { t[i] = Tok{m.Kind, text[m.Off:m.End], m.Off, m.End, m.Nl} } \treturn t } -func checkStreamEq(text string, meta []alignMeta) bool { -\tfresh := toMeta(tokenize(text)) -\tif len(fresh) != len(meta) { return false } -\tfor i := range fresh { -\t\tf, m := fresh[i], meta[i] -\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl { return false } -\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } -\t} -\treturn true -} -${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align } +${checkStreamEqFn}${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align } func NewDoc(src string) *Doc { \td := &Doc{text: src} -\td.toks = toMeta(tokenize(src)) +\td.toks = ${initToks} \td.root = parse(tokenize(src)) \treturn d } diff --git a/src/target-rust.ts b/src/target-rust.ts index 34df964..ebb0547 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -86,10 +86,13 @@ function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, t return ` if ${gate}true { let e = ${m}(src, pos as i64); if e > pos as i64 { let e = e as usize; ${push('e')}pos = e; continue; } }`; } -function newlinePartsRs(nl: NewlineCfg): { consts: string; fields: string; init: string; boundary: string; ws: string; hooks: string } { +function newlinePartsRs(nl: NewlineCfg): { consts: string; fields: string; init: string; boundary: string; ws: string; hooks: string; boundaryFrom: string; wsFrom: string; hooksFrom: string } { const commentSkip = nl.comment ? ` if src[p..].starts_with(${J(nl.comment)}) { let mut e = p; while e < n && b[e] != 10 { e += 1; } pos = e; continue; }\n` : ''; + const commentSkipFrom = nl.comment + ? ` if src[p..].starts_with(${J(nl.comment)}) { let mut e = p; while e < n && b[e] != 10 { e += 1; } pos = e; continue; }\n` + : ''; return { consts: `const _NLTOK: &str = ${J(nl.token)}; const _FLOW_OPEN: &[&str] = ${`&[${nl.flowOpen.map(J).join(', ')}]`}; @@ -119,17 +122,51 @@ ${commentSkip} pos = p; st.line_start = false; continue; } +`, + boundaryFrom: ` if st.flow_depth == 0 && st.line_start { + let mut p = pos; + while p < n && b[p] == 32 { p += 1; } + if p >= n { pos = p; st.line_start = false; continue; } + let ch = b[p] as u32; + if ch == 10 || ch == 13 { + pos = p + 1; if ch == 13 && pos < n && b[pos] == 10 { pos += 1; } continue; + } + if ch == 9 { + let mut bb = p; + while bb < n && (b[bb] == 32 || b[bb] == 9) { bb += 1; } + if bb >= n { pos = bb; continue; } + let bc = b[bb] as u32; + if bc == 10 || bc == 13 { + pos = bb + 1; if bc == 13 && pos < n && b[pos] == 10 { pos += 1; } continue; + } + } +${commentSkipFrom} pos = p; + if st.emitted_content { st.push_tok(_NLTOK, &src[pos..pos], pos, pos); } + st.line_start = false; + continue; + } `, ws: ` if c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos += 1; continue; } if c == 10 || c == 13 { pos += 1; if c == 13 && pos < n && b[pos] == 10 { pos += 1; } - if st.flow_depth == 0 { st.line_start = true; } else { st.pending_nl = true; } + if st.flow_depth == 0 { st.line_start = true; } + continue; + } +`, + wsFrom: ` if c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos += 1; continue; } + if c == 10 || c == 13 { + pos += 1; if c == 13 && pos < n && b[pos] == 10 { pos += 1; } + if st.flow_depth == 0 { st.line_start = true; } continue; } `, hooks: ` if kind != _NLTOK { self.emitted_content = true; } if kind == "" && _in(_FLOW_OPEN, text) { self.flow_depth += 1; } else if kind == "" && _in(_FLOW_CLOSE, text) { self.flow_depth = (self.flow_depth - 1).max(0); } +`, + hooksFrom: ` if kind != _NLTOK { emitted_content = true; } + if kind == "" && _in(_FLOW_OPEN, text) { flow_depth += 1; } + else if kind == "" && _in(_FLOW_CLOSE, text) { flow_depth = (flow_depth - 1).max(0); } `, }; } @@ -140,10 +177,11 @@ function lexer(ir: ParserIR): string { const tpl = ir.tpl; const nl = ir.newlineCfg; const nlRs = nl ? newlinePartsRs(nl) : null; - const stateful = !!(rx || tpl || nl); - const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); + const rxOrTpl = !!(rx || tpl); + const newlineOnly = !!(nl && !rx && !tpl); + const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); const puncts = ir.puncts.map((p) => - ` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n'); + ` if src[pos..].starts_with(${J(p)}) { ${rxOrTpl ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n'); const rsArr = (a: string[]) => `&[${a.map(J).join(', ')}]`; // Struct fields / emit hooks / init are assembled per-feature so a grammar can have regex, // templates, or both share one LexState. @@ -189,7 +227,7 @@ ${nlRs.consts}` : ''); ].filter(Boolean).join('\n'); const emitTail = rx ? ` self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true;` : ''; - const stateImpl = stateful ? `struct LexState<'a> { ${fields} } + const stateImpl = rxOrTpl ? `struct LexState<'a> { ${fields} } impl<'a> LexState<'a> { ${prevIsValue} fn emit(&mut self, kind: &'static str, text: &'a str, off: usize, end: usize) { ${emitHooks} @@ -201,8 +239,8 @@ ${emitHooks} rx ? 'prev_text: "", prev_kind: "", bp_text: "", has_prev: false, has_prev2: false, paren_head: Vec::new(), last_close: false, last_bang: false' : '', tpl ? 'template_stack: Vec::new()' : '', nlRs ? nlRs.init : ''].filter(Boolean).join(', '); - const open = stateful ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec = Vec::new();\n let mut pending_nl = false;`; - const nlVar = stateful ? 'st.pending_nl' : 'pending_nl'; + const open = rxOrTpl ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec = Vec::new();\n let mut pending_nl = false;`; + const nlVar = rxOrTpl ? 'st.pending_nl' : 'pending_nl'; const tplDispatch = tpl ? ` if !st.template_stack.is_empty() && src[pos..].starts_with(${J(tpl.interpClose)}) && *st.template_stack.last().unwrap() == 0 { st.template_stack.pop(); let (interp, e) = _scan_tpl_span(src, pos + ${tpl.interpClose.length}); @@ -224,7 +262,7 @@ ${emitHooks} ${nlWs}${tplDispatch}${toks} ${puncts} panic!("lex error at {}", pos);`; - if (stateful) { + if (rxOrTpl) { return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec> { let b = src.as_bytes(); let n = b.len(); @@ -233,7 +271,38 @@ ${open} while pos < n { ${loopBody} } - ${stateful ? 'st.toks' : 'toks'} + st.toks +}`; + } + if (newlineOnly) { + const rustNlScan = (s: string) => s + .replace(/toks\.push\(Tok \{ kind: ([^,]+), text: ([^,]+), off: pos, end: ([^,]+), nl: pending_nl \}\); pending_nl = false; ?/g, 'st.push_tok($1, $2, pos, $3); ') + .replace(/pending_nl/g, 'st.pending_nl'); + const nlLoopBody = `${nlRs!.boundaryFrom} let c = b[pos] as u32; +${nlRs!.wsFrom}${rustNlScan(toks)} +${rustNlScan(puncts)} + panic!("lex error at {}", pos);`; + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}struct NlScan<'a, 'b> { acc: &'a mut Vec>, pending_nl: bool, line_start: bool, emitted_content: bool, flow_depth: i64 } +impl<'a, 'b> NlScan<'a, 'b> { + fn push_tok(&mut self, kind: &'static str, text: &'b str, off: usize, end: usize) { +${nlRs!.hooksFrom.replace(/emitted_content/g, 'self.emitted_content').replace(/flow_depth/g, 'self.flow_depth').replace(/pending_nl/g, 'self.pending_nl')} + self.acc.push(Tok { kind, text, off, end, nl: self.pending_nl }); self.pending_nl = false; + } +} +fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, mut line_start: bool, mut emitted_content: bool, mut flow_depth: i64, acc: &mut Vec>, limit: usize) -> (usize, bool, bool, bool, i64) { + let b = src.as_bytes(); + let n = b.len(); + let base = acc.len(); + let mut st = NlScan { acc, pending_nl, line_start, emitted_content, flow_depth }; + while pos < n && (limit == 0 || st.acc.len() - base < limit) { +${nlLoopBody} + } + (pos, st.pending_nl, st.line_start, st.emitted_content, st.flow_depth) +} +fn lex<'a>(src: &'a str) -> Vec> { + let mut toks = Vec::new(); + lex_from(src, 0, false, true, false, 0, &mut toks, 0); + toks }`; } return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, acc: &mut Vec>, limit: usize) -> (usize, bool) { @@ -416,8 +485,66 @@ ${r.nudSeqs.map((seq) => ` { let save = self.pos; let sb = self.scratch.l } function docEditBlockRust(ir: ParserIR): string { - const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); - const windowHelpers = stateless ? ` + const windowLex = !(ir.regexCtx || ir.tpl); + const hasNewline = !!ir.newlineCfg; + const windowHelpers = windowLex ? (hasNewline ? ` +fn find_tok_at_off_kind(toks: &[AlignMeta], off: usize, kind: &'static str) -> Option { + let mut lo = 0usize; + let mut hi = toks.len(); + let mut hit = None; + while lo < hi { + let mid = (lo + hi) / 2; + if toks[mid].off < off { lo = mid + 1; } else { hi = mid; } + } + if lo < toks.len() && toks[lo].off == off { hit = Some(lo); } + let hit = hit?; + let mut start = hit; + while start > 0 && toks[start - 1].off == off { start -= 1; } + let mut i = start; + while i < toks.len() && toks[i].off == off { + if toks[i].kind == kind { return Some(i); } + i += 1; + } + None +} +fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, start: usize, end: usize, ins: &str) -> (Vec, usize) { + let delta = ins.len() as isize - (end - start) as isize; + let edit_end = start + ins.len(); + let mut max_idx = None::; + for (i, t) in old_toks.iter().enumerate() { + if t.end < start { max_idx = Some(i); } else { break; } + } + let rb = max_idx.map(|i| i as isize - 1).unwrap_or(-1); + let mut out: Vec = if rb >= 0 { old_toks[..=rb as usize].to_vec() } else { Vec::new() }; + let (mut scan_off, mut pending_nl, mut line_start, mut emitted_content, mut flow_depth) = if rb >= 0 { + (old_toks[rb as usize].end, false, false, true, old_toks[rb as usize].fd) + } else { + (0, false, true, false, 0) + }; + let mut scratch: Vec> = Vec::new(); + let mut relexed = 0usize; + while scan_off < new_text.len() { + let before = scratch.len(); + (scan_off, pending_nl, line_start, emitted_content, flow_depth) = lex_from(new_text, scan_off, pending_nl, line_start, emitted_content, flow_depth, &mut scratch, 1); + if scratch.len() == before { break; } + let t = &scratch[scratch.len() - 1]; + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth }); + relexed += 1; + if t.off >= edit_end { + if let Some(o_idx) = find_tok_at_off_kind(old_toks, (t.off as isize - delta) as usize, t.kind) { + let o = &old_toks[o_idx]; + if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && o.fd == flow_depth && old_text[o.off..o.end] == new_text[t.off..t.end] { + for ot in &old_toks[o_idx + 1..] { + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd }); + } + return (out, relexed); + } + } + } + } + (out, relexed) +} +` : ` fn find_tok_at_off(toks: &[AlignMeta], off: usize) -> Option { let mut lo = 0usize; let mut hi = toks.len(); @@ -445,14 +572,14 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta (scan_off, pending_nl) = lex_from(new_text, scan_off, pending_nl, &mut scratch, 1); if scratch.len() == before { break; } let t = &scratch[scratch.len() - 1]; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { let o = &old_toks[o_idx]; if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: 0 }); } return (out, relexed); } @@ -461,27 +588,8 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta } (out, relexed) } -fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { - let fresh = to_meta(&lex(text)); - if fresh.len() != meta.len() { return false; } - for (f, m) in fresh.iter().zip(meta.iter()) { - if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl { return false; } - if text[f.off..f.end] != text[m.off..m.end] { return false; } - } - true -} -` : ` -fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { - let fresh = to_meta(&lex(text)); - if fresh.len() != meta.len() { return false; } - for (f, m) in fresh.iter().zip(meta.iter()) { - if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl { return false; } - if text[f.off..f.end] != text[m.off..m.end] { return false; } - } - true -} -`; - const editBody = stateless +`) : ''; + const editBody = windowLex ? ` let mut cur_text = self.text.clone(); let mut cur_toks = self.toks.clone(); for e in edits { @@ -506,13 +614,51 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { } self.toks = to_meta(&lex(&self.text)); relexed = self.toks.len();`; + const toMetaFn = hasNewline ? ` +fn scan_meta(src: &str) -> Vec { + let mut toks: Vec> = Vec::new(); + let mut meta: Vec = Vec::new(); + let (mut pos, mut pending_nl, mut line_start, mut emitted_content, mut flow_depth) = (0usize, false, true, false, 0i64); + while pos < src.len() { + let before = toks.len(); + (pos, pending_nl, line_start, emitted_content, flow_depth) = lex_from(src, pos, pending_nl, line_start, emitted_content, flow_depth, &mut toks, 1); + if toks.len() == before { break; } + let t = &toks[toks.len() - 1]; + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth }); + } + meta +} +fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for newline") } +` : `fn to_meta(toks: &[Tok<'_>]) -> Vec { + toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }).collect() +}`; + const checkStreamEqFn = hasNewline ? ` +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = scan_meta(text); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl || f.fd != m.fd { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} +` : ` +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = to_meta(&lex(text)); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} +`; + const initToks = hasNewline ? 'scan_meta(&text)' : 'to_meta(&lex(&text))'; return `pub struct Edit { pub start: usize, pub end: usize, pub text: String } #[derive(Clone)] -struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool } +struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64 } struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool } -fn to_meta(toks: &[Tok<'_>]) -> Vec { - toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }).collect() -} +${toMetaFn} fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) { let old_n = old_toks.len(); let new_n = new_toks.len(); @@ -540,9 +686,9 @@ fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, ne fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec> { meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect() } -${windowHelpers}pub struct Doc { text: String, toks: Vec, align: Option } +${checkStreamEqFn}${windowHelpers}pub struct Doc { text: String, toks: Vec, align: Option } impl Doc { - pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: to_meta(&lex(&text)), align: None } } + pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: ${initToks}, align: None } } pub fn text(&self) -> &str { &self.text } pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() } pub fn edit(&mut self, edits: &[Edit]) { diff --git a/src/target-ts.ts b/src/target-ts.ts index 08af15a..6c351f0 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -81,7 +81,7 @@ function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, t return ` if (${gate}true) { const e = ${m}(pos); if (e > pos) { ${push('e')}pos = e; continue; } }`; } -function newlineParts(nl: NewlineCfg, pushFn: string): { state: string; boundary: string; ws: string; hooks: string } { +function newlineParts(nl: NewlineCfg, pushFn: string): { state: string; stateFrom: string; boundary: string; ws: string; hooks: string } { const commentSkip = nl.comment ? ` if (src.startsWith(${J(nl.comment)}, p)) { let e = p; while (e < n && src.charCodeAt(e) !== 10) e++; pos = e; continue; }\n` : ''; @@ -90,6 +90,10 @@ function newlineParts(nl: NewlineCfg, pushFn: string): { state: string; boundary const _flowOpen = new Set([${nl.flowOpen.map(J).join(', ')}]); const _flowClose = new Set([${nl.flowClose.map(J).join(', ')}]); const _nlTok = ${J(nl.token)}; +`, + stateFrom: ` const _flowOpen = new Set([${nl.flowOpen.map(J).join(', ')}]); + const _flowClose = new Set([${nl.flowClose.map(J).join(', ')}]); + const _nlTok = ${J(nl.token)}; `, boundary: ` if (flowDepth === 0 && lineStart) { let p = pos; @@ -120,7 +124,6 @@ ${commentSkip} pos = p; if (c === 10 || c === 13) { // LF/CR only — LS/PS fall through to the unexpected-character throw, matching the interpreter pos++; if (c === 13 && pos < n && src.charCodeAt(pos) === 10) pos++; if (flowDepth === 0) lineStart = true; - else pendingNl = true; continue; } `, @@ -136,9 +139,10 @@ function lexer(ir: ParserIR): string { const rx = ir.regexCtx; const tpl = ir.tpl; const nl = ir.newlineCfg; - const stateful = !!(rx || tpl || nl); - const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); - const pushFn = stateful ? 'emit' : 'push'; + const rxOrTpl = !!(rx || tpl); + const newlineOnly = !!(nl && !rx && !tpl); + const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); + const pushFn = rxOrTpl ? 'emit' : 'push'; const puncts = ir.puncts.map((p) => ` if (src.startsWith(${J(p)}, pos)) { ${pushFn}('', ${J(p)}, pos, pos + ${p.length}); pos += ${p.length}; continue; }`).join('\n'); const set = (a: string[]) => `new Set([${a.map(J).join(', ')}])`; @@ -176,7 +180,7 @@ function lexer(ir: ParserIR): string { nl ? newlineParts(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true;` : ''; - const emitFn = stateful ? ` function emit(kind: string, text: string, off: number, end: number): void { + const emitFn = rxOrTpl ? ` function emit(kind: string, text: string, off: number, end: number): void { ${emitHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false;${emitTail} } @@ -196,13 +200,14 @@ ${emitHooks} pos = sp.end; continue; } ` : ''; - const nlState = nl ? newlineParts(nl, stateful ? 'emit' : 'push').state : ''; - const nlBoundary = nl ? newlineParts(nl, stateful ? 'emit' : 'push').boundary : ''; - const nlWs = nl ? newlineParts(nl, stateful ? 'emit' : 'push').ws : ` if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } + const nlState = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').state : ''; + const nlStateFrom = nl ? newlineParts(nl, 'push').stateFrom : ''; + const nlBoundary = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').boundary : ''; + const nlWs = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').ws : ` if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; } `; - const pushHooks = nl && !stateful ? newlineParts(nl, 'push').hooks : ''; - const pushFnDef = stateful ? '' : nl + const pushHooks = nl && !rxOrTpl ? newlineParts(nl, 'push').hooks : ''; + const pushFnDef = rxOrTpl ? '' : nl ? ` const push = (kind: string, text: string, off: number, end: number) => { ${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; }; @@ -213,7 +218,7 @@ ${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = ${nlWs}${tplDispatch}${toks} ${puncts} throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos]));`; - if (stateful) { + if (rxOrTpl) { return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lex(src: string): Tok[] { const toks: Tok[] = []; const n = src.length; @@ -223,6 +228,21 @@ ${defs.length ? ' _s = src;\n' : ''}${rxState}${tplState}${nlState}${emitFn} w ${loopBody} } return toks; +}`; + } + if (newlineOnly) { + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, lineStart: boolean, emittedContent: boolean, flowDepth: number, toks: Tok[], limit?: number): { pos: number; pendingNl: boolean; lineStart: boolean; emittedContent: boolean; flowDepth: number } { + const n = src.length; + const base = toks.length; +${defs.length ? ' _s = src;\n' : ''}${nlStateFrom}${pushFnDef} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${loopBody} + } + return { pos, pendingNl, lineStart, emittedContent, flowDepth }; +} +function lex(src: string): Tok[] { + const toks: Tok[] = []; + lexFrom(src, 0, false, true, false, 0, toks); + return toks; }`; } return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, toks: Tok[], limit?: number): { pos: number; pendingNl: boolean } { @@ -370,8 +390,67 @@ ${r.nudSeqs.map((seq) => ` { const save = pos; const kids: Cst[] = []; if (${se } function docEditBlock(ir: ParserIR): string { - const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); - const windowHelpers = stateless ? ` + const windowLex = !(ir.regexCtx || ir.tpl); + const hasNewline = !!ir.newlineCfg; + const windowHelpers = windowLex ? (hasNewline ? ` +function findTokAtOffKind(toks: AlignMeta[], off: number, kind: string): number { + let lo = 0, hi = toks.length - 1, hit = -1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (toks[mid].off < off) lo = mid + 1; + else if (toks[mid].off > off) hi = mid - 1; + else { hit = mid; break; } + } + if (hit < 0) return -1; + let start = hit; + while (start > 0 && toks[start - 1].off === off) start--; + for (let i = start; i < toks.length && toks[i].off === off; i++) { + if (toks[i].kind === kind) return i; + } + return -1; +} +function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { + const delta = ins.length - (end - start); + const editEnd = start + ins.length; + let maxIdx = -1; + for (let i = 0; i < oldToks.length; i++) { + if (oldToks[i].end < start) maxIdx = i; + else break; + } + const rb = maxIdx >= 0 ? maxIdx - 1 : -1; + const out: AlignMeta[] = rb >= 0 ? oldToks.slice(0, rb + 1) : []; + let scanOff: number, pendingNl: boolean, lineStart: boolean, emittedContent: boolean, flowDepth: number; + if (rb >= 0) { + scanOff = oldToks[rb].end; pendingNl = false; lineStart = false; emittedContent = true; flowDepth = oldToks[rb].fd; + } else { + scanOff = 0; pendingNl = false; lineStart = true; emittedContent = false; flowDepth = 0; + } + const scratch: Tok[] = []; + let relexed = 0; + while (scanOff < newText.length) { + const before = scratch.length; + ({ pos: scanOff, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, scratch, 1)); + if (scratch.length === before) break; + const t = scratch[scratch.length - 1]; + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth }); + relexed++; + if (t.off >= editEnd) { + const oIdx = findTokAtOffKind(oldToks, t.off - delta, t.kind); + if (oIdx >= 0) { + const o = oldToks[oIdx]; + if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && o.fd === flowDepth && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { + for (let j = oIdx + 1; j < oldToks.length; j++) { + const ot = oldToks[j]; + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd }); + } + return { toks: out, relexed }; + } + } + } + } + return { toks: out, relexed }; +} +` : ` function findTokAtOff(toks: AlignMeta[], off: number): number { let lo = 0, hi = toks.length - 1; while (lo <= hi) { @@ -401,7 +480,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, ({ pos: scanOff, pendingNl } = lexFrom(newText, scanOff, pendingNl, scratch, 1)); if (scratch.length === before) break; const t = scratch[scratch.length - 1]; - out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl }); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }); relexed++; if (t.off >= editEnd) { const oIdx = findTokAtOff(oldToks, t.off - delta); @@ -410,7 +489,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { for (let j = oIdx + 1; j < oldToks.length; j++) { const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl }); + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: 0 }); } return { toks: out, relexed }; } @@ -419,8 +498,8 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, } return { toks: out, relexed }; } -` : ''; - const editBody = stateless +`) : ''; + const editBody = windowLex ? ` let curText = text; let curToks = oldToks; for (const e of edits) { @@ -441,10 +520,50 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, } prevToks = toMeta(tokenize(text)); relexed = prevToks.length;`; + const toMetaFn = hasNewline ? ` +function scanMeta(src: string): AlignMeta[] { + const toks: Tok[] = []; + const meta: AlignMeta[] = []; + let pos = 0, pendingNl = false, lineStart = true, emittedContent = false, flowDepth = 0; + while (pos < src.length) { + const before = toks.length; + ({ pos, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, toks, 1)); + if (toks.length === before) break; + const t = toks[toks.length - 1]; + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth }); + } + return meta; +} +const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for newline'); }; +` : `const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }));`; + const checkStreamEqFn = hasNewline ? ` +function checkStreamEq(text: string, meta: AlignMeta[]): boolean { + const fresh = scanMeta(text); + if (fresh.length !== meta.length) return false; + for (let i = 0; i < fresh.length; i++) { + const f = fresh[i], t = meta[i]; + if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl || f.fd !== t.fd) return false; + if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; + } + return true; +} +` : ` +function checkStreamEq(text: string, meta: AlignMeta[]): boolean { + const fresh = toMeta(tokenize(text)); + if (fresh.length !== meta.length) return false; + for (let i = 0; i < fresh.length; i++) { + const f = fresh[i], t = meta[i]; + if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl) return false; + if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; + } + return true; +} +`; + const initToks = hasNewline ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `export type Edit = { start: number; end: number; text: string }; -type AlignMeta = { kind: string; off: number; end: number; nl: boolean }; +type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number }; type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean }; -const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl })); +${toMetaFn} function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit { const oldN = oldToks.length, newN = newToks.length; let prefix = 0; @@ -468,19 +587,9 @@ function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, ne function toksFromMeta(text: string, meta: AlignMeta[]): Tok[] { return meta.map((m) => ({ kind: m.kind, text: text.slice(m.off, m.end), off: m.off, end: m.end, nl: m.nl })); } -function checkStreamEq(text: string, meta: AlignMeta[]): boolean { - const fresh = toMeta(tokenize(text)); - if (fresh.length !== meta.length) return false; - for (let i = 0; i < fresh.length; i++) { - const f = fresh[i], t = meta[i]; - if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl) return false; - if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; - } - return true; -} -${windowHelpers}export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } { +${checkStreamEqFn}${windowHelpers}export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } { let text = src; - let prevToks = toMeta(tokenize(src)); + let prevToks = ${initToks}; let align: Align | null = null; let root: Node | null = parse(tokenize(src)); return { diff --git a/test/portable-targets.ts b/test/portable-targets.ts index 7e594a1..765c1b8 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -327,6 +327,7 @@ const expectAlign = (g: CstGrammar, init: string, batches: EditBatch[]): Align = type EditBatch = [number, number, string][]; type EditScenario = { init: string; batches: EditBatch[]; large?: boolean; maxRelexed?: number; fullRelex?: boolean }; const calcLargeInit = '1+2*3+'.repeat(199) + '1+2*3'; +const envspecLargeInit = 'A=1\n'.repeat(300); const EDIT_SCENARIOS: Record = { calc: [ { init: '1+2*3', batches: [[[3, 3, '4']]] }, @@ -346,7 +347,14 @@ const EDIT_SCENARIOS: Record = { { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true, fullRelex: true }, ], envspec: [ - { init: 'A=1\nB=2', batches: [[[2, 2, '3']]], fullRelex: true }, + { init: 'A=1\nB=2', batches: [[[2, 2, '3']]] }, + { init: envspecLargeInit, batches: [[[600, 601, '9']]], large: true, maxRelexed: 10 }, + { init: 'A=fn(1,\n2)\nB=3', batches: [[[6, 7, '9']]] }, + { init: 'A=fn(1,\n2)\nB=3', batches: [[[4, 4, '(']]] }, + { init: 'A=1\n# note\nB=2', batches: [[[7, 7, 'x']]] }, + { init: 'A=1\n\n\nB=2', batches: [[[4, 4, '\n']]] }, + { init: envspecLargeInit, batches: [[[envspecLargeInit.length, envspecLargeInit.length, 'Z=9']]], large: true, maxRelexed: 10 }, + { init: 'A=fn(1,\n2)\nB=3', batches: [[[0, 0, 'X']], [[7, 7, '0']]] }, ], }; const applyEdits = (init: string, batches: EditBatch[]): string => {