From d34e35ac107d049b06059db1a3d256c2fa86c816 Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sun, 12 Jul 2026 06:29:15 +0800 Subject: [PATCH] Extend windowed relex to rx-only portable lexers (issue #57 S5a). Co-authored-by: Cursor --- src/target-go.ts | 211 +++++++++++++++++++++++++++++++++++---- src/target-rust.ts | 197 +++++++++++++++++++++++++++++++++--- src/target-ts.ts | 182 +++++++++++++++++++++++++++++---- test/portable-targets.ts | 9 ++ 4 files changed, 544 insertions(+), 55 deletions(-) diff --git a/src/target-go.ts b/src/target-go.ts index 04846e3..b78cf9c 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -136,10 +136,12 @@ function lexer(ir: ParserIR): string { const rx = ir.regexCtx; const tpl = ir.tpl; const nl = ir.newlineCfg; - const rxOrTpl = !!(rx || tpl); + const rxOnly = !!(rx && !tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly; + const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); - const pushPunct = rxOrTpl ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`; + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); + const pushPunct = stateful ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`; const puncts = ir.puncts.map((p) => `\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`).join('\n'); const goMap = (a: string[]) => `map[string]bool{${a.map((x) => `${J(x)}: true`).join(', ')}}`; @@ -186,11 +188,38 @@ function lexer(ir: ParserIR): string { nl ? newlinePartsGo(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true` : ''; - const emitFn = rxOrTpl ? `\temit := func(kind, text string, off, end int) { + const emitFn = stateful ? `\temit := func(kind, text string, off, end int) { ${emitHooks} \t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false${emitTail} \t} \t_ = emit +` : ''; + const rxStateFrom = rx ? `\t_divT := ${goMap(rx.divisionTexts)} +\t_divK := ${goMap(rx.divisionTypes)} +\t_rxT := ${goMap(rx.regexTexts)} +\t_phK := ${goMap(rx.parenHeadKw)} +\t_mem := ${goMap(rx.memberAccess)} +\t_pav := ${goMap(rx.postfixAfterValue)} +\tconst IDENT = ${J(rx.identToken)} +\tprevIsValue := func() bool { +\t\tif !hasPrev { return false } +\t\tif _pav[prevText] { return lastBang } +\t\tisExprKw := prevKind == IDENT && _rxT[prevText] +\t\tisParenHead := prevText == ")" && lastClose +\t\treturn !isExprKw && !isParenHead && (_divK[prevKind] || _divT[prevText]) +\t} +\temit := func(kind, text string, off, end int) { +\t\tif text == "(" { +\t\t\tisMember := hasPrev2 && _mem[bpText] +\t\t\tparenHead = append(parenHead, !isMember && prevKind == IDENT && _phK[prevText]) +\t\t} else if text == ")" { +\t\t\tif len(parenHead) > 0 { lastClose = parenHead[len(parenHead)-1]; parenHead = parenHead[:len(parenHead)-1] } else { lastClose = false } +\t\t} +\t\tif _pav[text] { lastBang = prevIsValue() } +\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false +\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true +\t} +\t_ = emit ` : ''; const tplDispatch = tpl ? `\t\tif len(templateStack) > 0 && strings.HasPrefix(src[pos:], ${J(tpl.interpClose)}) && templateStack[len(templateStack)-1] == 0 { \t\t\ttemplateStack = templateStack[:len(templateStack)-1] @@ -204,22 +233,22 @@ ${emitHooks} \t\t\tpos = e; continue \t\t} ` : ''; - const nlState = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').state : ''; + const nlState = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').state : ''; const nlStateFrom = nl ? newlinePartsGo(nl, 'pushTok').stateFrom : ''; - const nlBoundary = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').boundary : ''; - const nlWs = nl ? newlinePartsGo(nl, rxOrTpl ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) + const nlBoundary = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').boundary : ''; + const nlWs = nl ? newlinePartsGo(nl, stateful ? 'emit' : 'pushTok').ws : `\t\tif strings.HasPrefix(src[pos:], ${J('\u2028')}) || strings.HasPrefix(src[pos:], ${J('\u2029')}) { pendingNl = true; pos += 3; continue } // LS/PS (UTF-8) \t\tif c == 10 || c == 13 { pendingNl = true; pos++; continue } // LF/CR \t\tif c == 32 || c == 9 || c == 11 || c == 12 || c == 160 || c == 5760 || (c >= 8192 && c <= 8202) || c == 8239 || c == 8287 || c == 12288 || c == 65279 { pos++; continue } `; - const pushHooks = nl && !rxOrTpl ? newlinePartsGo(nl, 'pushTok').hooks : ''; - const pushTokFn = rxOrTpl ? '' : nl + const pushHooks = nl && !stateful ? newlinePartsGo(nl, 'pushTok').hooks : ''; + const pushTokFn = stateful ? '' : nl ? `\tpushTok := func(kind, text string, off, end int) { ${pushHooks}\t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false \t} \t_ = pushTok ` : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`; - const pushTokAccFn = nl && !rxOrTpl + const pushTokAccFn = nl && !stateful ? `\tpushTok := func(kind, text string, off, end int) { ${pushHooks}\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false \t} @@ -232,6 +261,21 @@ ${pushHooks}\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendi ${nlWs}${tplDispatch}${toks} ${puncts} \t\tpanic(fmt.Sprintf("lex error at %d", pos))`; + if (rxOnly) { + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, prevText, prevKind, bpText string, hasPrev, hasPrev2 bool, parenHead []bool, lastClose, lastBang bool, acc *[]Tok, limit int) (int, bool, string, string, string, bool, bool, []bool, bool, bool) { +\tn := len(src) +${rxStateFrom}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +\tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { +${loopBody} +\t} +\treturn pos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang +} +func lex(src string) []Tok { +\tvar out []Tok +\tlexFrom(src, 0, false, "", "", "", false, false, nil, false, false, &out, 0) +\treturn out +}`; + } if (rxOrTpl) { return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok { \ttoks := toks[:0] @@ -420,8 +464,10 @@ ${r.nudSeqs.map((seq) => `\t{ save := pos; sb := len(scratch); nb := len(nodes); } function docEditBlockGo(ir: ParserIR): string { - const windowLex = !(ir.regexCtx || ir.tpl); - const hasNewline = !!ir.newlineCfg; + const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); + const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); + const zeroMeta = ', Pd: 0, Lc: false, Lb: false, Hd: false'; const windowHelpers = windowLex ? (hasNewline ? ` func findTokAtOffKind(toks []alignMeta, off int, kind string) int { \tlo, hi := 0, len(toks)-1 @@ -464,7 +510,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\tscanOff, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, &scratch, 1) \t\tif len(scratch) == before { break } \t\tt := scratch[len(scratch)-1] -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOffKind(oldToks, t.Off-delta, t.Kind) @@ -473,7 +519,99 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && o.Fd == flowDepth && oldText[o.Off:o.End] == newText[t.Off:t.End] { \t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { \t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd}) +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) +\t\t\t\t\t} +\t\t\t\t\treturn out, relexed +\t\t\t\t} +\t\t\t} +\t\t} +\t} +\treturn out, relexed +} +` : rxOnly ? ` +func findTokAtOff(toks []alignMeta, off int) int { +\tlo, hi := 0, len(toks)-1 +\tfor lo <= hi { +\t\tmid := (lo + hi) >> 1 +\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { return mid } +\t} +\treturn -1 +} +func reconstructParens(toks []alignMeta, text string, b int) []bool { +\tneed := 0 +\tif b >= 0 { need = toks[b].Pd } +\tout := make([]bool, 0, need) +\tfor i := b; i >= 0 && need > 0; i-- { +\t\tt := toks[i] +\t\tif text[t.Off:t.End] == "(" && t.Pd == need { +\t\t\tout = append([]bool{t.Hd}, out...) +\t\t\tneed-- +\t\t} +\t} +\treturn out +} +func parenStacksEq(a, b []bool) bool { +\tif len(a) != len(b) { return false } +\tfor i := range a { if a[i] != b[i] { return false } } +\treturn true +} +func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { +\tdelta := len(ins) - (end - start) +\teditEnd := start + len(ins) +\tmaxIdx := -1 +\tfor i := 0; i < len(oldToks); i++ { +\t\tif oldToks[i].End < start { maxIdx = i } else { break } +\t} +\trb := -1 +\tif maxIdx >= 0 { rb = maxIdx - 1 } +\tvar out []alignMeta +\tif rb >= 0 { out = append(out, oldToks[:rb+1]...) } +\tvar scanOff int +\tpendingNl := false +\tprevText, prevKind, bpText := "", "", "" +\thasPrev, hasPrev2 := false, false +\tvar parenHead []bool +\tlastClose, lastBang := false, false +\tif rb >= 0 { +\t\tanchor := oldToks[rb] +\t\tscanOff = anchor.End +\t\tprevText = oldText[anchor.Off:anchor.End] +\t\tprevKind = anchor.Kind +\t\thasPrev = true +\t\tif rb >= 1 { +\t\t\tbpText = oldText[oldToks[rb-1].Off:oldToks[rb-1].End] +\t\t\thasPrev2 = true +\t\t} +\t\tlastClose = anchor.Lc +\t\tlastBang = anchor.Lb +\t\tparenHead = reconstructParens(oldToks, oldText, rb) +\t} +\tvar scratch []Tok +\trelexed := 0 +\tfor scanOff < len(newText) { +\t\tbefore := len(scratch) +\t\tscanOff, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang = lexFrom(newText, scanOff, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, &scratch, 1) +\t\tif len(scratch) == before { break } +\t\tt := scratch[len(scratch)-1] +\t\ttxt := newText[t.Off:t.End] +\t\thd := false +\t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd}) +\t\trelexed++ +\t\tif t.Off >= editEnd { +\t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) +\t\t\tif oIdx >= 0 { +\t\t\t\to := oldToks[oIdx] +\t\t\t\tnewPrevText := "" +\t\t\t\tif len(out) > 1 { p := out[len(out)-2]; newPrevText = newText[p.Off:p.End] } +\t\t\t\toldPrevText := "" +\t\t\t\tif oIdx >= 1 { p := oldToks[oIdx-1]; oldPrevText = oldText[p.Off:p.End] } +\t\t\t\tbpOk := newPrevText == oldPrevText +\t\t\t\toldStack := reconstructParens(oldToks, oldText, oIdx) +\t\t\t\tif o.Pd == len(parenHead) && parenStacksEq(oldStack, parenHead) && o.Lc == lastClose && o.Lb == lastBang && bpOk && o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { +\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { +\t\t\t\t\t\tot := oldToks[j] +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) \t\t\t\t\t} \t\t\t\t\treturn out, relexed \t\t\t\t} @@ -512,7 +650,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\tscanOff, pendingNl = lexFrom(newText, scanOff, pendingNl, &scratch, 1) \t\tif len(scratch) == before { break } \t\tt := scratch[len(scratch)-1] -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) @@ -521,7 +659,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { \t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { \t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, 0}) +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) \t\t\t\t\t} \t\t\t\t\treturn out, relexed \t\t\t\t} @@ -564,14 +702,36 @@ func scanMeta(src string) []alignMeta { \t\tpos, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, &toks, 1) \t\tif len(toks) == before { break } \t\tt := toks[len(toks)-1] -\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth}) +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false}) \t} \treturn meta } func toMeta(toks []Tok) []alignMeta { panic("use scanMeta for newline") } +` : rxOnly ? ` +func scanMeta(src string) []alignMeta { +\tvar toks []Tok +\tvar meta []alignMeta +\tpos, pendingNl := 0, false +\tprevText, prevKind, bpText := "", "", "" +\thasPrev, hasPrev2 := false, false +\tvar parenHead []bool +\tlastClose, lastBang := false, false +\tfor pos < len(src) { +\t\tbefore := len(toks) +\t\tpos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang = lexFrom(src, pos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, &toks, 1) +\t\tif len(toks) == before { break } +\t\tt := toks[len(toks)-1] +\t\ttxt := src[t.Off:t.End] +\t\thd := false +\t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd}) +\t} +\treturn meta +} +func toMeta(toks []Tok) []alignMeta { panic("use scanMeta for regex") } ` : `func toMeta(toks []Tok) []alignMeta { \tm := make([]alignMeta, len(toks)) -\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl, 0} } +\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false} } \treturn m }`; const checkStreamEqFn = hasNewline ? ` @@ -585,6 +745,17 @@ func checkStreamEq(text string, meta []alignMeta) bool { \t} \treturn true } +` : rxOnly ? ` +func checkStreamEq(text string, meta []alignMeta) bool { +\tfresh := scanMeta(text) +\tif len(fresh) != len(meta) { return false } +\tfor i := range fresh { +\t\tf, m := fresh[i], meta[i] +\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl || f.Pd != m.Pd || f.Lc != m.Lc || f.Lb != m.Lb || f.Hd != m.Hd { return false } +\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t} +\treturn true +} ` : ` func checkStreamEq(text string, meta []alignMeta) bool { \tfresh := toMeta(tokenize(text)) @@ -597,9 +768,9 @@ func checkStreamEq(text string, meta []alignMeta) bool { \treturn true } `; - const initToks = hasNewline ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; + const initToks = (hasNewline || rxOnly) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `type Edit struct { Start, End int; Text string } -type alignMeta struct { Kind string; Off, End int; Nl bool; Fd int } +type alignMeta struct { Kind string; Off, End int; Nl bool; Fd, Pd int; Lc, Lb, Hd bool } type Align struct { \tOldN int \`json:"oldN"\` \tNewN int \`json:"newN"\` diff --git a/src/target-rust.ts b/src/target-rust.ts index ebb0547..952a56c 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -177,11 +177,13 @@ function lexer(ir: ParserIR): string { const tpl = ir.tpl; const nl = ir.newlineCfg; const nlRs = nl ? newlinePartsRs(nl) : null; - const rxOrTpl = !!(rx || tpl); + const rxOnly = !!(rx && !tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly; + const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); const puncts = ir.puncts.map((p) => - ` if src[pos..].starts_with(${J(p)}) { ${rxOrTpl ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n'); + ` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n'); const rsArr = (a: string[]) => `&[${a.map(J).join(', ')}]`; // Struct fields / emit hooks / init are assembled per-feature so a grammar can have regex, // templates, or both share one LexState. @@ -227,20 +229,38 @@ ${nlRs.consts}` : ''); ].filter(Boolean).join('\n'); const emitTail = rx ? ` self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true;` : ''; - const stateImpl = rxOrTpl ? `struct LexState<'a> { ${fields} } + const stateImpl = stateful ? `struct LexState<'a> { ${fields} } impl<'a> LexState<'a> { ${prevIsValue} fn emit(&mut self, kind: &'static str, text: &'a str, off: usize, end: usize) { ${emitHooks} self.toks.push(Tok { kind, text, off, end, nl: self.pending_nl }); self.pending_nl = false;${emitTail} } } +` : ''; + const rxScanImpl = rxOnly ? `struct RxScan<'a, 'b> { acc: &'a mut Vec>, pending_nl: bool, prev_text: &'b str, prev_kind: &'static str, bp_text: &'b str, has_prev: bool, has_prev2: bool, paren_head: Vec, last_close: bool, last_bang: bool } +impl<'a, 'b> RxScan<'a, 'b> { + fn prev_is_value(&self) -> bool { + if !self.has_prev { return false; } + if _in(_PAV, self.prev_text) { return self.last_bang; } + let is_expr_kw = self.prev_kind == _IDENT && _in(_RXT, self.prev_text); + let is_paren_head = self.prev_text == ")" && self.last_close; + !is_expr_kw && !is_paren_head && (_in(_DIVK, self.prev_kind) || _in(_DIVT, self.prev_text)) + } + fn emit(&mut self, kind: &'static str, text: &'b str, off: usize, end: usize) { + if text == "(" { let is_member = self.has_prev2 && _in(_MEM, self.bp_text); self.paren_head.push(!is_member && self.prev_kind == _IDENT && _in(_PHK, self.prev_text)); } + else if text == ")" { self.last_close = self.paren_head.pop().unwrap_or(false); } + if _in(_PAV, text) { self.last_bang = self.prev_is_value(); } + self.acc.push(Tok { kind, text, off, end, nl: self.pending_nl }); self.pending_nl = false; + self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true; + } +} ` : ''; const initFields = ['toks: Vec::new()', 'pending_nl: false', rx ? 'prev_text: "", prev_kind: "", bp_text: "", has_prev: false, has_prev2: false, paren_head: Vec::new(), last_close: false, last_bang: false' : '', tpl ? 'template_stack: Vec::new()' : '', nlRs ? nlRs.init : ''].filter(Boolean).join(', '); - const open = rxOrTpl ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec = Vec::new();\n let mut pending_nl = false;`; - const nlVar = rxOrTpl ? 'st.pending_nl' : 'pending_nl'; + const open = stateful ? ` let mut st = LexState { ${initFields} };` : ` let mut toks: Vec = Vec::new();\n let mut pending_nl = false;`; + const nlVar = stateful ? 'st.pending_nl' : 'pending_nl'; const tplDispatch = tpl ? ` if !st.template_stack.is_empty() && src[pos..].starts_with(${J(tpl.interpClose)}) && *st.template_stack.last().unwrap() == 0 { st.template_stack.pop(); let (interp, e) = _scan_tpl_span(src, pos + ${tpl.interpClose.length}); @@ -262,6 +282,27 @@ ${emitHooks} ${nlWs}${tplDispatch}${toks} ${puncts} panic!("lex error at {}", pos);`; + if (rxOnly) { + const rxLoopBody = `${nlBoundary} let c = b[pos] as u32; +${nlWs}${toks} +${puncts} + panic!("lex error at {}", pos);`; + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${rxScanImpl}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, mut prev_text: &'a str, mut prev_kind: &'static str, mut bp_text: &'a str, mut has_prev: bool, mut has_prev2: bool, mut paren_head: Vec, mut last_close: bool, mut last_bang: bool, acc: &mut Vec>, limit: usize) -> (usize, bool, &'a str, &'static str, &'a str, bool, bool, Vec, bool, bool) { + let b = src.as_bytes(); + let n = b.len(); + let base = acc.len(); + let mut st = RxScan { acc, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang }; + while pos < n && (limit == 0 || st.acc.len() - base < limit) { +${rxLoopBody} + } + (pos, st.pending_nl, st.prev_text, st.prev_kind, st.bp_text, st.has_prev, st.has_prev2, st.paren_head, st.last_close, st.last_bang) +} +fn lex<'a>(src: &'a str) -> Vec> { + let mut toks = Vec::new(); + lex_from(src, 0, false, "", "", "", false, false, Vec::new(), false, false, &mut toks, 0); + toks +}`; + } if (rxOrTpl) { return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec> { let b = src.as_bytes(); @@ -485,8 +526,9 @@ ${r.nudSeqs.map((seq) => ` { let save = self.pos; let sb = self.scratch.l } function docEditBlockRust(ir: ParserIR): string { - const windowLex = !(ir.regexCtx || ir.tpl); - const hasNewline = !!ir.newlineCfg; + const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); + const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); const windowHelpers = windowLex ? (hasNewline ? ` fn find_tok_at_off_kind(toks: &[AlignMeta], off: usize, kind: &'static str) -> Option { let mut lo = 0usize; @@ -528,14 +570,106 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta (scan_off, pending_nl, line_start, emitted_content, flow_depth) = lex_from(new_text, scan_off, pending_nl, line_start, emitted_content, flow_depth, &mut scratch, 1); if scratch.len() == before { break; } let t = &scratch[scratch.len() - 1]; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off_kind(old_toks, (t.off as isize - delta) as usize, t.kind) { let o = &old_toks[o_idx]; if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && o.fd == flow_depth && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); + } + return (out, relexed); + } + } + } + } + (out, relexed) +} +` : rxOnly ? ` +fn find_tok_at_off(toks: &[AlignMeta], off: usize) -> Option { + let mut lo = 0usize; + let mut hi = toks.len(); + while lo < hi { + let mid = (lo + hi) / 2; + if toks[mid].off < off { lo = mid + 1; } else { hi = mid; } + } + if lo < toks.len() && toks[lo].off == off { Some(lo) } else { None } +} +fn reconstruct_parens(toks: &[AlignMeta], text: &str, b: isize) -> Vec { + let mut need = if b >= 0 { toks[b as usize].pd } else { 0 }; + let mut out: Vec = Vec::new(); + let mut i = b; + while i >= 0 && need > 0 { + let t = &toks[i as usize]; + if &text[t.off..t.end] == "(" && t.pd == need { + out.insert(0, t.hd); + need -= 1; + } + i -= 1; + } + out +} +fn paren_stacks_eq(a: &[bool], b: &[bool]) -> bool { + a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x == y) +} +fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, start: usize, end: usize, ins: &str) -> (Vec, usize) { + let delta = ins.len() as isize - (end - start) as isize; + let edit_end = start + ins.len(); + let mut max_idx = None::; + for (i, t) in old_toks.iter().enumerate() { + if t.end < start { max_idx = Some(i); } else { break; } + } + let rb = max_idx.map(|i| i as isize - 1).unwrap_or(-1); + let mut out: Vec = if rb >= 0 { old_toks[..=rb as usize].to_vec() } else { Vec::new() }; + let mut scan_off: usize; + let mut pending_nl = false; + let mut prev_text: &str = ""; + let mut prev_kind: &'static str = ""; + let mut bp_text: &str = ""; + let mut has_prev = false; + let mut has_prev2 = false; + let mut paren_head: Vec = Vec::new(); + let mut last_close = false; + let mut last_bang = false; + if rb >= 0 { + let anchor = &old_toks[rb as usize]; + scan_off = anchor.end; + prev_text = &old_text[anchor.off..anchor.end]; + prev_kind = anchor.kind; + has_prev = true; + if rb >= 1 { + let p = &old_toks[rb as usize - 1]; + bp_text = &old_text[p.off..p.end]; + has_prev2 = true; + } + last_close = anchor.lc; + last_bang = anchor.lb; + paren_head = reconstruct_parens(old_toks, old_text, rb); + } else { + scan_off = 0; + } + let mut scratch: Vec> = Vec::new(); + let mut relexed = 0usize; + while scan_off < new_text.len() { + let before = scratch.len(); + (scan_off, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang) = lex_from(new_text, scan_off, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang, &mut scratch, 1); + if scratch.len() == before { break; } + let t = &scratch[scratch.len() - 1]; + let txt = &new_text[t.off..t.end]; + let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd }); + relexed += 1; + if t.off >= edit_end { + if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { + let o = &old_toks[o_idx]; + let new_prev_text = if out.len() > 1 { let p = &out[out.len() - 2]; &new_text[p.off..p.end] } else { "" }; + let old_prev_text = if o_idx >= 1 { let p = &old_toks[o_idx - 1]; &old_text[p.off..p.end] } else { "" }; + let bp_ok = new_prev_text == old_prev_text; + let old_stack = reconstruct_parens(old_toks, old_text, o_idx as isize); + if o.pd == paren_head.len() as i64 && paren_stacks_eq(&old_stack, &paren_head) && o.lc == last_close && o.lb == last_bang && bp_ok && o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { + for ot in &old_toks[o_idx + 1..] { + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); } return (out, relexed); } @@ -572,14 +706,14 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta (scan_off, pending_nl) = lex_from(new_text, scan_off, pending_nl, &mut scratch, 1); if scratch.len() == before { break; } let t = &scratch[scratch.len() - 1]; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { let o = &old_toks[o_idx]; if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: 0 }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); } return (out, relexed); } @@ -624,13 +758,34 @@ fn scan_meta(src: &str) -> Vec { (pos, pending_nl, line_start, emitted_content, flow_depth) = lex_from(src, pos, pending_nl, line_start, emitted_content, flow_depth, &mut toks, 1); if toks.len() == before { break; } let t = &toks[toks.len() - 1]; - meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth }); + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false }); } meta } fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for newline") } +` : rxOnly ? ` +fn scan_meta(src: &str) -> Vec { + let mut toks: Vec> = Vec::new(); + let mut meta: Vec = Vec::new(); + let (mut pos, mut pending_nl) = (0usize, false); + let (mut prev_text, mut prev_kind, mut bp_text) = ("", "", ""); + let (mut has_prev, mut has_prev2) = (false, false); + let mut paren_head: Vec = Vec::new(); + let (mut last_close, mut last_bang) = (false, false); + while pos < src.len() { + let before = toks.len(); + (pos, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang) = lex_from(src, pos, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang, &mut toks, 1); + if toks.len() == before { break; } + let t = &toks[toks.len() - 1]; + let txt = &src[t.off..t.end]; + let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd }); + } + meta +} +fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for regex") } ` : `fn to_meta(toks: &[Tok<'_>]) -> Vec { - toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }).collect() + toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false }).collect() }`; const checkStreamEqFn = hasNewline ? ` fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { @@ -642,6 +797,16 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { } true } +` : rxOnly ? ` +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = scan_meta(text); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl || f.pd != m.pd || f.lc != m.lc || f.lb != m.lb || f.hd != m.hd { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} ` : ` fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { let fresh = to_meta(&lex(text)); @@ -653,10 +818,10 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { true } `; - const initToks = hasNewline ? 'scan_meta(&text)' : 'to_meta(&lex(&text))'; + const initToks = (hasNewline || rxOnly) ? 'scan_meta(&text)' : 'to_meta(&lex(&text))'; return `pub struct Edit { pub start: usize, pub end: usize, pub text: String } #[derive(Clone)] -struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64 } +struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool } struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool } ${toMetaFn} fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) { diff --git a/src/target-ts.ts b/src/target-ts.ts index 6c351f0..8c7ee1d 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -139,10 +139,12 @@ function lexer(ir: ParserIR): string { const rx = ir.regexCtx; const tpl = ir.tpl; const nl = ir.newlineCfg; - const rxOrTpl = !!(rx || tpl); + const rxOnly = !!(rx && !tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly; + const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); - const toks = ir.tokens.map((t) => scanTok(t, defs, rxOrTpl, rx?.regexToken, tpl?.token)).join('\n'); - const pushFn = rxOrTpl ? 'emit' : 'push'; + const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); + const pushFn = stateful ? 'emit' : 'push'; const puncts = ir.puncts.map((p) => ` if (src.startsWith(${J(p)}, pos)) { ${pushFn}('', ${J(p)}, pos, pos + ${p.length}); pos += ${p.length}; continue; }`).join('\n'); const set = (a: string[]) => `new Set([${a.map(J).join(', ')}])`; @@ -180,10 +182,28 @@ function lexer(ir: ParserIR): string { nl ? newlineParts(nl, 'emit').hooks : '', ].filter(Boolean).join('\n'); const emitTail = rx ? `\n bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true;` : ''; - const emitFn = rxOrTpl ? ` function emit(kind: string, text: string, off: number, end: number): void { + const emitFn = stateful ? ` function emit(kind: string, text: string, off: number, end: number): void { ${emitHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false;${emitTail} } +` : ''; + const rxStateFrom = rx ? ` const _divT = ${set(rx.divisionTexts)}, _divK = ${set(rx.divisionTypes)}, _rxT = ${set(rx.regexTexts)}; + const _phK = ${set(rx.parenHeadKw)}, _mem = ${set(rx.memberAccess)}, _pav = ${set(rx.postfixAfterValue)}; + const IDENT = ${J(rx.identToken)}; + function prevIsValue(): boolean { + if (!hasPrev) return false; + if (_pav.has(prevText)) return lastBang; + const isExprKw = prevKind === IDENT && _rxT.has(prevText); + const isParenHead = prevText === ')' && lastClose; + return !isExprKw && !isParenHead && (_divK.has(prevKind) || _divT.has(prevText)); + } + function emit(kind: string, text: string, off: number, end: number): void { + if (text === '(') { const isMember = hasPrev2 && _mem.has(bpText); parenHead.push(!isMember && prevKind === IDENT && _phK.has(prevText)); } + else if (text === ')') { lastClose = parenHead.pop() ?? false; } + if (_pav.has(text)) lastBang = prevIsValue(); + toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; + bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true; + } ` : ''; // Template dispatch runs at the top of the loop, before token/punct scanning. const tplDispatch = tpl ? ` if (templateStack.length > 0 && src.startsWith(${J(tpl.interpClose)}, pos) && templateStack[templateStack.length - 1] === 0) { @@ -200,14 +220,14 @@ ${emitHooks} pos = sp.end; continue; } ` : ''; - const nlState = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').state : ''; + const nlState = nl ? newlineParts(nl, stateful ? 'emit' : 'push').state : ''; const nlStateFrom = nl ? newlineParts(nl, 'push').stateFrom : ''; - const nlBoundary = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').boundary : ''; - const nlWs = nl ? newlineParts(nl, rxOrTpl ? 'emit' : 'push').ws : ` if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } + const nlBoundary = nl ? newlineParts(nl, stateful ? 'emit' : 'push').boundary : ''; + const nlWs = nl ? newlineParts(nl, stateful ? 'emit' : 'push').ws : ` if (c === 10 || c === 13 || c === 8232 || c === 8233) { pendingNl = true; pos++; continue; } if (c === 32 || c === 9 || c === 11 || c === 12 || c === 160 || c === 5760 || (c >= 8192 && c <= 8202) || c === 8239 || c === 8287 || c === 12288 || c === 65279) { pos++; continue; } `; - const pushHooks = nl && !rxOrTpl ? newlineParts(nl, 'push').hooks : ''; - const pushFnDef = rxOrTpl ? '' : nl + const pushHooks = nl && !stateful ? newlineParts(nl, 'push').hooks : ''; + const pushFnDef = stateful ? '' : nl ? ` const push = (kind: string, text: string, off: number, end: number) => { ${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; }; @@ -218,6 +238,21 @@ ${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = ${nlWs}${tplDispatch}${toks} ${puncts} throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos]));`; + if (rxOnly) { + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, prevText: string, prevKind: string, hasPrev: boolean, bpText: string, hasPrev2: boolean, parenHead: boolean[], lastClose: boolean, lastBang: boolean, toks: Tok[], limit?: number): { pos: number; pendingNl: boolean; prevText: string; prevKind: string; hasPrev: boolean; bpText: string; hasPrev2: boolean; parenHead: boolean[]; lastClose: boolean; lastBang: boolean } { + const n = src.length; + const base = toks.length; +${defs.length ? ' _s = src;\n' : ''}${rxStateFrom} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${loopBody} + } + return { pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang }; +} +function lex(src: string): Tok[] { + const toks: Tok[] = []; + lexFrom(src, 0, false, '', '', false, '', false, [], false, false, toks); + return toks; +}`; + } if (rxOrTpl) { return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lex(src: string): Tok[] { const toks: Tok[] = []; @@ -390,8 +425,10 @@ ${r.nudSeqs.map((seq) => ` { const save = pos; const kids: Cst[] = []; if (${se } function docEditBlock(ir: ParserIR): string { - const windowLex = !(ir.regexCtx || ir.tpl); - const hasNewline = !!ir.newlineCfg; + const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); + const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); + const zeroMeta = ', fd: 0, pd: 0, lc: false, lb: false, hd: false'; const windowHelpers = windowLex ? (hasNewline ? ` function findTokAtOffKind(toks: AlignMeta[], off: number, kind: string): number { let lo = 0, hi = toks.length - 1, hit = -1; @@ -432,7 +469,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, ({ pos: scanOff, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, scratch, 1)); if (scratch.length === before) break; const t = scratch[scratch.length - 1]; - out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth }); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false }); relexed++; if (t.off >= editEnd) { const oIdx = findTokAtOffKind(oldToks, t.off - delta, t.kind); @@ -441,7 +478,85 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && o.fd === flowDepth && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { for (let j = oIdx + 1; j < oldToks.length; j++) { const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd }); + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); + } + return { toks: out, relexed }; + } + } + } + } + return { toks: out, relexed }; +} +` : rxOnly ? ` +function findTokAtOff(toks: AlignMeta[], off: number): number { + let lo = 0, hi = toks.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (toks[mid].off < off) lo = mid + 1; + else if (toks[mid].off > off) hi = mid - 1; + else return mid; + } + return -1; +} +function reconstructParens(toks: AlignMeta[], text: string, b: number): boolean[] { + let need = b >= 0 ? toks[b].pd : 0; + const out: boolean[] = []; + for (let i = b; i >= 0 && need > 0; i--) { + const t = toks[i]; + if (text.slice(t.off, t.end) === '(' && t.pd === need) { out[need - 1] = t.hd; need--; } + } + return out; +} +function parenStacksEq(a: boolean[], b: boolean[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} +function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { + const delta = ins.length - (end - start); + const editEnd = start + ins.length; + let maxIdx = -1; + for (let i = 0; i < oldToks.length; i++) { + if (oldToks[i].end < start) maxIdx = i; + else break; + } + const rb = maxIdx >= 0 ? maxIdx - 1 : -1; + const out: AlignMeta[] = rb >= 0 ? oldToks.slice(0, rb + 1) : []; + let scanOff: number, pendingNl: boolean; + let prevText = '', prevKind = '', bpText = '', hasPrev = false, hasPrev2 = false; + let parenHead: boolean[] = [], lastClose = false, lastBang = false; + if (rb >= 0) { + const anchor = oldToks[rb]; + scanOff = anchor.end; pendingNl = false; + prevText = oldText.slice(anchor.off, anchor.end); prevKind = anchor.kind; hasPrev = true; + if (rb >= 1) { bpText = oldText.slice(oldToks[rb - 1].off, oldToks[rb - 1].end); hasPrev2 = true; } + lastClose = anchor.lc; lastBang = anchor.lb; + parenHead = reconstructParens(oldToks, oldText, rb); + } else { + scanOff = 0; pendingNl = false; + } + const scratch: Tok[] = []; + let relexed = 0; + while (scanOff < newText.length) { + const before = scratch.length; + ({ pos: scanOff, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang } = lexFrom(newText, scanOff, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, scratch, 1)); + if (scratch.length === before) break; + const t = scratch[scratch.length - 1]; + const txt = newText.slice(t.off, t.end); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false }); + relexed++; + if (t.off >= editEnd) { + const oIdx = findTokAtOff(oldToks, t.off - delta); + if (oIdx >= 0) { + const o = oldToks[oIdx]; + const newPrevText = out.length > 1 ? newText.slice(out[out.length - 2].off, out[out.length - 2].end) : ''; + const oldPrevText = oIdx >= 1 ? oldText.slice(oldToks[oIdx - 1].off, oldToks[oIdx - 1].end) : ''; + const bpOk = newPrevText === oldPrevText; + const oldStack = reconstructParens(oldToks, oldText, oIdx); + if (o.pd === parenHead.length && parenStacksEq(oldStack, parenHead) && o.lc === lastClose && o.lb === lastBang && bpOk && o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { + for (let j = oIdx + 1; j < oldToks.length; j++) { + const ot = oldToks[j]; + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); } return { toks: out, relexed }; } @@ -480,7 +595,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, ({ pos: scanOff, pendingNl } = lexFrom(newText, scanOff, pendingNl, scratch, 1)); if (scratch.length === before) break; const t = scratch[scratch.length - 1]; - out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl${zeroMeta} }); relexed++; if (t.off >= editEnd) { const oIdx = findTokAtOff(oldToks, t.off - delta); @@ -489,7 +604,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { for (let j = oIdx + 1; j < oldToks.length; j++) { const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: 0 }); + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); } return { toks: out, relexed }; } @@ -530,12 +645,30 @@ function scanMeta(src: string): AlignMeta[] { ({ pos, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, toks, 1)); if (toks.length === before) break; const t = toks[toks.length - 1]; - meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth }); + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false }); } return meta; } const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for newline'); }; -` : `const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0 }));`; +` : rxOnly ? ` +function scanMeta(src: string): AlignMeta[] { + const toks: Tok[] = []; + const meta: AlignMeta[] = []; + let pos = 0, pendingNl = false; + let prevText = '', prevKind = '', bpText = '', hasPrev = false, hasPrev2 = false; + let parenHead: boolean[] = [], lastClose = false, lastBang = false; + while (pos < src.length) { + const before = toks.length; + ({ pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang } = lexFrom(src, pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, toks, 1)); + if (toks.length === before) break; + const t = toks[toks.length - 1]; + const txt = src.slice(t.off, t.end); + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false }); + } + return meta; +} +const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for regex'); }; +` : `const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl${zeroMeta} }));`; const checkStreamEqFn = hasNewline ? ` function checkStreamEq(text: string, meta: AlignMeta[]): boolean { const fresh = scanMeta(text); @@ -547,6 +680,17 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean { } return true; } +` : rxOnly ? ` +function checkStreamEq(text: string, meta: AlignMeta[]): boolean { + const fresh = scanMeta(text); + if (fresh.length !== meta.length) return false; + for (let i = 0; i < fresh.length; i++) { + const f = fresh[i], t = meta[i]; + if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl || f.pd !== t.pd || f.lc !== t.lc || f.lb !== t.lb || f.hd !== t.hd) return false; + if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; + } + return true; +} ` : ` function checkStreamEq(text: string, meta: AlignMeta[]): boolean { const fresh = toMeta(tokenize(text)); @@ -559,9 +703,9 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean { return true; } `; - const initToks = hasNewline ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; + const initToks = (hasNewline || rxOnly) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `export type Edit = { start: number; end: number; text: string }; -type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number }; +type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean }; type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean }; ${toMetaFn} function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit { diff --git a/test/portable-targets.ts b/test/portable-targets.ts index 765c1b8..7ddd29e 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -328,6 +328,7 @@ type EditBatch = [number, number, string][]; type EditScenario = { init: string; batches: EditBatch[]; large?: boolean; maxRelexed?: number; fullRelex?: boolean }; const calcLargeInit = '1+2*3+'.repeat(199) + '1+2*3'; const envspecLargeInit = 'A=1\n'.repeat(300); +const regexjsLargeInit = 'var r = /abc/g; a / b;\n'.repeat(60); const EDIT_SCENARIOS: Record = { calc: [ { init: '1+2*3', batches: [[[3, 3, '4']]] }, @@ -356,6 +357,14 @@ const EDIT_SCENARIOS: Record = { { init: envspecLargeInit, batches: [[[envspecLargeInit.length, envspecLargeInit.length, 'Z=9']]], large: true, maxRelexed: 10 }, { init: 'A=fn(1,\n2)\nB=3', batches: [[[0, 0, 'X']], [[7, 7, '0']]] }, ], + regexjs: [ + { init: 'var re = /[a-z]+/i; x / y;', batches: [[[11, 12, '0']]] }, + { init: 'x = a / b / c;', batches: [[[4, 6, '']]] }, + { init: 'if (x) /y/;', batches: [[[0, 2, 'f']]] }, + { init: regexjsLargeInit, batches: [[[600, 601, '9']]], large: true, maxRelexed: 15 }, + { init: 'f(x); a / b;', batches: [[[2, 2, '(']]] }, + { init: 'if (x) /y/;', batches: [[[11, 11, ' /z/;']]] }, + ], }; const applyEdits = (init: string, batches: EditBatch[]): string => { let text = init;