diff --git a/src/target-go.ts b/src/target-go.ts index b78cf9c..7ce513a 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -137,7 +137,9 @@ function lexer(ir: ParserIR): string { const tpl = ir.tpl; const nl = ir.newlineCfg; const rxOnly = !!(rx && !tpl && !nl); - const rxOrTpl = !!(rx || tpl) && !rxOnly; + const tplOnly = !!(tpl && !rx && !nl); + const rxTpl = !!(rx && tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly && !tplOnly && !rxTpl; const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); @@ -208,7 +210,19 @@ ${emitHooks} \t\tisParenHead := prevText == ")" && lastClose \t\treturn !isExprKw && !isParenHead && (_divK[prevKind] || _divT[prevText]) \t} -\temit := func(kind, text string, off, end int) { +` : ''; + const tplStateFrom = tpl ? `\tscanTplSpan := func(p int) (bool, int) { +\t\tfor p < n { +\t\t\tif strings.HasPrefix(src[p:], ${J(tpl.interpOpen)}) { return true, p + ${tpl.interpOpen.length} } +\t\t\tif src[p] == 92 { p += 2; continue } +\t\t\tif strings.HasPrefix(src[p:], ${J(tpl.open)}) { return false, p + ${tpl.open.length} } +\t\t\tp++ +\t\t} +\t\treturn false, p +\t} +\t_ = scanTplSpan +` : ''; + const emitRxOnly = rx ? `\temit := func(kind, text string, off, end int) { \t\tif text == "(" { \t\t\tisMember := hasPrev2 && _mem[bpText] \t\t\tparenHead = append(parenHead, !isMember && prevKind == IDENT && _phK[prevText]) @@ -220,6 +234,26 @@ ${emitHooks} \t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true \t} \t_ = emit +` : ''; + const emitTplOnly = tpl ? `\temit := func(kind, text string, off, end int) { +\t\tif len(templateStack) > 0 { if text == ${J(tpl.braceOpen)} { templateStack[len(templateStack)-1]++ } else if text == ${J(tpl.interpClose)} { templateStack[len(templateStack)-1]-- } } +\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false +\t} +\t_ = emit +` : ''; + const emitRxTpl = (rx && tpl) ? `\temit := func(kind, text string, off, end int) { +\t\tif text == "(" { +\t\t\tisMember := hasPrev2 && _mem[bpText] +\t\t\tparenHead = append(parenHead, !isMember && prevKind == IDENT && _phK[prevText]) +\t\t} else if text == ")" { +\t\t\tif len(parenHead) > 0 { lastClose = parenHead[len(parenHead)-1]; parenHead = parenHead[:len(parenHead)-1] } else { lastClose = false } +\t\t} +\t\tif _pav[text] { lastBang = prevIsValue() } +\t\tif len(templateStack) > 0 { if text == ${J(tpl.braceOpen)} { templateStack[len(templateStack)-1]++ } else if text == ${J(tpl.interpClose)} { templateStack[len(templateStack)-1]-- } } +\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false +\t\tbpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true +\t} +\t_ = emit ` : ''; const tplDispatch = tpl ? `\t\tif len(templateStack) > 0 && strings.HasPrefix(src[pos:], ${J(tpl.interpClose)}) && templateStack[len(templateStack)-1] == 0 { \t\t\ttemplateStack = templateStack[:len(templateStack)-1] @@ -264,7 +298,7 @@ ${puncts} if (rxOnly) { return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, prevText, prevKind, bpText string, hasPrev, hasPrev2 bool, parenHead []bool, lastClose, lastBang bool, acc *[]Tok, limit int) (int, bool, string, string, string, bool, bool, []bool, bool, bool) { \tn := len(src) -${rxStateFrom}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +${rxStateFrom}${emitRxOnly}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) \tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { ${loopBody} \t} @@ -274,6 +308,36 @@ func lex(src string) []Tok { \tvar out []Tok \tlexFrom(src, 0, false, "", "", "", false, false, nil, false, false, &out, 0) \treturn out +}`; + } + if (tplOnly) { + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, templateStack []int, acc *[]Tok, limit int) (int, bool, []int) { +\tn := len(src) +${tplStateFrom}${emitTplOnly}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +\tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { +${loopBody} +\t} +\treturn pos, pendingNl, templateStack +} +func lex(src string) []Tok { +\tvar out []Tok +\tlexFrom(src, 0, false, nil, &out, 0) +\treturn out +}`; + } + if (rxTpl) { + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, prevText, prevKind, bpText string, hasPrev, hasPrev2 bool, parenHead []bool, lastClose, lastBang bool, templateStack []int, acc *[]Tok, limit int) (int, bool, string, string, string, bool, bool, []bool, bool, bool, []int) { +\tn := len(src) +${rxStateFrom}${tplStateFrom}${emitRxTpl}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +\tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { +${loopBody} +\t} +\treturn pos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, templateStack +} +func lex(src string) []Tok { +\tvar out []Tok +\tlexFrom(src, 0, false, "", "", "", false, false, nil, false, false, nil, &out, 0) +\treturn out }`; } if (rxOrTpl) { @@ -464,10 +528,59 @@ ${r.nudSeqs.map((seq) => `\t{ save := pos; sb := len(scratch); nb := len(nodes); } function docEditBlockGo(ir: ParserIR): string { - const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const windowLex = (!ir.regexCtx && !ir.tpl) || !ir.newlineCfg; const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); - const zeroMeta = ', Pd: 0, Lc: false, Lb: false, Hd: false'; + const tplOnly = !!(ir.tpl && !ir.regexCtx && !ir.newlineCfg); + const rxTpl = !!(ir.regexCtx && ir.tpl && !ir.newlineCfg); + const zeroMeta = ', 0, 0, false, false, false, 0'; // Fd,Pd,Lc,Lb,Hd,Td after Kind,Off,End,Nl + const adoptSuffix = `\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { +\t\t\t\t\tot := oldToks[j] +\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd, ot.Td}) +\t\t\t\t}`; + const findTokAtOff = ` +func findTokAtOff(toks []alignMeta, off int) int { +\tlo, hi := 0, len(toks)-1 +\tfor lo <= hi { +\t\tmid := (lo + hi) >> 1 +\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { return mid } +\t} +\treturn -1 +}`; + const reconstructParens = ` +func reconstructParens(toks []alignMeta, text string, b int) []bool { +\tneed := 0 +\tif b >= 0 { need = toks[b].Pd } +\tout := make([]bool, 0, need) +\tfor i := b; i >= 0 && need > 0; i-- { +\t\tt := toks[i] +\t\tif text[t.Off:t.End] == "(" && t.Pd == need { +\t\t\tout = append([]bool{t.Hd}, out...) +\t\t\tneed-- +\t\t} +\t} +\treturn out +} +func parenStacksEq(a, b []bool) bool { +\tif len(a) != len(b) { return false } +\tfor i := range a { if a[i] != b[i] { return false } } +\treturn true +}`; + const tplAnchor = `\tmaxIdx := -1 +\tfor i := 0; i < len(oldToks); i++ { +\t\tif oldToks[i].End < start { maxIdx = i } else { break } +\t} +\trb0 := -1 +\tif maxIdx >= 0 { rb0 = maxIdx - 1 } +\trb := -1 +\tif rb0 >= 0 { +\t\tfor i := rb0; i < len(oldToks); i++ { +\t\t\tif oldToks[i].End > start { break } +\t\t\tif oldToks[i].Td == 0 { rb = i; break } +\t\t} +\t} +\tvar out []alignMeta +\tif rb >= 0 { out = append(out, oldToks[:rb+1]...) }`; const windowHelpers = windowLex ? (hasNewline ? ` func findTokAtOffKind(toks []alignMeta, off int, kind string) int { \tlo, hi := 0, len(toks)-1 @@ -510,17 +623,14 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\tscanOff, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, &scratch, 1) \t\tif len(scratch) == before { break } \t\tt := scratch[len(scratch)-1] -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false, 0}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOffKind(oldToks, t.Off-delta, t.Kind) \t\t\tif oIdx >= 0 { \t\t\t\to := oldToks[oIdx] \t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && o.Fd == flowDepth && oldText[o.Off:o.End] == newText[t.Off:t.End] { -\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { -\t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) -\t\t\t\t\t} +${adoptSuffix} \t\t\t\t\treturn out, relexed \t\t\t\t} \t\t\t} @@ -528,33 +638,7 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t} \treturn out, relexed } -` : rxOnly ? ` -func findTokAtOff(toks []alignMeta, off int) int { -\tlo, hi := 0, len(toks)-1 -\tfor lo <= hi { -\t\tmid := (lo + hi) >> 1 -\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { return mid } -\t} -\treturn -1 -} -func reconstructParens(toks []alignMeta, text string, b int) []bool { -\tneed := 0 -\tif b >= 0 { need = toks[b].Pd } -\tout := make([]bool, 0, need) -\tfor i := b; i >= 0 && need > 0; i-- { -\t\tt := toks[i] -\t\tif text[t.Off:t.End] == "(" && t.Pd == need { -\t\t\tout = append([]bool{t.Hd}, out...) -\t\t\tneed-- -\t\t} -\t} -\treturn out -} -func parenStacksEq(a, b []bool) bool { -\tif len(a) != len(b) { return false } -\tfor i := range a { if a[i] != b[i] { return false } } -\treturn true -} +` : rxOnly ? `${findTokAtOff}${reconstructParens} func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { \tdelta := len(ins) - (end - start) \teditEnd := start + len(ins) @@ -596,23 +680,19 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\ttxt := newText[t.Off:t.End] \t\thd := false \t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd, 0}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) \t\t\tif oIdx >= 0 { \t\t\t\to := oldToks[oIdx] \t\t\t\tnewPrevText := "" -\t\t\t\tif len(out) > 1 { p := out[len(out)-2]; newPrevText = newText[p.Off:p.End] } +\t\t\t\tif len(out) > 1 { newPrevText = newText[out[len(out)-2].Off:out[len(out)-2].End] } \t\t\t\toldPrevText := "" -\t\t\t\tif oIdx >= 1 { p := oldToks[oIdx-1]; oldPrevText = oldText[p.Off:p.End] } -\t\t\t\tbpOk := newPrevText == oldPrevText +\t\t\t\tif oIdx >= 1 { oldPrevText = oldText[oldToks[oIdx-1].Off:oldToks[oIdx-1].End] } \t\t\t\toldStack := reconstructParens(oldToks, oldText, oIdx) -\t\t\t\tif o.Pd == len(parenHead) && parenStacksEq(oldStack, parenHead) && o.Lc == lastClose && o.Lb == lastBang && bpOk && o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { -\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { -\t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) -\t\t\t\t\t} +\t\t\t\tif o.Pd == len(parenHead) && parenStacksEq(oldStack, parenHead) && o.Lc == lastClose && o.Lb == lastBang && newPrevText == oldPrevText && o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { +${adoptSuffix} \t\t\t\t\treturn out, relexed \t\t\t\t} \t\t\t} @@ -620,15 +700,94 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t} \treturn out, relexed } -` : ` -func findTokAtOff(toks []alignMeta, off int) int { -\tlo, hi := 0, len(toks)-1 -\tfor lo <= hi { -\t\tmid := (lo + hi) >> 1 -\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { return mid } +` : rxTpl ? `${findTokAtOff}${reconstructParens} +func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { +\tdelta := len(ins) - (end - start) +\teditEnd := start + len(ins) +${tplAnchor} +\tvar scanOff int +\tpendingNl := false +\tprevText, prevKind, bpText := "", "", "" +\thasPrev, hasPrev2 := false, false +\tvar parenHead []bool +\tlastClose, lastBang := false, false +\tvar templateStack []int +\tif rb >= 0 { +\t\tanchor := oldToks[rb] +\t\tscanOff = anchor.End +\t\tprevText = oldText[anchor.Off:anchor.End] +\t\tprevKind = anchor.Kind +\t\thasPrev = true +\t\tif rb >= 1 { +\t\t\tbpText = oldText[oldToks[rb-1].Off:oldToks[rb-1].End] +\t\t\thasPrev2 = true +\t\t} +\t\tlastClose = anchor.Lc +\t\tlastBang = anchor.Lb +\t\tparenHead = reconstructParens(oldToks, oldText, rb) \t} -\treturn -1 +\tvar scratch []Tok +\trelexed := 0 +\tfor scanOff < len(newText) { +\t\tbefore := len(scratch) +\t\tscanOff, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, templateStack = lexFrom(newText, scanOff, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, templateStack, &scratch, 1) +\t\tif len(scratch) == before { break } +\t\tt := scratch[len(scratch)-1] +\t\ttxt := newText[t.Off:t.End] +\t\thd := false +\t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd, len(templateStack)}) +\t\trelexed++ +\t\tif t.Off >= editEnd { +\t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) +\t\t\tif oIdx >= 0 { +\t\t\t\to := oldToks[oIdx] +\t\t\t\tnewPrevText := "" +\t\t\t\tif len(out) > 1 { newPrevText = newText[out[len(out)-2].Off:out[len(out)-2].End] } +\t\t\t\toldPrevText := "" +\t\t\t\tif oIdx >= 1 { oldPrevText = oldText[oldToks[oIdx-1].Off:oldToks[oIdx-1].End] } +\t\t\t\toldStack := reconstructParens(oldToks, oldText, oIdx) +\t\t\t\tif o.Td == 0 && len(templateStack) == 0 && o.Pd == len(parenHead) && parenStacksEq(oldStack, parenHead) && o.Lc == lastClose && o.Lb == lastBang && newPrevText == oldPrevText && o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { +${adoptSuffix} +\t\t\t\t\treturn out, relexed +\t\t\t\t} +\t\t\t} +\t\t} +\t} +\treturn out, relexed } +` : tplOnly ? `${findTokAtOff} +func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { +\tdelta := len(ins) - (end - start) +\teditEnd := start + len(ins) +${tplAnchor} +\tscanOff := 0 +\tif rb >= 0 { scanOff = oldToks[rb].End } +\tpendingNl := false +\tvar templateStack []int +\tvar scratch []Tok +\trelexed := 0 +\tfor scanOff < len(newText) { +\t\tbefore := len(scratch) +\t\tscanOff, pendingNl, templateStack = lexFrom(newText, scanOff, pendingNl, templateStack, &scratch, 1) +\t\tif len(scratch) == before { break } +\t\tt := scratch[len(scratch)-1] +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false, len(templateStack)}) +\t\trelexed++ +\t\tif t.Off >= editEnd { +\t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) +\t\t\tif oIdx >= 0 { +\t\t\t\to := oldToks[oIdx] +\t\t\t\tif o.Td == 0 && len(templateStack) == 0 && o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { +${adoptSuffix} +\t\t\t\t\treturn out, relexed +\t\t\t\t} +\t\t\t} +\t\t} +\t} +\treturn out, relexed +} +` : `${findTokAtOff} func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { \tdelta := len(ins) - (end - start) \teditEnd := start + len(ins) @@ -650,17 +809,14 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, \t\tscanOff, pendingNl = lexFrom(newText, scanOff, pendingNl, &scratch, 1) \t\tif len(scratch) == before { break } \t\tt := scratch[len(scratch)-1] -\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false}) +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl${zeroMeta}}) \t\trelexed++ \t\tif t.Off >= editEnd { \t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) \t\t\tif oIdx >= 0 { \t\t\t\to := oldToks[oIdx] \t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { -\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { -\t\t\t\t\t\tot := oldToks[j] -\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl, ot.Fd, ot.Pd, ot.Lc, ot.Lb, ot.Hd}) -\t\t\t\t\t} +${adoptSuffix} \t\t\t\t\treturn out, relexed \t\t\t\t} \t\t\t} @@ -696,22 +852,25 @@ func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, func scanMeta(src string) []alignMeta { \tvar toks []Tok \tvar meta []alignMeta -\tpos, pendingNl, lineStart, emittedContent, flowDepth := 0, false, true, false, 0 +\tpos := 0 +\tpendingNl, lineStart, emittedContent := false, true, false +\tflowDepth := 0 \tfor pos < len(src) { \t\tbefore := len(toks) \t\tpos, pendingNl, lineStart, emittedContent, flowDepth = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, &toks, 1) \t\tif len(toks) == before { break } \t\tt := toks[len(toks)-1] -\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false}) +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, flowDepth, 0, false, false, false, 0}) \t} \treturn meta } -func toMeta(toks []Tok) []alignMeta { panic("use scanMeta for newline") } +func toMeta(_ []Tok) []alignMeta { panic("use scanMeta for newline") } ` : rxOnly ? ` func scanMeta(src string) []alignMeta { \tvar toks []Tok \tvar meta []alignMeta -\tpos, pendingNl := 0, false +\tpos := 0 +\tpendingNl := false \tprevText, prevKind, bpText := "", "", "" \thasPrev, hasPrev2 := false, false \tvar parenHead []bool @@ -724,24 +883,67 @@ func scanMeta(src string) []alignMeta { \t\ttxt := src[t.Off:t.End] \t\thd := false \t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } -\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd}) +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd, 0}) +\t} +\treturn meta +} +func toMeta(_ []Tok) []alignMeta { panic("use scanMeta for regex") } +` : rxTpl ? ` +func scanMeta(src string) []alignMeta { +\tvar toks []Tok +\tvar meta []alignMeta +\tpos := 0 +\tpendingNl := false +\tprevText, prevKind, bpText := "", "", "" +\thasPrev, hasPrev2 := false, false +\tvar parenHead []bool +\tlastClose, lastBang := false, false +\tvar templateStack []int +\tfor pos < len(src) { +\t\tbefore := len(toks) +\t\tpos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, templateStack = lexFrom(src, pos, pendingNl, prevText, prevKind, bpText, hasPrev, hasPrev2, parenHead, lastClose, lastBang, templateStack, &toks, 1) +\t\tif len(toks) == before { break } +\t\tt := toks[len(toks)-1] +\t\ttxt := src[t.Off:t.End] +\t\thd := false +\t\tif txt == "(" && len(parenHead) > 0 { hd = parenHead[len(parenHead)-1] } +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, len(parenHead), lastClose, lastBang, hd, len(templateStack)}) +\t} +\treturn meta +} +func toMeta(_ []Tok) []alignMeta { panic("use scanMeta for rx+tpl") } +` : tplOnly ? ` +func scanMeta(src string) []alignMeta { +\tvar toks []Tok +\tvar meta []alignMeta +\tpos := 0 +\tpendingNl := false +\tvar templateStack []int +\tfor pos < len(src) { +\t\tbefore := len(toks) +\t\tpos, pendingNl, templateStack = lexFrom(src, pos, pendingNl, templateStack, &toks, 1) +\t\tif len(toks) == before { break } +\t\tt := toks[len(toks)-1] +\t\tmeta = append(meta, alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false, len(templateStack)}) \t} \treturn meta } -func toMeta(toks []Tok) []alignMeta { panic("use scanMeta for regex") } -` : `func toMeta(toks []Tok) []alignMeta { -\tm := make([]alignMeta, len(toks)) -\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl, 0, 0, false, false, false} } -\treturn m +func toMeta(_ []Tok) []alignMeta { panic("use scanMeta for tpl") } +` : ` +func toMeta(toks []Tok) []alignMeta { +\tout := make([]alignMeta, len(toks)) +\tfor i, t := range toks { out[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl${zeroMeta}} } +\treturn out }`; + const rxTdCheck = rxTpl ? ' || f.Pd != t.Pd || f.Lc != t.Lc || f.Lb != t.Lb || f.Hd != t.Hd' : ''; const checkStreamEqFn = hasNewline ? ` func checkStreamEq(text string, meta []alignMeta) bool { \tfresh := scanMeta(text) \tif len(fresh) != len(meta) { return false } \tfor i := range fresh { -\t\tf, m := fresh[i], meta[i] -\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl || f.Fd != m.Fd { return false } -\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t\tf, t := fresh[i], meta[i] +\t\tif f.Kind != t.Kind || f.Off != t.Off || f.End != t.End || f.Nl != t.Nl || f.Fd != t.Fd { return false } +\t\tif text[f.Off:f.End] != text[t.Off:t.End] { return false } \t} \treturn true } @@ -750,9 +952,20 @@ func checkStreamEq(text string, meta []alignMeta) bool { \tfresh := scanMeta(text) \tif len(fresh) != len(meta) { return false } \tfor i := range fresh { -\t\tf, m := fresh[i], meta[i] -\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl || f.Pd != m.Pd || f.Lc != m.Lc || f.Lb != m.Lb || f.Hd != m.Hd { return false } -\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t\tf, t := fresh[i], meta[i] +\t\tif f.Kind != t.Kind || f.Off != t.Off || f.End != t.End || f.Nl != t.Nl || f.Pd != t.Pd || f.Lc != t.Lc || f.Lb != t.Lb || f.Hd != t.Hd { return false } +\t\tif text[f.Off:f.End] != text[t.Off:t.End] { return false } +\t} +\treturn true +} +` : (rxTpl || tplOnly) ? ` +func checkStreamEq(text string, meta []alignMeta) bool { +\tfresh := scanMeta(text) +\tif len(fresh) != len(meta) { return false } +\tfor i := range fresh { +\t\tf, t := fresh[i], meta[i] +\t\tif f.Kind != t.Kind || f.Off != t.Off || f.End != t.End || f.Nl != t.Nl || f.Td != t.Td${rxTdCheck} { return false } +\t\tif text[f.Off:f.End] != text[t.Off:t.End] { return false } \t} \treturn true } @@ -761,16 +974,16 @@ func checkStreamEq(text string, meta []alignMeta) bool { \tfresh := toMeta(tokenize(text)) \tif len(fresh) != len(meta) { return false } \tfor i := range fresh { -\t\tf, m := fresh[i], meta[i] -\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl { return false } -\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t\tf, t := fresh[i], meta[i] +\t\tif f.Kind != t.Kind || f.Off != t.Off || f.End != t.End || f.Nl != t.Nl { return false } +\t\tif text[f.Off:f.End] != text[t.Off:t.End] { return false } \t} \treturn true } `; - const initToks = (hasNewline || rxOnly) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; + const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `type Edit struct { Start, End int; Text string } -type alignMeta struct { Kind string; Off, End int; Nl bool; Fd, Pd int; Lc, Lb, Hd bool } +type alignMeta struct { Kind string; Off, End int; Nl bool; Fd, Pd int; Lc, Lb, Hd bool; Td int } type Align struct { \tOldN int \`json:"oldN"\` \tNewN int \`json:"newN"\` diff --git a/src/target-rust.ts b/src/target-rust.ts index 952a56c..73ec9fb 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -178,7 +178,9 @@ function lexer(ir: ParserIR): string { const nl = ir.newlineCfg; const nlRs = nl ? newlinePartsRs(nl) : null; const rxOnly = !!(rx && !tpl && !nl); - const rxOrTpl = !!(rx || tpl) && !rxOnly; + const tplOnly = !!(tpl && !rx && !nl); + const rxTpl = !!(rx && tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly && !tplOnly && !rxTpl; const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); @@ -254,6 +256,33 @@ impl<'a, 'b> RxScan<'a, 'b> { self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true; } } +` : ''; + const tplScanImpl = tplOnly ? `struct TplScan<'a, 'b> { acc: &'a mut Vec>, pending_nl: bool, template_stack: Vec } +impl<'a, 'b> TplScan<'a, 'b> { + fn emit(&mut self, kind: &'static str, text: &'b str, off: usize, end: usize) { + if !self.template_stack.is_empty() { if text == ${J(tpl.braceOpen)} { *self.template_stack.last_mut().unwrap() += 1; } else if text == ${J(tpl.interpClose)} { *self.template_stack.last_mut().unwrap() -= 1; } } + self.acc.push(Tok { kind, text, off, end, nl: self.pending_nl }); self.pending_nl = false; + } +} +` : ''; + const rxTplScanImpl = rxTpl ? `struct RxTplScan<'a, 'b> { acc: &'a mut Vec>, pending_nl: bool, prev_text: &'b str, prev_kind: &'static str, bp_text: &'b str, has_prev: bool, has_prev2: bool, paren_head: Vec, last_close: bool, last_bang: bool, template_stack: Vec } +impl<'a, 'b> RxTplScan<'a, 'b> { + fn prev_is_value(&self) -> bool { + if !self.has_prev { return false; } + if _in(_PAV, self.prev_text) { return self.last_bang; } + let is_expr_kw = self.prev_kind == _IDENT && _in(_RXT, self.prev_text); + let is_paren_head = self.prev_text == ")" && self.last_close; + !is_expr_kw && !is_paren_head && (_in(_DIVK, self.prev_kind) || _in(_DIVT, self.prev_text)) + } + fn emit(&mut self, kind: &'static str, text: &'b str, off: usize, end: usize) { + if text == "(" { let is_member = self.has_prev2 && _in(_MEM, self.bp_text); self.paren_head.push(!is_member && self.prev_kind == _IDENT && _in(_PHK, self.prev_text)); } + else if text == ")" { self.last_close = self.paren_head.pop().unwrap_or(false); } + if _in(_PAV, text) { self.last_bang = self.prev_is_value(); } + if !self.template_stack.is_empty() { if text == ${J(tpl.braceOpen)} { *self.template_stack.last_mut().unwrap() += 1; } else if text == ${J(tpl.interpClose)} { *self.template_stack.last_mut().unwrap() -= 1; } } + self.acc.push(Tok { kind, text, off, end, nl: self.pending_nl }); self.pending_nl = false; + self.bp_text = self.prev_text; self.has_prev2 = self.has_prev; self.prev_kind = kind; self.prev_text = text; self.has_prev = true; + } +} ` : ''; const initFields = ['toks: Vec::new()', 'pending_nl: false', rx ? 'prev_text: "", prev_kind: "", bp_text: "", has_prev: false, has_prev2: false, paren_head: Vec::new(), last_close: false, last_bang: false' : '', @@ -301,6 +330,40 @@ fn lex<'a>(src: &'a str) -> Vec> { let mut toks = Vec::new(); lex_from(src, 0, false, "", "", "", false, false, Vec::new(), false, false, &mut toks, 0); toks +}`; + } + if (tplOnly) { + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${tplScanImpl}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, template_stack: Vec, acc: &mut Vec>, limit: usize) -> (usize, bool, Vec) { + let b = src.as_bytes(); + let n = b.len(); + let base = acc.len(); + let mut st = TplScan { acc, pending_nl, template_stack }; + while pos < n && (limit == 0 || st.acc.len() - base < limit) { +${loopBody} + } + (pos, st.pending_nl, st.template_stack) +} +fn lex<'a>(src: &'a str) -> Vec> { + let mut toks = Vec::new(); + lex_from(src, 0, false, Vec::new(), &mut toks, 0); + toks +}`; + } + if (rxTpl) { + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${rxTplScanImpl}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, mut prev_text: &'a str, mut prev_kind: &'static str, mut bp_text: &'a str, mut has_prev: bool, mut has_prev2: bool, mut paren_head: Vec, mut last_close: bool, mut last_bang: bool, template_stack: Vec, acc: &mut Vec>, limit: usize) -> (usize, bool, &'a str, &'static str, &'a str, bool, bool, Vec, bool, bool, Vec) { + let b = src.as_bytes(); + let n = b.len(); + let base = acc.len(); + let mut st = RxTplScan { acc, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang, template_stack }; + while pos < n && (limit == 0 || st.acc.len() - base < limit) { +${loopBody} + } + (pos, st.pending_nl, st.prev_text, st.prev_kind, st.bp_text, st.has_prev, st.has_prev2, st.paren_head, st.last_close, st.last_bang, st.template_stack) +} +fn lex<'a>(src: &'a str) -> Vec> { + let mut toks = Vec::new(); + lex_from(src, 0, false, "", "", "", false, false, Vec::new(), false, false, Vec::new(), &mut toks, 0); + toks }`; } if (rxOrTpl) { @@ -526,9 +589,53 @@ ${r.nudSeqs.map((seq) => ` { let save = self.pos; let sb = self.scratch.l } function docEditBlockRust(ir: ParserIR): string { - const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const windowLex = (!ir.regexCtx && !ir.tpl) || !ir.newlineCfg; const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); + const tplOnly = !!(ir.tpl && !ir.regexCtx && !ir.newlineCfg); + const rxTpl = !!(ir.regexCtx && ir.tpl && !ir.newlineCfg); + const adoptSuffix = ` for j in (o_idx + 1)..old_toks.len() { + let ot = &old_toks[j]; + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd, td: ot.td }); + }`; + const findTokAtOff = ` +fn find_tok_at_off(toks: &[AlignMeta], off: usize) -> Option { + let mut lo = 0usize; let mut hi = toks.len(); + while lo < hi { + let mid = (lo + hi) / 2; + if toks[mid].off < off { lo = mid + 1; } else if toks[mid].off > off { hi = mid; } else { return Some(mid); } + } + None +}`; + const reconstructParens = ` +fn reconstruct_parens(toks: &[AlignMeta], text: &str, b: isize) -> Vec { + let mut need = if b >= 0 { toks[b as usize].pd } else { 0 }; + let mut out = vec![false; need as usize]; + let mut i = b; + while i >= 0 && need > 0 { + let t = &toks[i as usize]; + if &text[t.off..t.end] == "(" && t.pd == need { + out[(need - 1) as usize] = t.hd; + need -= 1; + } + i -= 1; + } + out +} +fn paren_stacks_eq(a: &[bool], b: &[bool]) -> bool { a == b }`; + const tplAnchor = ` let mut max_idx: isize = -1; + for (i, t) in old_toks.iter().enumerate() { + if t.end < start { max_idx = i as isize; } else { break; } + } + let rb0: isize = if max_idx >= 0 { max_idx - 1 } else { -1 }; + let mut rb: isize = -1; + if rb0 >= 0 { + for i in rb0 as usize..old_toks.len() { + if old_toks[i].end > start { break; } + if old_toks[i].td == 0 { rb = i as isize; break; } + } + } + let mut out: Vec = if rb >= 0 { old_toks[..=rb as usize].to_vec() } else { Vec::new() };`; const windowHelpers = windowLex ? (hasNewline ? ` fn find_tok_at_off_kind(toks: &[AlignMeta], off: usize, kind: &'static str) -> Option { let mut lo = 0usize; @@ -570,14 +677,14 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta (scan_off, pending_nl, line_start, emitted_content, flow_depth) = lex_from(new_text, scan_off, pending_nl, line_start, emitted_content, flow_depth, &mut scratch, 1); if scratch.len() == before { break; } let t = &scratch[scratch.len() - 1]; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false, td: 0 }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off_kind(old_toks, (t.off as isize - delta) as usize, t.kind) { let o = &old_toks[o_idx]; if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && o.fd == flow_depth && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd, td: ot.td }); } return (out, relexed); } @@ -658,7 +765,7 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta let t = &scratch[scratch.len() - 1]; let txt = &new_text[t.off..t.end]; let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd, td: 0 }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { @@ -669,7 +776,7 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta let old_stack = reconstruct_parens(old_toks, old_text, o_idx as isize); if o.pd == paren_head.len() as i64 && paren_stacks_eq(&old_stack, &paren_head) && o.lc == last_close && o.lb == last_bang && bp_ok && o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd, td: ot.td }); } return (out, relexed); } @@ -678,6 +785,96 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta } (out, relexed) } +` : rxTpl ? `${findTokAtOff}${reconstructParens} +fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, start: usize, end: usize, ins: &str) -> (Vec, usize) { + let delta = ins.len() as isize - (end - start) as isize; + let edit_end = start + ins.len(); +${tplAnchor} + let mut scan_off: usize; + let mut pending_nl = false; + let mut prev_text: &str = ""; + let mut prev_kind: &'static str = ""; + let mut bp_text: &str = ""; + let mut has_prev = false; + let mut has_prev2 = false; + let mut paren_head: Vec = Vec::new(); + let mut last_close = false; + let mut last_bang = false; + let mut template_stack: Vec = Vec::new(); + if rb >= 0 { + let anchor = &old_toks[rb as usize]; + scan_off = anchor.end; + prev_text = &old_text[anchor.off..anchor.end]; + prev_kind = anchor.kind; + has_prev = true; + if rb >= 1 { + let p = &old_toks[(rb - 1) as usize]; + bp_text = &old_text[p.off..p.end]; + has_prev2 = true; + } + last_close = anchor.lc; + last_bang = anchor.lb; + paren_head = reconstruct_parens(old_toks, old_text, rb); + } else { + scan_off = 0; + } + let mut scratch: Vec> = Vec::new(); + let mut relexed = 0usize; + while scan_off < new_text.len() { + let before = scratch.len(); + let r = lex_from(new_text, scan_off, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang, template_stack, &mut scratch, 1); + scan_off = r.0; pending_nl = r.1; prev_text = r.2; prev_kind = r.3; bp_text = r.4; has_prev = r.5; has_prev2 = r.6; paren_head = r.7; last_close = r.8; last_bang = r.9; template_stack = r.10; + if scratch.len() == before { break; } + let t = &scratch[scratch.len() - 1]; + let txt = &new_text[t.off..t.end]; + let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd, td: template_stack.len() as i64 }); + relexed += 1; + if t.off >= edit_end { + if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { + let o = &old_toks[o_idx]; + let new_prev = if out.len() > 1 { &new_text[out[out.len()-2].off..out[out.len()-2].end] } else { "" }; + let old_prev = if o_idx >= 1 { &old_text[old_toks[o_idx-1].off..old_toks[o_idx-1].end] } else { "" }; + let old_stack = reconstruct_parens(old_toks, old_text, o_idx as isize); + if o.td == 0 && template_stack.is_empty() && o.pd == paren_head.len() as i64 && paren_stacks_eq(&old_stack, &paren_head) && o.lc == last_close && o.lb == last_bang && new_prev == old_prev && o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { +${adoptSuffix} + return (out, relexed); + } + } + } + } + (out, relexed) +} +` : tplOnly ? `${findTokAtOff} +fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, start: usize, end: usize, ins: &str) -> (Vec, usize) { + let delta = ins.len() as isize - (end - start) as isize; + let edit_end = start + ins.len(); +${tplAnchor} + let mut scan_off = if rb >= 0 { old_toks[rb as usize].end } else { 0 }; + let mut pending_nl = false; + let mut template_stack: Vec = Vec::new(); + let mut scratch: Vec> = Vec::new(); + let mut relexed = 0usize; + while scan_off < new_text.len() { + let before = scratch.len(); + let r = lex_from(new_text, scan_off, pending_nl, template_stack, &mut scratch, 1); + scan_off = r.0; pending_nl = r.1; template_stack = r.2; + if scratch.len() == before { break; } + let t = &scratch[scratch.len() - 1]; + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: template_stack.len() as i64 }); + relexed += 1; + if t.off >= edit_end { + if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { + let o = &old_toks[o_idx]; + if o.td == 0 && template_stack.is_empty() && o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { +${adoptSuffix} + return (out, relexed); + } + } + } + } + (out, relexed) +} ` : ` fn find_tok_at_off(toks: &[AlignMeta], off: usize) -> Option { let mut lo = 0usize; @@ -706,14 +903,14 @@ fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, sta (scan_off, pending_nl) = lex_from(new_text, scan_off, pending_nl, &mut scratch, 1); if scratch.len() == before { break; } let t = &scratch[scratch.len() - 1]; - out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false }); + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: 0 }); relexed += 1; if t.off >= edit_end { if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { let o = &old_toks[o_idx]; if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { for ot in &old_toks[o_idx + 1..] { - out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd, td: ot.td }); } return (out, relexed); } @@ -758,7 +955,7 @@ fn scan_meta(src: &str) -> Vec { (pos, pending_nl, line_start, emitted_content, flow_depth) = lex_from(src, pos, pending_nl, line_start, emitted_content, flow_depth, &mut toks, 1); if toks.len() == before { break; } let t = &toks[toks.len() - 1]; - meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false }); + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flow_depth, pd: 0, lc: false, lb: false, hd: false, td: 0 }); } meta } @@ -779,13 +976,59 @@ fn scan_meta(src: &str) -> Vec { let t = &toks[toks.len() - 1]; let txt = &src[t.off..t.end]; let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; - meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd }); + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd, td: 0 }); } meta } fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for regex") } +` : rxTpl ? ` +fn scan_meta(src: &str) -> Vec { + let mut toks: Vec = Vec::new(); + let mut meta: Vec = Vec::new(); + let mut pos = 0usize; + let mut pending_nl = false; + let mut prev_text: &str = ""; + let mut prev_kind: &'static str = ""; + let mut bp_text: &str = ""; + let mut has_prev = false; + let mut has_prev2 = false; + let mut paren_head: Vec = Vec::new(); + let mut last_close = false; + let mut last_bang = false; + let mut template_stack: Vec = Vec::new(); + while pos < src.len() { + let before = toks.len(); + let r = lex_from(src, pos, pending_nl, prev_text, prev_kind, bp_text, has_prev, has_prev2, paren_head, last_close, last_bang, template_stack, &mut toks, 1); + pos = r.0; pending_nl = r.1; prev_text = r.2; prev_kind = r.3; bp_text = r.4; has_prev = r.5; has_prev2 = r.6; paren_head = r.7; last_close = r.8; last_bang = r.9; template_stack = r.10; + if toks.len() == before { break; } + let t = &toks[toks.len() - 1]; + let txt = &src[t.off..t.end]; + let hd = if txt == "(" && !paren_head.is_empty() { paren_head[paren_head.len() - 1] } else { false }; + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: paren_head.len() as i64, lc: last_close, lb: last_bang, hd, td: template_stack.len() as i64 }); + } + meta +} +fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for rx+tpl") } +` : tplOnly ? ` +fn scan_meta(src: &str) -> Vec { + let mut toks: Vec = Vec::new(); + let mut meta: Vec = Vec::new(); + let mut pos = 0usize; + let mut pending_nl = false; + let mut template_stack: Vec = Vec::new(); + while pos < src.len() { + let before = toks.len(); + let r = lex_from(src, pos, pending_nl, template_stack, &mut toks, 1); + pos = r.0; pending_nl = r.1; template_stack = r.2; + if toks.len() == before { break; } + let t = &toks[toks.len() - 1]; + meta.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: template_stack.len() as i64 }); + } + meta +} +fn to_meta(_toks: &[Tok<'_>]) -> Vec { panic!("use scan_meta for tpl") } ` : `fn to_meta(toks: &[Tok<'_>]) -> Vec { - toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false }).collect() + toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: 0 }).collect() }`; const checkStreamEqFn = hasNewline ? ` fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { @@ -807,6 +1050,16 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { } true } +` : (rxTpl || tplOnly) ? ` +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = scan_meta(text); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl || f.td != m.td${rxTpl ? ' || f.pd != m.pd || f.lc != m.lc || f.lb != m.lb || f.hd != m.hd' : ''} { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} ` : ` fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { let fresh = to_meta(&lex(text)); @@ -818,10 +1071,10 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { true } `; - const initToks = (hasNewline || rxOnly) ? 'scan_meta(&text)' : 'to_meta(&lex(&text))'; + const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scan_meta(&text)' : 'to_meta(&lex(&text))'; return `pub struct Edit { pub start: usize, pub end: usize, pub text: String } #[derive(Clone)] -struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool } +struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool, td: i64 } struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool } ${toMetaFn} fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) { @@ -877,6 +1130,7 @@ ${editBody} }`; } + export const rustTarget: Target = { name: 'rust', ext: 'rs', diff --git a/src/target-ts.ts b/src/target-ts.ts index 8c7ee1d..c9e2a44 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -140,7 +140,9 @@ function lexer(ir: ParserIR): string { const tpl = ir.tpl; const nl = ir.newlineCfg; const rxOnly = !!(rx && !tpl && !nl); - const rxOrTpl = !!(rx || tpl) && !rxOnly; + const tplOnly = !!(tpl && !rx && !nl); + const rxTpl = !!(rx && tpl && !nl); + const rxOrTpl = !!(rx || tpl) && !rxOnly && !tplOnly && !rxTpl; const stateful = !!(rx || tpl); const newlineOnly = !!(nl && !rx && !tpl); const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n'); @@ -197,10 +199,35 @@ ${emitHooks} const isParenHead = prevText === ')' && lastClose; return !isExprKw && !isParenHead && (_divK.has(prevKind) || _divT.has(prevText)); } - function emit(kind: string, text: string, off: number, end: number): void { +` : ''; + const tplStateFrom = tpl ? ` function scanTplSpan(p: number): { interp: boolean; end: number } { + while (p < n) { + if (src.startsWith(${J(tpl.interpOpen)}, p)) return { interp: true, end: p + ${tpl.interpOpen.length} }; + if (src.charCodeAt(p) === 92) { p += 2; continue; } + if (src.startsWith(${J(tpl.open)}, p)) return { interp: false, end: p + ${tpl.open.length} }; + p++; + } + return { interp: false, end: p }; + } +` : ''; + const emitRxOnly = rx ? ` function emit(kind: string, text: string, off: number, end: number): void { + if (text === '(') { const isMember = hasPrev2 && _mem.has(bpText); parenHead.push(!isMember && prevKind === IDENT && _phK.has(prevText)); } + else if (text === ')') { lastClose = parenHead.pop() ?? false; } + if (_pav.has(text)) lastBang = prevIsValue(); + toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; + bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true; + } +` : ''; + const emitTplOnly = tpl ? ` function emit(kind: string, text: string, off: number, end: number): void { + if (templateStack.length > 0) { if (text === ${J(tpl.braceOpen)}) templateStack[templateStack.length - 1]++; else if (text === ${J(tpl.interpClose)}) templateStack[templateStack.length - 1]--; } + toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; + } +` : ''; + const emitRxTpl = (rx && tpl) ? ` function emit(kind: string, text: string, off: number, end: number): void { if (text === '(') { const isMember = hasPrev2 && _mem.has(bpText); parenHead.push(!isMember && prevKind === IDENT && _phK.has(prevText)); } else if (text === ')') { lastClose = parenHead.pop() ?? false; } if (_pav.has(text)) lastBang = prevIsValue(); + if (templateStack.length > 0) { if (text === ${J(tpl.braceOpen)}) templateStack[templateStack.length - 1]++; else if (text === ${J(tpl.interpClose)}) templateStack[templateStack.length - 1]--; } toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; bpText = prevText; hasPrev2 = hasPrev; prevKind = kind; prevText = text; hasPrev = true; } @@ -242,7 +269,7 @@ ${puncts} return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, prevText: string, prevKind: string, hasPrev: boolean, bpText: string, hasPrev2: boolean, parenHead: boolean[], lastClose: boolean, lastBang: boolean, toks: Tok[], limit?: number): { pos: number; pendingNl: boolean; prevText: string; prevKind: string; hasPrev: boolean; bpText: string; hasPrev2: boolean; parenHead: boolean[]; lastClose: boolean; lastBang: boolean } { const n = src.length; const base = toks.length; -${defs.length ? ' _s = src;\n' : ''}${rxStateFrom} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${defs.length ? ' _s = src;\n' : ''}${rxStateFrom}${emitRxOnly} while (pos < n && (limit === undefined || toks.length - base < limit)) { ${loopBody} } return { pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang }; @@ -251,6 +278,36 @@ function lex(src: string): Tok[] { const toks: Tok[] = []; lexFrom(src, 0, false, '', '', false, '', false, [], false, false, toks); return toks; +}`; + } + if (tplOnly) { + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, templateStack: number[], toks: Tok[], limit?: number): { pos: number; pendingNl: boolean; templateStack: number[] } { + const n = src.length; + const base = toks.length; +${defs.length ? ' _s = src;\n' : ''}${tplStateFrom}${emitTplOnly} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${loopBody} + } + return { pos, pendingNl, templateStack }; +} +function lex(src: string): Tok[] { + const toks: Tok[] = []; + lexFrom(src, 0, false, [], toks); + return toks; +}`; + } + if (rxTpl) { + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, prevText: string, prevKind: string, hasPrev: boolean, bpText: string, hasPrev2: boolean, parenHead: boolean[], lastClose: boolean, lastBang: boolean, templateStack: number[], toks: Tok[], limit?: number): { pos: number; pendingNl: boolean; prevText: string; prevKind: string; hasPrev: boolean; bpText: string; hasPrev2: boolean; parenHead: boolean[]; lastClose: boolean; lastBang: boolean; templateStack: number[] } { + const n = src.length; + const base = toks.length; +${defs.length ? ' _s = src;\n' : ''}${rxStateFrom}${tplStateFrom}${emitRxTpl} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${loopBody} + } + return { pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, templateStack }; +} +function lex(src: string): Tok[] { + const toks: Tok[] = []; + lexFrom(src, 0, false, '', '', false, '', false, [], false, false, [], toks); + return toks; }`; } if (rxOrTpl) { @@ -425,10 +482,56 @@ ${r.nudSeqs.map((seq) => ` { const save = pos; const kids: Cst[] = []; if (${se } function docEditBlock(ir: ParserIR): string { - const windowLex = !ir.tpl && (!ir.regexCtx || !ir.newlineCfg); + const windowLex = (!ir.regexCtx && !ir.tpl) || !ir.newlineCfg; const hasNewline = !!(ir.newlineCfg && !ir.regexCtx && !ir.tpl); const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg); - const zeroMeta = ', fd: 0, pd: 0, lc: false, lb: false, hd: false'; + const tplOnly = !!(ir.tpl && !ir.regexCtx && !ir.newlineCfg); + const rxTpl = !!(ir.regexCtx && ir.tpl && !ir.newlineCfg); + const zeroMeta = ', fd: 0, pd: 0, lc: false, lb: false, hd: false, td: 0'; + const adoptSuffix = `for (let j = oIdx + 1; j < oldToks.length; j++) { + const ot = oldToks[j]; + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd, td: ot.td }); + }`; + const findTokAtOff = ` +function findTokAtOff(toks: AlignMeta[], off: number): number { + let lo = 0, hi = toks.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (toks[mid].off < off) lo = mid + 1; + else if (toks[mid].off > off) hi = mid - 1; + else return mid; + } + return -1; +}`; + const reconstructParens = ` +function reconstructParens(toks: AlignMeta[], text: string, b: number): boolean[] { + let need = b >= 0 ? toks[b].pd : 0; + const out: boolean[] = []; + for (let i = b; i >= 0 && need > 0; i--) { + const t = toks[i]; + if (text.slice(t.off, t.end) === '(' && t.pd === need) { out[need - 1] = t.hd; need--; } + } + return out; +} +function parenStacksEq(a: boolean[], b: boolean[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +}`; + const tplAnchor = ` let maxIdx = -1; + for (let i = 0; i < oldToks.length; i++) { + if (oldToks[i].end < start) maxIdx = i; + else break; + } + const rb0 = maxIdx >= 0 ? maxIdx - 1 : -1; + let rb = -1; + if (rb0 >= 0) { + for (let i = rb0; i < oldToks.length; i++) { + if (oldToks[i].end > start) break; + if (oldToks[i].td === 0) { rb = i; break; } + } + } + const out: AlignMeta[] = rb >= 0 ? oldToks.slice(0, rb + 1) : [];`; const windowHelpers = windowLex ? (hasNewline ? ` function findTokAtOffKind(toks: AlignMeta[], off: number, kind: string): number { let lo = 0, hi = toks.length - 1, hit = -1; @@ -469,17 +572,14 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, ({ pos: scanOff, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(newText, scanOff, pendingNl, lineStart, emittedContent, flowDepth, scratch, 1)); if (scratch.length === before) break; const t = scratch[scratch.length - 1]; - out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false }); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false, td: 0 }); relexed++; if (t.off >= editEnd) { const oIdx = findTokAtOffKind(oldToks, t.off - delta, t.kind); if (oIdx >= 0) { const o = oldToks[oIdx]; if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && o.fd === flowDepth && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { - for (let j = oIdx + 1; j < oldToks.length; j++) { - const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); - } + ${adoptSuffix} return { toks: out, relexed }; } } @@ -487,31 +587,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, } return { toks: out, relexed }; } -` : rxOnly ? ` -function findTokAtOff(toks: AlignMeta[], off: number): number { - let lo = 0, hi = toks.length - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - if (toks[mid].off < off) lo = mid + 1; - else if (toks[mid].off > off) hi = mid - 1; - else return mid; - } - return -1; -} -function reconstructParens(toks: AlignMeta[], text: string, b: number): boolean[] { - let need = b >= 0 ? toks[b].pd : 0; - const out: boolean[] = []; - for (let i = b; i >= 0 && need > 0; i--) { - const t = toks[i]; - if (text.slice(t.off, t.end) === '(' && t.pd === need) { out[need - 1] = t.hd; need--; } - } - return out; -} -function parenStacksEq(a: boolean[], b: boolean[]): boolean { - if (a.length !== b.length) return false; - for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; - return true; -} +` : rxOnly ? `${findTokAtOff}${reconstructParens} function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { const delta = ins.length - (end - start); const editEnd = start + ins.length; @@ -543,7 +619,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, if (scratch.length === before) break; const t = scratch[scratch.length - 1]; const txt = newText.slice(t.off, t.end); - out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false }); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false, td: 0 }); relexed++; if (t.off >= editEnd) { const oIdx = findTokAtOff(oldToks, t.off - delta); @@ -554,10 +630,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, const bpOk = newPrevText === oldPrevText; const oldStack = reconstructParens(oldToks, oldText, oIdx); if (o.pd === parenHead.length && parenStacksEq(oldStack, parenHead) && o.lc === lastClose && o.lb === lastBang && bpOk && o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { - for (let j = oIdx + 1; j < oldToks.length; j++) { - const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); - } + ${adoptSuffix} return { toks: out, relexed }; } } @@ -565,17 +638,83 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, } return { toks: out, relexed }; } -` : ` -function findTokAtOff(toks: AlignMeta[], off: number): number { - let lo = 0, hi = toks.length - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - if (toks[mid].off < off) lo = mid + 1; - else if (toks[mid].off > off) hi = mid - 1; - else return mid; +` : rxTpl ? `${findTokAtOff}${reconstructParens} +function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { + const delta = ins.length - (end - start); + const editEnd = start + ins.length; +${tplAnchor} + let scanOff: number, pendingNl: boolean; + let prevText = '', prevKind = '', bpText = '', hasPrev = false, hasPrev2 = false; + let parenHead: boolean[] = [], lastClose = false, lastBang = false; + let templateStack: number[] = []; + if (rb >= 0) { + const anchor = oldToks[rb]; + scanOff = anchor.end; pendingNl = false; + prevText = oldText.slice(anchor.off, anchor.end); prevKind = anchor.kind; hasPrev = true; + if (rb >= 1) { bpText = oldText.slice(oldToks[rb - 1].off, oldToks[rb - 1].end); hasPrev2 = true; } + lastClose = anchor.lc; lastBang = anchor.lb; + parenHead = reconstructParens(oldToks, oldText, rb); + } else { + scanOff = 0; pendingNl = false; } - return -1; + const scratch: Tok[] = []; + let relexed = 0; + while (scanOff < newText.length) { + const before = scratch.length; + ({ pos: scanOff, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, templateStack } = lexFrom(newText, scanOff, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, templateStack, scratch, 1)); + if (scratch.length === before) break; + const t = scratch[scratch.length - 1]; + const txt = newText.slice(t.off, t.end); + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false, td: templateStack.length }); + relexed++; + if (t.off >= editEnd) { + const oIdx = findTokAtOff(oldToks, t.off - delta); + if (oIdx >= 0) { + const o = oldToks[oIdx]; + const newPrevText = out.length > 1 ? newText.slice(out[out.length - 2].off, out[out.length - 2].end) : ''; + const oldPrevText = oIdx >= 1 ? oldText.slice(oldToks[oIdx - 1].off, oldToks[oIdx - 1].end) : ''; + const bpOk = newPrevText === oldPrevText; + const oldStack = reconstructParens(oldToks, oldText, oIdx); + if (o.td === 0 && templateStack.length === 0 && o.pd === parenHead.length && parenStacksEq(oldStack, parenHead) && o.lc === lastClose && o.lb === lastBang && bpOk && o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { + ${adoptSuffix} + return { toks: out, relexed }; + } + } + } + } + return { toks: out, relexed }; } +` : tplOnly ? `${findTokAtOff} +function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { + const delta = ins.length - (end - start); + const editEnd = start + ins.length; +${tplAnchor} + let scanOff = rb >= 0 ? oldToks[rb].end : 0; + let pendingNl = false; + let templateStack: number[] = []; + const scratch: Tok[] = []; + let relexed = 0; + while (scanOff < newText.length) { + const before = scratch.length; + ({ pos: scanOff, pendingNl, templateStack } = lexFrom(newText, scanOff, pendingNl, templateStack, scratch, 1)); + if (scratch.length === before) break; + const t = scratch[scratch.length - 1]; + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: templateStack.length }); + relexed++; + if (t.off >= editEnd) { + const oIdx = findTokAtOff(oldToks, t.off - delta); + if (oIdx >= 0) { + const o = oldToks[oIdx]; + if (o.td === 0 && templateStack.length === 0 && o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { + ${adoptSuffix} + return { toks: out, relexed }; + } + } + } + } + return { toks: out, relexed }; +} +` : `${findTokAtOff} function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { const delta = ins.length - (end - start); const editEnd = start + ins.length; @@ -602,10 +741,7 @@ function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, if (oIdx >= 0) { const o = oldToks[oIdx]; if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { - for (let j = oIdx + 1; j < oldToks.length; j++) { - const ot = oldToks[j]; - out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl, fd: ot.fd, pd: ot.pd, lc: ot.lc, lb: ot.lb, hd: ot.hd }); - } + ${adoptSuffix} return { toks: out, relexed }; } } @@ -645,7 +781,7 @@ function scanMeta(src: string): AlignMeta[] { ({ pos, pendingNl, lineStart, emittedContent, flowDepth } = lexFrom(src, pos, pendingNl, lineStart, emittedContent, flowDepth, toks, 1)); if (toks.length === before) break; const t = toks[toks.length - 1]; - meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false }); + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: flowDepth, pd: 0, lc: false, lb: false, hd: false, td: 0 }); } return meta; } @@ -663,11 +799,46 @@ function scanMeta(src: string): AlignMeta[] { if (toks.length === before) break; const t = toks[toks.length - 1]; const txt = src.slice(t.off, t.end); - meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false }); + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false, td: 0 }); } return meta; } const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for regex'); }; +` : rxTpl ? ` +function scanMeta(src: string): AlignMeta[] { + const toks: Tok[] = []; + const meta: AlignMeta[] = []; + let pos = 0, pendingNl = false; + let prevText = '', prevKind = '', bpText = '', hasPrev = false, hasPrev2 = false; + let parenHead: boolean[] = [], lastClose = false, lastBang = false; + let templateStack: number[] = []; + while (pos < src.length) { + const before = toks.length; + ({ pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, templateStack } = lexFrom(src, pos, pendingNl, prevText, prevKind, hasPrev, bpText, hasPrev2, parenHead, lastClose, lastBang, templateStack, toks, 1)); + if (toks.length === before) break; + const t = toks[toks.length - 1]; + const txt = src.slice(t.off, t.end); + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: parenHead.length, lc: lastClose, lb: lastBang, hd: txt === '(' ? parenHead[parenHead.length - 1]! : false, td: templateStack.length }); + } + return meta; +} +const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for rx+tpl'); }; +` : tplOnly ? ` +function scanMeta(src: string): AlignMeta[] { + const toks: Tok[] = []; + const meta: AlignMeta[] = []; + let pos = 0, pendingNl = false; + let templateStack: number[] = []; + while (pos < src.length) { + const before = toks.length; + ({ pos, pendingNl, templateStack } = lexFrom(src, pos, pendingNl, templateStack, toks, 1)); + if (toks.length === before) break; + const t = toks[toks.length - 1]; + meta.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl, fd: 0, pd: 0, lc: false, lb: false, hd: false, td: templateStack.length }); + } + return meta; +} +const toMeta = (_toks: Tok[]): AlignMeta[] => { throw new Error('use scanMeta for tpl'); }; ` : `const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl${zeroMeta} }));`; const checkStreamEqFn = hasNewline ? ` function checkStreamEq(text: string, meta: AlignMeta[]): boolean { @@ -691,6 +862,17 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean { } return true; } +` : (rxTpl || tplOnly) ? ` +function checkStreamEq(text: string, meta: AlignMeta[]): boolean { + const fresh = scanMeta(text); + if (fresh.length !== meta.length) return false; + for (let i = 0; i < fresh.length; i++) { + const f = fresh[i], t = meta[i]; + if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl || f.td !== t.td${rxTpl ? ' || f.pd !== t.pd || f.lc !== t.lc || f.lb !== t.lb || f.hd !== t.hd' : ''}) return false; + if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; + } + return true; +} ` : ` function checkStreamEq(text: string, meta: AlignMeta[]): boolean { const fresh = toMeta(tokenize(text)); @@ -703,9 +885,9 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean { return true; } `; - const initToks = (hasNewline || rxOnly) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; + const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scanMeta(src)' : 'toMeta(tokenize(src))'; return `export type Edit = { start: number; end: number; text: string }; -type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean }; +type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean; td: number }; type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean }; ${toMetaFn} function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit { diff --git a/test/portable-targets.ts b/test/portable-targets.ts index 7ddd29e..a28af60 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -341,11 +341,20 @@ const EDIT_SCENARIOS: Record = { { init: calcLargeInit, batches: [[[0, 1, '9']]], maxRelexed: 8 }, ], javascript: [ - { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]], fullRelex: true }, - { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]], fullRelex: true }, - { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]], fullRelex: true }, - { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]], fullRelex: true }, - { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true, fullRelex: true }, + { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]] }, + { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]] }, + { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]] }, + { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]] }, + { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true, maxRelexed: 15 }, + { init: 'var q = `x${a/b}y`;', batches: [[[12, 13, 'c']]] }, + ], + templatejs: [ + { init: '`a${x}b`;', batches: [[[4, 5, 'y']]] }, + { init: '`a${ {x} }b`;', batches: [[[6, 7, 'y']]] }, + { init: '`a${x}b`;', batches: [[[2, 4, '']]] }, + { init: '`a${x}b${y}c`;', batches: [[[6, 7, 'z']]] }, + { init: ('`a${x}b`; f(1);\n').repeat(50), batches: [[[600, 601, '9']]], large: true, maxRelexed: 15 }, + { init: '`a`; z; `b${x}c`;', batches: [[[5, 6, 'w']]] }, ], envspec: [ { init: 'A=1\nB=2', batches: [[[2, 2, '3']]] },