Skip to content

emit-portable: make streamEq self-check opt-in (S6a, #57) #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
johnsoncodehk merged 1 commit into from
Jul 12, 2026
+56 −25
Merged

emit-portable: make streamEq self-check opt-in (S6a, #57) #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions src/target-go.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -985,12 +985,12 @@ func checkStreamEq(text string, meta []alignMeta) bool {
return `type Edit struct { Start, End int; Text string }
type alignMeta struct { Kind string; Off, End int; Nl bool; Fd, Pd int; Lc, Lb, Hd bool; Td int }
type Align struct {
\tOldN int \`json:"oldN"\`
\tNewN int \`json:"newN"\`
\tPrefix int \`json:"prefix"\`
\tSuffix int \`json:"suffix"\`
\tRelexed int \`json:"relexed"\`
\tStreamEq bool \`json:"streamEq"\`
\tOldN int \`json:"oldN"\`
\tNewN int \`json:"newN"\`
\tPrefix int \`json:"prefix"\`
\tSuffix int \`json:"suffix"\`
\tRelexed int \`json:"relexed"\`
\tStreamEq *bool \`json:"streamEq,omitempty"\`
}
${toMetaFn}
func computeAlignCore(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) (oldN, newN, prefix, suffix int) {
Expand All @@ -1016,13 +1016,14 @@ func toksFromMeta(text string, meta []alignMeta) []Tok {
\tfor i, m := range meta { t[i] = Tok{m.Kind, text[m.Off:m.End], m.Off, m.End, m.Nl} }
\treturn t
}
${checkStreamEqFn}${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align }
${checkStreamEqFn}${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align; validate bool }
func NewDoc(src string) *Doc {
\td := &Doc{text: src}
\td.toks = ${initToks}
\td.root = parse(tokenize(src))
\treturn d
}
func (d *Doc) SetValidate(v bool) { d.validate = v }
func (d *Doc) Text() string { return d.text }
func (d *Doc) Root() int32 { return d.root }
func (d *Doc) Align() *Align { return d.align }
Expand All @@ -1039,9 +1040,13 @@ func (d *Doc) Edit(edits []Edit) int32 {
\toldText, oldToks := d.text, d.toks
\trelexed := 0
${editBody}
\tstreamEq := checkStreamEq(d.text, d.toks)
\toldN, newN, prefix, suffix := computeAlignCore(oldText, oldToks, d.text, d.toks)
\td.align = &Align{oldN, newN, prefix, suffix, relexed, streamEq}
\ta := &Align{OldN: oldN, NewN: newN, Prefix: prefix, Suffix: suffix, Relexed: relexed}
\tif d.validate {
\t\tv := checkStreamEq(d.text, d.toks)
\t\ta.StreamEq = &v
\t}
\td.align = a
\td.root = parse(toksFromMeta(d.text, d.toks))
\treturn d.root
}`;
Expand Down Expand Up @@ -1247,8 +1252,10 @@ import (
func main() {
\tdata, _ := io.ReadAll(os.Stdin)
\tsrc := string(data)
\teditFast := len(os.Args) > 1 && os.Args[1] == "edit-session-fast"
\teditSess := editFast || (len(os.Args) > 1 && os.Args[1] == "edit-session")
\t// Self-bench: a numeric arg N times the lex+parse loop and prints ms/iteration.
\tif len(os.Args) > 1 && os.Args[1] != "edit-session" {
\tif len(os.Args) > 1 && !editSess {
\t\tif iters, err := strconv.Atoi(os.Args[1]); err == nil && iters > 0 {
\t\t\tfor i := 0; i < 3; i++ { parse(tokenize(src)) }
\t\t\tt0 := time.Now()
Expand All @@ -1257,13 +1264,14 @@ func main() {
\t\t\treturn
\t\t}
\t}
\tif len(os.Args) > 1 && os.Args[1] == "edit-session" {
\tif editSess {
\t\tvar sess struct {
\t\t\tInit string \`json:"init"\`
\t\t\tBatches [][][3]interface{} \`json:"batches"\`
\t\t}
\t\tif json.Unmarshal(data, &sess) != nil { os.Exit(1) }
\t\td := NewDoc(sess.Init)
\t\tif !editFast { d.SetValidate(true) }
\t\tfor _, batch := range sess.Batches {
\t\t\tedits := make([]Edit, len(batch))
\t\t\tfor i, t := range batch {
Expand Down
17 changes: 11 additions & 6 deletions src/target-rust.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1075,7 +1075,7 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool {
return `pub struct Edit { pub start: usize, pub end: usize, pub text: String }
#[derive(Clone)]
struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool, td: i64 }
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool }
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: Option<bool> }
${toMetaFn}
fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) {
let old_n = old_toks.len();
Expand Down Expand Up @@ -1104,18 +1104,19 @@ fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, ne
fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec<Tok<'a>> {
meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect()
}
${checkStreamEqFn}${windowHelpers}pub struct Doc { text: String, toks: Vec<AlignMeta>, align: Option<Align> }
${checkStreamEqFn}${windowHelpers}pub struct Doc { text: String, toks: Vec<AlignMeta>, align: Option<Align>, validate: bool }
impl Doc {
pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: ${initToks}, align: None } }
pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: ${initToks}, align: None, validate: false } }
pub fn set_validate(&mut self, v: bool) { self.validate = v; }
pub fn text(&self) -> &str { &self.text }
pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() }
pub fn edit(&mut self, edits: &[Edit]) {
let old_text = self.text.clone();
let old_toks = self.toks.clone();
let mut relexed = 0usize;
${editBody}
let stream_eq = check_stream_eq(&self.text, &self.toks);
let (old_n, new_n, prefix, suffix) = compute_align_core(&old_text, &old_toks, &self.text, &self.toks);
let stream_eq = if self.validate { Some(check_stream_eq(&self.text, &self.toks)) } else { None };
self.align = Some(Align { old_n, new_n, prefix, suffix, relexed, stream_eq });
}
pub fn parse(&self) -> Option<(Parser<'_>, i32)> {
Expand Down Expand Up @@ -1382,15 +1383,19 @@ fn main() {
let mut src = String::new();
std::io::stdin().read_to_string(&mut src).unwrap();
let args: Vec<String> = std::env::args().collect();
if args.len() > 1 && args[1] == "edit-session" {
if args.len() > 1 && (args[1] == "edit-session" || args[1] == "edit-session-fast") {
let (init, batches) = parse_edit_session(&src).unwrap();
let mut doc = Doc::new(init);
if args[1] == "edit-session" { doc.set_validate(true); }
for batch in &batches {
let edits: Vec<Edit> = batch.iter().map(|&(s, e, ref t)| Edit { start: s, end: e, text: t.clone() }).collect();
doc.edit(&edits);
}
if let Some(a) = doc.alignment() {
eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, a.stream_eq);
match a.stream_eq {
Some(eq) => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, eq),
None => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed),
}
}
match doc.parse() {
Some((p, root)) => {
Expand Down
14 changes: 8 additions & 6 deletions src/target-ts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -888,7 +888,7 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean {
const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scanMeta(src)' : 'toMeta(tokenize(src))';
return `export type Edit = { start: number; end: number; text: string };
type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean; td: number };
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean };
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq?: boolean };
${toMetaFn}
function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit<Align, 'relexed' | 'streamEq'> {
const oldN = oldToks.length, newN = newToks.length;
Expand All @@ -913,7 +913,8 @@ function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, ne
function toksFromMeta(text: string, meta: AlignMeta[]): Tok[] {
return meta.map((m) => ({ kind: m.kind, text: text.slice(m.off, m.end), off: m.off, end: m.end, nl: m.nl }));
}
${checkStreamEqFn}${windowHelpers}export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
${checkStreamEqFn}${windowHelpers}export function createDoc(src: string, opts?: { validate?: boolean }): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
const validate = opts?.validate === true;
let text = src;
let prevToks = ${initToks};
let align: Align | null = null;
Expand All @@ -926,8 +927,8 @@ ${checkStreamEqFn}${windowHelpers}export function createDoc(src: string): { text
const oldText = text, oldToks = prevToks;
let relexed = 0;
${editBody}
const streamEq = checkStreamEq(text, prevToks);
align = { ...computeAlign(oldText, oldToks, text, prevToks), relexed, streamEq };
const core = { ...computeAlign(oldText, oldToks, text, prevToks), relexed };
align = validate ? { ...core, streamEq: checkStreamEq(text, prevToks) } : core;
root = parse(toksFromMeta(text, prevToks));
return root;
},
Expand Down Expand Up @@ -1062,9 +1063,10 @@ ${docEditBlock(ir)}
// NOT part of the emitted parser. The import is hoisted, so it may follow the library code.
import { readFileSync } from 'node:fs';
const _raw = readFileSync(0, 'utf8');
if (process.argv.includes('edit-session')) {
const _editFast = process.argv.includes('edit-session-fast');
if (_editFast || process.argv.includes('edit-session')) {
const { init, batches } = JSON.parse(_raw) as { init: string; batches: [number, number, string][][] };
const doc = createDoc(init);
const doc = createDoc(init, { validate: !_editFast });
for (const batch of batches) doc.edit(batch.map(([start, end, text]) => ({ start, end, text })));
const a = doc.align();
if (a) process.stderr.write(JSON.stringify(a) + '\\n');
Expand Down
20 changes: 18 additions & 2 deletions test/portable-targets.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,13 +446,16 @@ for (const c of CASES) {

const editScenarios = EDIT_SCENARIOS[c.grammar];
if (editScenarios) {
let editOk = 0, alignOk = 0;
let editOk = 0, alignOk = 0, fastOk = 0;
const editCmd = r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`;
const editArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session'];
const fastArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session-fast'] : ['edit-session-fast'];
const runEdit = (json: string) => runEditSession(editCmd, editArgs, json);
const runFast = (json: string) => runEditSession(editCmd, fastArgs, json);
for (const sc of editScenarios) {
const final = applyEdits(sc.init, sc.batches);
const a = runEdit(JSON.stringify({ init: sc.init, batches: sc.batches }));
const payload = JSON.stringify({ init: sc.init, batches: sc.batches });
const a = runEdit(payload);
const b = r.run(final);
if (a.ok === b.ok && (!a.ok || a.cst === b.cst) && a.ok === oracleOut(final).ok) editOk++;
else {
Expand Down Expand Up @@ -482,9 +485,22 @@ for (const c of CASES) {
failures++;
console.log(` ${c.grammar}/${r.label}: token-align mismatch want=${JSON.stringify(want)} got=${JSON.stringify(a.align)}`);
}
const f = runFast(payload);
const fiveEq = !!(a.align && f.align
&& a.align.oldN === f.align.oldN && a.align.newN === f.align.newN
&& a.align.prefix === f.align.prefix && a.align.suffix === f.align.suffix
&& a.align.relexed === f.align.relexed);
const noStreamEq = f.align !== undefined && !('streamEq' in f.align);
const outEq = a.ok === f.ok && (!a.ok || a.cst === f.cst);
if (fiveEq && noStreamEq && outEq) fastOk++;
else {
failures++;
console.log(` ${c.grammar}/${r.label}: fast≢validated fiveEq=${fiveEq} noStreamEq=${noStreamEq} outEq=${outEq} validated=${JSON.stringify(a.align)} fast=${JSON.stringify(f.align)}`);
}
}
console.log(` ${c.grammar}/${r.label}: ${editOk}/${editScenarios.length} edit-sessions ≡ fresh`);
console.log(` ${c.grammar}/${r.label}: ${alignOk}/${editScenarios.length} token-alignments ≡ oracle`);
console.log(` ${c.grammar}/${r.label}: ${fastOk}/${editScenarios.length} fast ≡ validated`);
}
}
}
Expand Down
Loading