perf(kirari): stream JSX, TSX, and JSONC in checkpoint mode

.changeset/kirari-stream-jsx.md

@@ -0,0 +1,7 @@
+---
+"@vortexlabs/kirari": patch
+---
+
+Stream built-in JSX, TSX, and JSONC in checkpoint mode. `createTokenStream(tsx)` now rescans from the last newline outside elements instead of retokenizing all accumulated source on every write. Streaming 8 KB of TSX one character at a time drops from about 13 seconds to about 0.2 seconds. Streamed tokens still equal batch tokenization after every write, and copies or extensions of these definitions keep using full mode. A long element rescans from its opening line until it closes, and a sibling element that has not closed yet keeps the lines after it revisable.
+
+No public API changes.

.changeset/kirari-stream-slash.md

@@ -0,0 +1,5 @@
+---
+"@vortexlabs/kirari": patch
+---
+
+Fix streamed JavaScript and TypeScript names whose function check reads past a slash on the last line. The answer was final even though more input could turn the slash into a regex literal and change it; it now stays revisable until that line ends.

docs/accuracy.md

@@ -70,5 +70,5 @@ not a language-wide accuracy score or superiority over another highlighter.
 
 Streaming patches must equal batch tokenization after every update, including
 its lexical limits. Checkpointing still optimizes only the exact built-in
-JavaScript, TypeScript, JSON, and Python definitions. See [streaming](streaming.md) for
+JavaScript, TypeScript, JSX, TSX, JSON, JSONC, and Python definitions. See [streaming](streaming.md) for
 open-tail and retained-memory limits.

docs/streaming.md

@@ -54,15 +54,18 @@ rules; create a derived definition for custom behavior.
 ## Performance contract
 
 `stream.mode` is `checkpoint` for the exact built-in JavaScript, TypeScript,
-JSON, and Python definitions imported from the same package instance. It is
-`full` for all other definitions, including derived/custom grammars, JSX, TSX,
-and JSONC. Full mode deliberately re-tokenizes all source and returns a patch
+JSX, TSX, JSON, JSONC, and Python definitions imported from the same package
+instance. It is `full` for all other definitions, including derived/custom
+grammars. Full mode deliberately re-tokenizes all source and returns a patch
 from zero. It preserves the same caller API without making unsupported
 performance promises.
 
 Checkpoint mode reuses completed lexical context before a safe newline. It keeps
 enough trailing input to revise name/key lookahead, CRLF, and template state.
-Python checkpoints sit outside f-strings, and function names stay revisable
+JSX and TSX checkpoints sit outside elements, so a long element is rescanned
+from its opening line until it closes, and a sibling element that has not
+closed yet keeps the lines after it revisable. Python checkpoints sit outside
+f-strings, and function names stay revisable
 while later lines can still change them, such as inside an assigned
 parenthesis, inside a bracketed annotation, or when the next line starts with
 `=` or `:`. Each write still

packages/kirari/README.md

@@ -184,8 +184,8 @@ uses `.kirari-line` wrappers and one-based `data-line` attributes. Define your o
 CSS for custom classes; line numbering UI is not added automatically.
 
 Streaming returns `{ from, tokens, length }`: replace your retained tokens from
-the source offset `from` onward. Exact built-in JavaScript, TypeScript, JSON, and
-Python use conservative checkpoints; all other grammars use full re-tokenization.
+the source offset `from` onward. Exact built-in JavaScript, TypeScript, JSX, TSX,
+JSON, JSONC, and Python use conservative checkpoints; all other grammars use full re-tokenization.
 Long lines and unfinished multiline tokens can still require large rescans.
 Streams retain the full source. Do not mutate built-in definitions; copy them
 for customization. Calling `reset()` also requires clearing your retained view.

packages/kirari/package.json

@@ -67,6 +67,7 @@
     "themes": "bun scripts/themes.ts",
     "test": "bun test ./test",
     "test:corpus": "bun test ./test/corpus.test.ts",
+    "test:fuzz": "KIRARI_FUZZ_RUNS=20000 bun test ./test/stream-fuzz.test.ts",
     "typecheck": "tsc --noEmit",
     "build": "tsc -p tsconfig.build.json",
     "size": "bun scripts/size.ts",
@@ -77,5 +78,8 @@
   "packageManager": "bun@1.3.11",
   "publishConfig": {
     "access": "public"
+  },
+  "devDependencies": {
+    "fast-check": "^4.10.2"
   }
 }

packages/kirari/src/grammars/functions.ts

@@ -3,6 +3,10 @@ import type { Token } from '../types.js'
 
 type Result = { function: boolean; pending: boolean }
 type Group = { end: number; close: number; complete: boolean }
+/** Source-wide lookahead data; share only between scanners of the same immutable source. */
+export interface FunctionLookupCache {
+  lastLine?: number
+}
 export interface LookupOptions {
   /** The name follows `const`, `let`, or `var` in TypeScript, so `:` starts a type annotation. */
   readonly annotation: boolean
@@ -22,7 +26,7 @@ export interface LookupOptions {
  * `pending` in the result means the look-ahead reached the end of the source, so a
  * streaming caller must treat the answer as provisional.
  */
-export const createFunctionLookup = (source: string, typed: boolean, scan: (start: number) => Generator<Token, void, number | undefined>) => {
+export const createFunctionLookup = (source: string, typed: boolean, scan: (start: number) => Generator<Token, void, number | undefined>, cache: FunctionLookupCache = {}) => {
   // Reuse nested groups, including unfinished ones, instead of rescanning each signature suffix.
   const groups = new Map<number, Group>()
   return (start: number, { annotation, generic, property, asyncModifier }: LookupOptions): Result => {
@@ -38,6 +42,19 @@ export const createFunctionLookup = (source: string, typed: boolean, scan: (star
       do {
         const value = reader.next()
         token = value.done || value.value.start >= limit ? undefined : value.value
+        if (token?.type === 'operator' && /^\/=?$/.test(token.value)) {
+          if (cache.lastLine === undefined) {
+            let lastLine = source.length - 1
+            while (lastLine >= 0) {
+              const char = source.charCodeAt(lastLine)
+              if (char === 10 || char === 13 || char === 0x2028 || char === 0x2029) break
+              lastLine--
+            }
+            cache.lastLine = lastLine
+          }
+          // A slash on the last line may still become a regex once more source arrives.
+          if (token.start > cache.lastLine) pending = true
+        }
       } while (token && !isSignificant(token.type))
     }
     // Restarts scanning at `position`.

packages/kirari/src/grammars/markup.ts

@@ -1,6 +1,8 @@
-import { appendToken, scan, tokenize } from '../tokenize.js'
+import { appendToken, cloneContext, scan, tokenize } from '../tokenize.js'
+import type { Frame, ScanContext } from '../tokenize.js'
 import { isSignificant } from '../types.js'
 import type { Language, Token } from '../types.js'
+import type { ResumePoint } from '../resumable.js'
 
 type Code = {
   kind: 'code'
@@ -24,24 +26,38 @@ type Markup = {
   sibling?: { start: number; tokens: number; openings: { name: string; start: number }[] }
 }
 
-export const markup = (source: string, code?: Language, embedded?: Readonly<Record<string, Language>>, template?: 'vue' | 'svelte'): Token[] => {
+/** Where a JSX scan starts, and where to report root-level newlines it can resume from. */
+interface MarkupResume {
+  readonly from?: ResumePoint | undefined
+  readonly checkpoint?: ((point: ResumePoint, index: number) => void) | undefined
+}
+
+export const markup = (source: string, code?: Language, embedded?: Readonly<Record<string, Language>>, template?: 'vue' | 'svelte', resume?: MarkupResume): Token[] => {
   const tokens: Token[] = []
   const name = /[$_\p{ID_Start}][\p{ID_Continue}.$:-]*/uy
   const attribute = /[^\s"'<>/={}]+/y
   const entity = /&(?:#x[\da-f]+|#\d+|[a-z][\da-z]*);/iy
   const whitespace = /\s+/y
-  let position = 0
+  let position = resume?.from?.offset ?? 0
   const createMarkup = (bounded: boolean): Markup => ({ kind: 'markup', bounded, depth: 0, phase: 'text', closing: false, name: '', value: false, attribute: '', language: '', rawEnd: undefined })
   const expressionLanguage = code ?? embedded?.['script:ts'] ?? embedded?.script
   const typed = code?.name === 'tsx'
   // Expression scanners use the same source offsets; failed lookahead remains reusable.
   const failedRegexStates = new Set<number>()
+  const lookupCache = {}
   const failedSiblingStarts = new Set<number>()
-  const createCode = (container: boolean): Code => ({ kind: 'code', scanner: scan(source, expressionLanguage!, position, undefined, failedRegexStates), container, braces: 0, resume: undefined, previous: undefined })
-  const stack: (Code | Markup)[] = [code ? createCode(false) : createMarkup(false)]
+  const createCode = (container: boolean, previous?: Token, frames?: Frame[], context?: ScanContext): Code => ({ kind: 'code', scanner: scan(source, expressionLanguage!, position, previous, failedRegexStates, frames, context, true, lookupCache), container, braces: 0, resume: undefined, previous })
+  // Root script state, read at newlines outside elements to report resume points.
+  const rootFrames: Frame[] = []
+  const rootContext = resume?.from?.context ? cloneContext(resume.from.context) : { parentheses: [] }
+  const points: { point: ResumePoint; index: number }[] = []
+  // Rejected tags may become valid once more source arrives, so later resume points are dropped.
+  let rewound = source.length
+  const stack: (Code | Markup)[] = [code ? createCode(false, resume?.from?.previous, rootFrames, rootContext) : createMarkup(false)]
 
   const rejectSibling = (context: Markup): void => {
     const sibling = context.sibling!
+    rewound = Math.min(rewound, sibling.start)
     failedSiblingStarts.add(sibling.start)
     for (const opening of sibling.openings) failedSiblingStarts.add(opening.start)
     tokens.length = sibling.tokens
@@ -111,7 +127,7 @@ export const markup = (source: string, code?: Language, embedded?: Readonly<Reco
   const typeArguments = (): void => {
     let depth = 1
     emit('operator', position + 1)
-    for (const token of scan(source, code!, position, undefined, undefined, undefined, { parentheses: [], annotation: true })) {
+    for (const token of scan(source, code!, position, undefined, undefined, undefined, { parentheses: [], annotation: true }, true, lookupCache)) {
       if (token.type === 'operator' && token.value === '<') depth++
       const closing = token.type === 'operator' && /^>+/.exec(token.value)
       if (closing) {
@@ -143,14 +159,23 @@ export const markup = (source: string, code?: Language, embedded?: Readonly<Reco
     context.resume = undefined
     if (next.done) return false
     const token = next.value
+    if (resume?.checkpoint && stack.length === 1 && token.type === 'text' && /[\r\n\u2028\u2029]/.test(token.value) && token.end < source.length && !rootFrames.length && rootContext.functionPending === undefined) {
+      const last = tokens[tokens.length - 1]
+      const merged = last?.type === 'text'
+      points.push({ point: { offset: merged ? last.start : token.start, previous: context.previous, context: cloneContext(rootContext) }, index: merged ? tokens.length - 1 : tokens.length })
+    }
     const sibling = context.previous?.type === 'tag' && !failedSiblingStarts.has(token.start)
-    if (!template && token.value === '<' && (canStartElement(context.previous) || sibling) && looksLikeTag(source, token.start, typed)) {
-      position = token.start
-      const element = createMarkup(true)
-      // Accept adjacent JSX snippets only when the element closes; otherwise keep comparison operators.
-      if (sibling) element.sibling = { start: position, tokens: tokens.length, openings: [] }
-      stack.push(element)
-      return true
+    if (!template && token.value === '<' && (canStartElement(context.previous) || sibling)) {
+      const tag = looksLikeTag(source, token.start, typed)
+      if (tag === undefined) rewound = Math.min(rewound, token.start)
+      if (tag) {
+        position = token.start
+        const element = createMarkup(true)
+        // Accept adjacent JSX snippets only when the element closes; otherwise keep comparison operators.
+        if (sibling) element.sibling = { start: position, tokens: tokens.length, openings: [] }
+        stack.push(element)
+        return true
+      }
     }
     if (context.container && token.type === 'punctuation') {
       if (token.value === '}' && context.braces === 0 && (template !== 'vue' || source[token.end] === '}')) {
@@ -264,15 +289,19 @@ export const markup = (source: string, code?: Language, embedded?: Readonly<Reco
     } else if (context.phase === 'tag') stepTag(context)
     else stepContent(context)
   }
+  for (const { point, index } of points) if (point.offset < rewound) resume!.checkpoint!(point, index)
   return tokens
 }
 
-const looksLikeTag = (source: string, start: number, typed = false): boolean => {
+const looksLikeTag = (source: string, start: number, typed = false): boolean | undefined => {
   if (source.startsWith('<>', start) || source.startsWith('</>', start)) return true
   const head = /^<\/?[$_\p{ID_Start}][\p{ID_Continue}.$:-]*/u.exec(source.slice(start))
   if (!head) return false
   const rest = source.slice(start + head[0].length)
-  return (/^[\s/>]|^$/.test(rest) || typed && rest.startsWith('<')) && !/^\s+extends\b/.test(rest)
+  const constraint = /^\s+extends\b/.exec(rest)
+  // An EOF word boundary can disappear when the next chunk extends the identifier.
+  if (constraint) return constraint[0].length === rest.length ? undefined : false
+  return /^[\s/>]|^$/.test(rest) || typed && rest.startsWith('<')
 }
 
 const canStartElement = (previous?: Token): boolean => {

packages/kirari/src/languages/jsx.ts

@@ -1,9 +1,15 @@
 import type { Language } from '../types.js'
 import { javascript } from './javascript.js'
 import { markup } from '../grammars/markup.js'
+import { resumableScan } from '../resumable.js'
+import type { ResumableScan } from '../resumable.js'
 
 export const jsx: Language = {
   ...javascript,
   name: 'jsx',
   tokenize: (source, language) => markup(source, language),
 }
+
+const scanJsx: ResumableScan = (source, language, from, checkpoint) => markup(source, language, undefined, undefined, { from, checkpoint })
+
+Object.defineProperty(jsx, resumableScan, { value: scanJsx })

packages/kirari/src/languages/tsx.ts

@@ -1,9 +1,12 @@
 import type { Language } from '../types.js'
 import { typescript } from './typescript.js'
 import { jsx } from './jsx.js'
+import { resumableScan, resumableScanOf } from '../resumable.js'
 
 export const tsx: Language = {
   ...typescript,
   name: 'tsx',
   tokenize: jsx.tokenize!,
 }
+
+Object.defineProperty(tsx, resumableScan, { value: resumableScanOf(jsx) })

packages/kirari/src/resumable.ts

@@ -1,9 +1,12 @@
+import type { ScanContext } from './tokenize.js'
 import type { Language, Token } from './types.js'
 
 /** A root-level point where scanning can restart with the same results as a full scan. */
 export interface ResumePoint {
   readonly offset: number
   readonly previous: Token | undefined
+  /** Script scan state at the point, for grammars that embed JavaScript. */
+  readonly context?: ScanContext
 }
 
 /**

packages/kirari/src/stream.ts

@@ -1,8 +1,9 @@
-import { appendToken, scan, tokenize } from './tokenize.js'
+import { appendToken, cloneContext, scan, tokenize } from './tokenize.js'
 import type { Frame, ScanContext } from './tokenize.js'
 import { javascript } from './languages/javascript.js'
 import { typescript } from './languages/typescript.js'
 import { json } from './languages/json.js'
+import { jsonc } from './languages/jsonc.js'
 import { resumableScanOf } from './resumable.js'
 import type { ResumableScan, ResumePoint } from './resumable.js'
 import { advanceDefinitionState, breakDefinitionLine, classifyDefinitions, copyDefinitionState, createDefinitionState, endsLine, isDefinitionBoundary } from './grammars/definitions.js'
@@ -24,10 +25,6 @@ export interface TokenStream {
   reset(): void
 }
 
-const cloneContext = (context: ScanContext): ScanContext => {
-  return { ...context, parentheses: [...context.parentheses], scopes: [...context.scopes ?? []] }
-}
-
 const openBrackets = (state: DefinitionState): string[] => state.brackets.map(bracket => bracket.close)
 
 /** Advances `state` through `tokens[from..to]`, returning the next index. */
@@ -41,10 +38,12 @@ const walkDefinitions = (state: DefinitionState, tokens: readonly Token[], from:
 }
 
 /**
- * Streams for built-in Python rescan from the last resume point, then reclassify definitions from
- * the last point where `classifyDefinitions` cannot see across. Tokens before that point are final.
+ * Streams for built-in resumable grammars rescan from the last resume point. Python then
+ * reclassifies definitions from the last point where `classifyDefinitions` cannot see across.
+ * Tokens before that point are final.
  */
 const createResumableStream = (language: Language, scanResumable: ResumableScan): TokenStream => {
+  const classified = language.name === 'python'
   let source = ''
   let raw: Token[] = []
   let resume: ResumePoint | undefined
@@ -63,6 +62,11 @@ const createResumableStream = (language: Language, scanResumable: ResumableScan)
     const found: { resume: ResumePoint; index: number }[] = []
     const tail = scanResumable(next, language, resume, (point, index) => found.push({ resume: point, index: resumeIndex + index }))
     source = next
+    if (!classified) {
+      const from = resume?.offset ?? 0
+      resume = found.at(-1)?.resume ?? resume
+      return { from, tokens: tail, length: source.length }
+    }
     raw.length = resumeIndex
     for (const token of tail) raw.push(token)
     points = points.filter(point => point.index < resumeIndex)
@@ -119,7 +123,7 @@ export const createTokenStream = (language: Language): TokenStream => {
   const scanResumable = resumableScanOf(language)
   if (scanResumable) return createResumableStream(language, scanResumable)
   // shortcut: long/open tails are rescanned; track more lexical state if those workloads become a bottleneck.
-  const mode = !language.tokenize && (language === javascript || language === typescript || language === json) ? 'checkpoint' : 'full'
+  const mode = !language.tokenize && (language === javascript || language === typescript || language === json || language === jsonc) ? 'checkpoint' : 'full'
   let source = ''
   let offset = 0
   let previous: Token | undefined