diff --git a/.changeset/kirari-stream-jsx.md b/.changeset/kirari-stream-jsx.md new file mode 100644 index 0000000..381fd0a --- /dev/null +++ b/.changeset/kirari-stream-jsx.md @@ -0,0 +1,7 @@ +--- +"@vortexlabs/kirari": patch +--- + +Stream built-in JSX, TSX, and JSONC in checkpoint mode. `createTokenStream(tsx)` now rescans from the last newline outside elements instead of retokenizing all accumulated source on every write. Streaming 8 KB of TSX one character at a time drops from about 13 seconds to about 0.2 seconds. Streamed tokens still equal batch tokenization after every write, and copies or extensions of these definitions keep using full mode. A long element rescans from its opening line until it closes, and a sibling element that has not closed yet keeps the lines after it revisable. + +No public API changes. diff --git a/.changeset/kirari-stream-slash.md b/.changeset/kirari-stream-slash.md new file mode 100644 index 0000000..a650a88 --- /dev/null +++ b/.changeset/kirari-stream-slash.md @@ -0,0 +1,5 @@ +--- +"@vortexlabs/kirari": patch +--- + +Fix streamed JavaScript and TypeScript names whose function check reads past a slash on the last line. The answer was final even though more input could turn the slash into a regex literal and change it; it now stays revisable until that line ends. diff --git a/bun.lock b/bun.lock index 4d6cca2..f575ba5 100644 --- a/bun.lock +++ b/bun.lock @@ -20,6 +20,9 @@ "packages/kirari": { "name": "@vortexlabs/kirari", "version": "1.0.4", + "devDependencies": { + "fast-check": "^4.10.2", + }, }, "packages/react": { "name": "@vortexlabs/kirari-react", @@ -384,6 +387,8 @@ "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + "fast-check": ["fast-check@4.10.2", "", { "dependencies": { "pure-rand": "^8.0.0" } }, "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw=="], + "fast-string-truncated-width": ["fast-string-truncated-width@3.0.3", "", {}, "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g=="], "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], @@ -558,6 +563,8 @@ "property-information": ["property-information@7.2.0", "", {}, "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg=="], + "pure-rand": ["pure-rand@8.4.2", "", {}, "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng=="], + "react": ["react@19.3.1", "", {}, "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw=="], "react-dom": ["react-dom@19.3.1", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.1" } }, "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ=="], diff --git a/docs/accuracy.md b/docs/accuracy.md index 20d0510..a428356 100644 --- a/docs/accuracy.md +++ b/docs/accuracy.md @@ -70,5 +70,5 @@ not a language-wide accuracy score or superiority over another highlighter. Streaming patches must equal batch tokenization after every update, including its lexical limits. Checkpointing still optimizes only the exact built-in -JavaScript, TypeScript, JSON, and Python definitions. See [streaming](streaming.md) for +JavaScript, TypeScript, JSX, TSX, JSON, JSONC, and Python definitions. See [streaming](streaming.md) for open-tail and retained-memory limits. diff --git a/docs/streaming.md b/docs/streaming.md index 90bcf65..0533809 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -54,15 +54,18 @@ rules; create a derived definition for custom behavior. ## Performance contract `stream.mode` is `checkpoint` for the exact built-in JavaScript, TypeScript, -JSON, and Python definitions imported from the same package instance. It is -`full` for all other definitions, including derived/custom grammars, JSX, TSX, -and JSONC. Full mode deliberately re-tokenizes all source and returns a patch +JSX, TSX, JSON, JSONC, and Python definitions imported from the same package +instance. It is `full` for all other definitions, including derived/custom +grammars. Full mode deliberately re-tokenizes all source and returns a patch from zero. It preserves the same caller API without making unsupported performance promises. Checkpoint mode reuses completed lexical context before a safe newline. It keeps enough trailing input to revise name/key lookahead, CRLF, and template state. -Python checkpoints sit outside f-strings, and function names stay revisable +JSX and TSX checkpoints sit outside elements, so a long element is rescanned +from its opening line until it closes, and a sibling element that has not +closed yet keeps the lines after it revisable. Python checkpoints sit outside +f-strings, and function names stay revisable while later lines can still change them, such as inside an assigned parenthesis, inside a bracketed annotation, or when the next line starts with `=` or `:`. Each write still diff --git a/packages/kirari/README.md b/packages/kirari/README.md index e708498..df4e7ab 100644 --- a/packages/kirari/README.md +++ b/packages/kirari/README.md @@ -184,8 +184,8 @@ uses `.kirari-line` wrappers and one-based `data-line` attributes. Define your o CSS for custom classes; line numbering UI is not added automatically. Streaming returns `{ from, tokens, length }`: replace your retained tokens from -the source offset `from` onward. Exact built-in JavaScript, TypeScript, JSON, and -Python use conservative checkpoints; all other grammars use full re-tokenization. +the source offset `from` onward. Exact built-in JavaScript, TypeScript, JSX, TSX, +JSON, JSONC, and Python use conservative checkpoints; all other grammars use full re-tokenization. Long lines and unfinished multiline tokens can still require large rescans. Streams retain the full source. Do not mutate built-in definitions; copy them for customization. Calling `reset()` also requires clearing your retained view. diff --git a/packages/kirari/package.json b/packages/kirari/package.json index 97d3992..7823f48 100644 --- a/packages/kirari/package.json +++ b/packages/kirari/package.json @@ -67,6 +67,7 @@ "themes": "bun scripts/themes.ts", "test": "bun test ./test", "test:corpus": "bun test ./test/corpus.test.ts", + "test:fuzz": "KIRARI_FUZZ_RUNS=20000 bun test ./test/stream-fuzz.test.ts", "typecheck": "tsc --noEmit", "build": "tsc -p tsconfig.build.json", "size": "bun scripts/size.ts", @@ -77,5 +78,8 @@ "packageManager": "bun@1.3.11", "publishConfig": { "access": "public" + }, + "devDependencies": { + "fast-check": "^4.10.2" } } diff --git a/packages/kirari/src/grammars/functions.ts b/packages/kirari/src/grammars/functions.ts index 5e22e63..e57f11f 100644 --- a/packages/kirari/src/grammars/functions.ts +++ b/packages/kirari/src/grammars/functions.ts @@ -3,6 +3,10 @@ import type { Token } from '../types.js' type Result = { function: boolean; pending: boolean } type Group = { end: number; close: number; complete: boolean } +/** Source-wide lookahead data; share only between scanners of the same immutable source. */ +export interface FunctionLookupCache { + lastLine?: number +} export interface LookupOptions { /** The name follows `const`, `let`, or `var` in TypeScript, so `:` starts a type annotation. */ readonly annotation: boolean @@ -22,7 +26,7 @@ export interface LookupOptions { * `pending` in the result means the look-ahead reached the end of the source, so a * streaming caller must treat the answer as provisional. */ -export const createFunctionLookup = (source: string, typed: boolean, scan: (start: number) => Generator) => { +export const createFunctionLookup = (source: string, typed: boolean, scan: (start: number) => Generator, cache: FunctionLookupCache = {}) => { // Reuse nested groups, including unfinished ones, instead of rescanning each signature suffix. const groups = new Map() return (start: number, { annotation, generic, property, asyncModifier }: LookupOptions): Result => { @@ -38,6 +42,19 @@ export const createFunctionLookup = (source: string, typed: boolean, scan: (star do { const value = reader.next() token = value.done || value.value.start >= limit ? undefined : value.value + if (token?.type === 'operator' && /^\/=?$/.test(token.value)) { + if (cache.lastLine === undefined) { + let lastLine = source.length - 1 + while (lastLine >= 0) { + const char = source.charCodeAt(lastLine) + if (char === 10 || char === 13 || char === 0x2028 || char === 0x2029) break + lastLine-- + } + cache.lastLine = lastLine + } + // A slash on the last line may still become a regex once more source arrives. + if (token.start > cache.lastLine) pending = true + } } while (token && !isSignificant(token.type)) } // Restarts scanning at `position`. diff --git a/packages/kirari/src/grammars/markup.ts b/packages/kirari/src/grammars/markup.ts index 2b180d0..f79e46e 100644 --- a/packages/kirari/src/grammars/markup.ts +++ b/packages/kirari/src/grammars/markup.ts @@ -1,6 +1,8 @@ -import { appendToken, scan, tokenize } from '../tokenize.js' +import { appendToken, cloneContext, scan, tokenize } from '../tokenize.js' +import type { Frame, ScanContext } from '../tokenize.js' import { isSignificant } from '../types.js' import type { Language, Token } from '../types.js' +import type { ResumePoint } from '../resumable.js' type Code = { kind: 'code' @@ -24,24 +26,38 @@ type Markup = { sibling?: { start: number; tokens: number; openings: { name: string; start: number }[] } } -export const markup = (source: string, code?: Language, embedded?: Readonly>, template?: 'vue' | 'svelte'): Token[] => { +/** Where a JSX scan starts, and where to report root-level newlines it can resume from. */ +interface MarkupResume { + readonly from?: ResumePoint | undefined + readonly checkpoint?: ((point: ResumePoint, index: number) => void) | undefined +} + +export const markup = (source: string, code?: Language, embedded?: Readonly>, template?: 'vue' | 'svelte', resume?: MarkupResume): Token[] => { const tokens: Token[] = [] const name = /[$_\p{ID_Start}][\p{ID_Continue}.$:-]*/uy const attribute = /[^\s"'<>/={}]+/y const entity = /&(?:#x[\da-f]+|#\d+|[a-z][\da-z]*);/iy const whitespace = /\s+/y - let position = 0 + let position = resume?.from?.offset ?? 0 const createMarkup = (bounded: boolean): Markup => ({ kind: 'markup', bounded, depth: 0, phase: 'text', closing: false, name: '', value: false, attribute: '', language: '', rawEnd: undefined }) const expressionLanguage = code ?? embedded?.['script:ts'] ?? embedded?.script const typed = code?.name === 'tsx' // Expression scanners use the same source offsets; failed lookahead remains reusable. const failedRegexStates = new Set() + const lookupCache = {} const failedSiblingStarts = new Set() - const createCode = (container: boolean): Code => ({ kind: 'code', scanner: scan(source, expressionLanguage!, position, undefined, failedRegexStates), container, braces: 0, resume: undefined, previous: undefined }) - const stack: (Code | Markup)[] = [code ? createCode(false) : createMarkup(false)] + const createCode = (container: boolean, previous?: Token, frames?: Frame[], context?: ScanContext): Code => ({ kind: 'code', scanner: scan(source, expressionLanguage!, position, previous, failedRegexStates, frames, context, true, lookupCache), container, braces: 0, resume: undefined, previous }) + // Root script state, read at newlines outside elements to report resume points. + const rootFrames: Frame[] = [] + const rootContext = resume?.from?.context ? cloneContext(resume.from.context) : { parentheses: [] } + const points: { point: ResumePoint; index: number }[] = [] + // Rejected tags may become valid once more source arrives, so later resume points are dropped. + let rewound = source.length + const stack: (Code | Markup)[] = [code ? createCode(false, resume?.from?.previous, rootFrames, rootContext) : createMarkup(false)] const rejectSibling = (context: Markup): void => { const sibling = context.sibling! + rewound = Math.min(rewound, sibling.start) failedSiblingStarts.add(sibling.start) for (const opening of sibling.openings) failedSiblingStarts.add(opening.start) tokens.length = sibling.tokens @@ -111,7 +127,7 @@ export const markup = (source: string, code?: Language, embedded?: Readonly { let depth = 1 emit('operator', position + 1) - for (const token of scan(source, code!, position, undefined, undefined, undefined, { parentheses: [], annotation: true })) { + for (const token of scan(source, code!, position, undefined, undefined, undefined, { parentheses: [], annotation: true }, true, lookupCache)) { if (token.type === 'operator' && token.value === '<') depth++ const closing = token.type === 'operator' && /^>+/.exec(token.value) if (closing) { @@ -143,14 +159,23 @@ export const markup = (source: string, code?: Language, embedded?: Readonly { +const looksLikeTag = (source: string, start: number, typed = false): boolean | undefined => { if (source.startsWith('<>', start) || source.startsWith('', start)) return true const head = /^<\/?[$_\p{ID_Start}][\p{ID_Continue}.$:-]*/u.exec(source.slice(start)) if (!head) return false const rest = source.slice(start + head[0].length) - return (/^[\s/>]|^$/.test(rest) || typed && rest.startsWith('<')) && !/^\s+extends\b/.test(rest) + const constraint = /^\s+extends\b/.exec(rest) + // An EOF word boundary can disappear when the next chunk extends the identifier. + if (constraint) return constraint[0].length === rest.length ? undefined : false + return /^[\s/>]|^$/.test(rest) || typed && rest.startsWith('<') } const canStartElement = (previous?: Token): boolean => { diff --git a/packages/kirari/src/languages/jsx.ts b/packages/kirari/src/languages/jsx.ts index 36467aa..7857a70 100644 --- a/packages/kirari/src/languages/jsx.ts +++ b/packages/kirari/src/languages/jsx.ts @@ -1,9 +1,15 @@ import type { Language } from '../types.js' import { javascript } from './javascript.js' import { markup } from '../grammars/markup.js' +import { resumableScan } from '../resumable.js' +import type { ResumableScan } from '../resumable.js' export const jsx: Language = { ...javascript, name: 'jsx', tokenize: (source, language) => markup(source, language), } + +const scanJsx: ResumableScan = (source, language, from, checkpoint) => markup(source, language, undefined, undefined, { from, checkpoint }) + +Object.defineProperty(jsx, resumableScan, { value: scanJsx }) diff --git a/packages/kirari/src/languages/tsx.ts b/packages/kirari/src/languages/tsx.ts index 4e38c45..e25b304 100644 --- a/packages/kirari/src/languages/tsx.ts +++ b/packages/kirari/src/languages/tsx.ts @@ -1,9 +1,12 @@ import type { Language } from '../types.js' import { typescript } from './typescript.js' import { jsx } from './jsx.js' +import { resumableScan, resumableScanOf } from '../resumable.js' export const tsx: Language = { ...typescript, name: 'tsx', tokenize: jsx.tokenize!, } + +Object.defineProperty(tsx, resumableScan, { value: resumableScanOf(jsx) }) diff --git a/packages/kirari/src/resumable.ts b/packages/kirari/src/resumable.ts index 01c6cf0..9699e2e 100644 --- a/packages/kirari/src/resumable.ts +++ b/packages/kirari/src/resumable.ts @@ -1,9 +1,12 @@ +import type { ScanContext } from './tokenize.js' import type { Language, Token } from './types.js' /** A root-level point where scanning can restart with the same results as a full scan. */ export interface ResumePoint { readonly offset: number readonly previous: Token | undefined + /** Script scan state at the point, for grammars that embed JavaScript. */ + readonly context?: ScanContext } /** diff --git a/packages/kirari/src/stream.ts b/packages/kirari/src/stream.ts index 85ea3c4..6da83e0 100644 --- a/packages/kirari/src/stream.ts +++ b/packages/kirari/src/stream.ts @@ -1,8 +1,9 @@ -import { appendToken, scan, tokenize } from './tokenize.js' +import { appendToken, cloneContext, scan, tokenize } from './tokenize.js' import type { Frame, ScanContext } from './tokenize.js' import { javascript } from './languages/javascript.js' import { typescript } from './languages/typescript.js' import { json } from './languages/json.js' +import { jsonc } from './languages/jsonc.js' import { resumableScanOf } from './resumable.js' import type { ResumableScan, ResumePoint } from './resumable.js' import { advanceDefinitionState, breakDefinitionLine, classifyDefinitions, copyDefinitionState, createDefinitionState, endsLine, isDefinitionBoundary } from './grammars/definitions.js' @@ -24,10 +25,6 @@ export interface TokenStream { reset(): void } -const cloneContext = (context: ScanContext): ScanContext => { - return { ...context, parentheses: [...context.parentheses], scopes: [...context.scopes ?? []] } -} - const openBrackets = (state: DefinitionState): string[] => state.brackets.map(bracket => bracket.close) /** Advances `state` through `tokens[from..to]`, returning the next index. */ @@ -41,10 +38,12 @@ const walkDefinitions = (state: DefinitionState, tokens: readonly Token[], from: } /** - * Streams for built-in Python rescan from the last resume point, then reclassify definitions from - * the last point where `classifyDefinitions` cannot see across. Tokens before that point are final. + * Streams for built-in resumable grammars rescan from the last resume point. Python then + * reclassifies definitions from the last point where `classifyDefinitions` cannot see across. + * Tokens before that point are final. */ const createResumableStream = (language: Language, scanResumable: ResumableScan): TokenStream => { + const classified = language.name === 'python' let source = '' let raw: Token[] = [] let resume: ResumePoint | undefined @@ -63,6 +62,11 @@ const createResumableStream = (language: Language, scanResumable: ResumableScan) const found: { resume: ResumePoint; index: number }[] = [] const tail = scanResumable(next, language, resume, (point, index) => found.push({ resume: point, index: resumeIndex + index })) source = next + if (!classified) { + const from = resume?.offset ?? 0 + resume = found.at(-1)?.resume ?? resume + return { from, tokens: tail, length: source.length } + } raw.length = resumeIndex for (const token of tail) raw.push(token) points = points.filter(point => point.index < resumeIndex) @@ -119,7 +123,7 @@ export const createTokenStream = (language: Language): TokenStream => { const scanResumable = resumableScanOf(language) if (scanResumable) return createResumableStream(language, scanResumable) // shortcut: long/open tails are rescanned; track more lexical state if those workloads become a bottleneck. - const mode = !language.tokenize && (language === javascript || language === typescript || language === json) ? 'checkpoint' : 'full' + const mode = !language.tokenize && (language === javascript || language === typescript || language === json || language === jsonc) ? 'checkpoint' : 'full' let source = '' let offset = 0 let previous: Token | undefined diff --git a/packages/kirari/src/tokenize.ts b/packages/kirari/src/tokenize.ts index 21c068f..7287695 100644 --- a/packages/kirari/src/tokenize.ts +++ b/packages/kirari/src/tokenize.ts @@ -17,6 +17,7 @@ import { isSignificant } from './types.js' import type { Language, Token } from './types.js' import { createFunctionLookup } from './grammars/functions.js' +import type { FunctionLookupCache } from './grammars/functions.js' import { classifyDefinitions } from './grammars/definitions.js' export type Frame = { kind: 'template'; opening: boolean } | { kind: 'expression'; depth: number } @@ -35,6 +36,10 @@ export interface ScanContext { scopes?: boolean[] } +export const cloneContext = (context: ScanContext): ScanContext => { + return { ...context, parentheses: [...context.parentheses], scopes: [...context.scopes ?? []] } +} + const sticky = (pattern: RegExp): RegExp => { return new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, '') + 'y') } @@ -80,7 +85,7 @@ export const appendToken = (tokens: Token[], token: Token): void => { * and `functions` enables look-ahead function detection (disabled inside that look-ahead). */ // biome-ignore lint/plugin: generators have no arrow form. -export function* scan(source: string, language: Language, start = 0, previous?: Token, failedRegexStates = new Set(), frames: Frame[] = [], context: ScanContext = { parentheses: [] }, functions = true): Generator { +export function* scan(source: string, language: Language, start = 0, previous?: Token, failedRegexStates = new Set(), frames: Frame[] = [], context: ScanContext = { parentheses: [] }, functions = true, lookupCache: FunctionLookupCache = {}): Generator { const rules = language.rules.map(rule => ({ type: rule.type, pattern: sticky(rule.pattern) })) const identifier = language.identifier && sticky(language.identifier) const keywords = new Set(language.keywords) @@ -89,7 +94,7 @@ export function* scan(source: string, language: Language, start = 0, previous?: const nulls = new Set(language.nulls) const script = language.regexLiterals && language.templates const typed = language.name === 'typescript' || language.name === 'tsx' - const lookup = functions && script ? createFunctionLookup(source, typed, offset => scan(source, language, offset, undefined, failedRegexStates, [], { parentheses: [] }, false)) : undefined + const lookup = functions && script ? createFunctionLookup(source, typed, offset => scan(source, language, offset, undefined, failedRegexStates, [], { parentheses: [] }, false), lookupCache) : undefined delete context.functionPending let pending: Token[] = [] let position = start diff --git a/packages/kirari/test/language-performance.test.ts b/packages/kirari/test/language-performance.test.ts index c12ed16..56a64f6 100644 --- a/packages/kirari/test/language-performance.test.ts +++ b/packages/kirari/test/language-performance.test.ts @@ -5,12 +5,28 @@ import { csharp } from '../src/languages/csharp.js' import { php } from '../src/languages/php.js' import { css } from '../src/languages/css.js' import { javascript } from '../src/languages/javascript.js' +import { jsx } from '../src/languages/jsx.js' import { makefile } from '../src/languages/makefile.js' import { toml } from '../src/languages/toml.js' import { yaml } from '../src/languages/yaml.js' import { createMarkdown } from '../src/markdown.js' import { createRegistry } from '../src/registry.js' +test('JSX expression lookups do not repeatedly search the full source for absent line separators', () => { + const source = '
' + '{x}\n'.repeat(64_000) + '
' + const start = performance.now() + expect(tokenize(source, jsx).map(token => token.value).join('')).toBe(source) + // Leave room for shared CI runners while catching repeated whole-source searches. + expect(performance.now() - start).toBeLessThan(5000) +}, 10_000) + +test('JSX division lookups share the last-line search across expression scanners', () => { + const source = '
' + '{x / 2}\n'.repeat(16_000) + 'a'.repeat(256_000) + '
' + const start = performance.now() + expect(tokenize(source, jsx).map(token => token.value).join('')).toBe(source) + expect(performance.now() - start).toBeLessThan(5000) +}, 10_000) + test('unfinished raw string prefixes do not repeatedly scan the entire prefix run', () => { const start = performance.now() for (const [language, prefix] of [[swift, '#'], [csharp, '$'], [php, '$']] as const) { diff --git a/packages/kirari/test/markup.test.ts b/packages/kirari/test/markup.test.ts index fc41e90..b60ad09 100644 --- a/packages/kirari/test/markup.test.ts +++ b/packages/kirari/test/markup.test.ts @@ -5,6 +5,7 @@ import { tsx } from '../src/languages/tsx.js' import { html } from '../src/languages/html.js' import { xml } from '../src/languages/xml.js' import { createTokenStream } from '../src/stream.js' +import type { Token } from '../src/types.js' const pairs = (source: string, language = jsx) => tokenize(source, language).map(t => [t.type, t.value]) @@ -60,10 +61,12 @@ for (const language of [jsx, tsx]) { test(`${language.name} updates sibling highlighting as streamed elements close`, () => { const source = '\n{value}\n' const stream = createTokenStream(language) + let tokens: Token[] = [] for (let end = 1; end <= source.length; end++) { const patch = stream.write(source[end - 1]!) - expect(patch.from).toBe(0) - expect(patch.tokens).toEqual(tokenize(source.slice(0, end), language)) + expect(patch.from === 0 || tokens.some(token => token.end === patch.from)).toBe(true) + tokens = [...tokens.filter(token => token.start < patch.from), ...patch.tokens] + expect(tokens).toEqual(tokenize(source.slice(0, end), language)) } }) } diff --git a/packages/kirari/test/stream-fuzz.test.ts b/packages/kirari/test/stream-fuzz.test.ts index fad4df2..b4fe258 100644 --- a/packages/kirari/test/stream-fuzz.test.ts +++ b/packages/kirari/test/stream-fuzz.test.ts @@ -1,28 +1,24 @@ import { expect, test } from 'bun:test' +import fc from 'fast-check' import { createTokenStream } from '../src/stream.js' import { tokenize } from '../src/tokenize.js' import { javascript } from '../src/languages/javascript.js' import { typescript } from '../src/languages/typescript.js' import { json } from '../src/languages/json.js' +import { jsonc } from '../src/languages/jsonc.js' +import { jsx } from '../src/languages/jsx.js' +import { tsx } from '../src/languages/tsx.js' import { python } from '../src/languages/python.js' import type { Language, Token } from '../src/types.js' -const random = (seed: number) => { - return () => { - seed ^= seed << 13 - seed ^= seed >>> 17 - seed ^= seed << 5 - return seed >>> 0 - } -} - -const checkPartitions = (source: string, language: Language, nextSize: () => number) => { +/** Streams `source` in chunks of `sizes`, cycled, and expects batch tokens after every write. */ +const checkPartitions = (source: string, language: Language, sizes: readonly number[]) => { const stream = createTokenStream(language) let tokens: readonly Token[] = [] let end = 0 const chunks: string[] = [] while (end < source.length) { - const chunk = source.slice(end, end + nextSize()) + const chunk = source.slice(end, end + sizes[chunks.length % sizes.length]!) chunks.push(chunk) end += chunk.length const patch = stream.write(chunk) @@ -37,15 +33,31 @@ const checkPartitions = (source: string, language: Language, nextSize: () => num expect(tokens).toEqual(tokenize(source, language)) } -const fragments = [ +// `bun run test:fuzz` raises the run count to search for rare mismatches while adding a language. +const runs = Number(process.env.KIRARI_FUZZ_RUNS ?? 300) +const timeout = Math.max(30_000, runs * 20) + +/** + * Checks generated sources and chunk sizes. A failure shrinks to the fewest fragments and the + * smallest chunks, and prints the seed and path that replay it through `fc.assert` options. + */ +const checkStreams = (language: Language, fragments: readonly string[], sources: readonly string[]) => { + const source = fc.array(fc.constantFrom(...fragments), { maxLength: 30, size: 'medium' }).map(parts => parts.join('')) + // Single UTF-16 unit chunks visit every prefix, including split surrogate pairs. + const sizes = fc.oneof(fc.constant([1]), fc.array(fc.integer({ min: 1, max: 11 }), { minLength: 1, maxLength: 8 })) + const examples = sources.flatMap(text => [[text, [1]], [text, [3, 1, 7]]] as [string, number[]][]) + fc.assert(fc.property(source, sizes, (text, chunks) => checkPartitions(text, language, chunks)), { numRuns: runs, examples }) +} + +const scriptFragments = [ 'name', 'class', 'return', 'true', 'null', 'number', '"key"', '0', 'e+', '_', '/', '/[', ']', '//', '/*', '*/', '\\', '"', "'", '`', '${', '{', '}', '(', ')', ':', '.', '?.', ';', - ' ', '\t', '\n', '\r', '\r\n', '\u2028', '\u2029', + ' ', '\t', '\n', '\r', '\r\n', '
', '
', '\ud801', '\udc00', '🙂', '𐐀', '<&>', ] -const sources = [ +const scriptSources = [ 'name\n\t(\nname\n)', '{name\n\t: true, "key"\r\n: null}', 'return /[a\\/]*/g\nname()', @@ -53,25 +65,14 @@ const sources = [ '/*\n*/ name\n//\n(\n)', '`head\n${`nested\n${name\n()}`}\ntail`\nnext()', '"escaped\\\r\nline"\nnext()', - '\ud801\n\udc00\n𐐀\n()\n🙂\n\ud801\udc00', - 'true\nfalse\rnull\u2028name\u2029()', + '\ud801\n\udc00\n𐐀\n()\n🙂\n𐐀', + 'true\nfalse\rnull
name
()', ] -const pick = random(0x1a2b3c4d) -for (let index = 0; index < 160; index++) { - let source = '' - for (let part = 0; part < 12; part++) source += fragments[pick() % fragments.length]! - sources.push(source) -} -for (const language of [javascript, typescript, json]) { +for (const language of [javascript, typescript, json, jsonc]) { test(`stream patches match batch tokens for generated prefixes and partitions: ${language.name}`, () => { - const partition = random(0x56789abc) - for (const source of sources) { - // UTF-16 unit chunks visit every prefix, including split surrogate pairs. - checkPartitions(source, language, () => 1) - checkPartitions(source, language, () => 1 + partition() % 11) - } - }) + checkStreams(language, scriptFragments, scriptSources) + }, timeout) } const pythonFragments = [ @@ -79,25 +80,36 @@ const pythonFragments = [ 'name: T', 'x: int', 'y: Callable[[int], str]', '= lambda: 0', 'x = (', 'lambda: 1)', 'def a.b', ':', '=', ':=', '(', ')', '[', ']', '{', '}', ',', '.', '*', '**', '&', '<', '>', '>>', '->', '@dec', '1', '.5', '"s"', '"""', "'''", 'f"', "f'", 'f"""', '{a}', 'rb"', '#c', '\\', - '\n', '\n', '\r\n', '\r', ' ', + ' ', ' ', ' ', '\n', '\n', '\r\n', '\r', ' ', ] const pythonSources = [ 'handler: int if a < b else str = lambda: 0\nif x > 2:\n pass\n', 'value = f"{a\n}" + f\'\'\'\n{b}\n\'\'\'\nnext()\r\n', ] -const pickPython = random(0x2468ace0) -for (let index = 0; index < 400; index++) { - let source = '' - const count = 4 + pickPython() % 24 - for (let part = 0; part < count; part++) source += pythonFragments[pickPython() % pythonFragments.length]! + (pickPython() % 3 ? ' ' : '') - pythonSources.push(source) -} test('stream patches match batch tokens for generated prefixes and partitions: python', () => { - const partition = random(0x13579bdf) - for (const source of pythonSources) { - checkPartitions(source, python, () => 1) - checkPartitions(source, python, () => 1 + partition() % 6) - } -}) + checkStreams(python, pythonFragments, pythonSources) +}, timeout) + +const markupFragments = [ + 'return', 'const x =', '(', ')', '{', '}', '[', ']', ',', ';', ':', '=>', '?', 'a < b', 'a > b', '<', '>', '/', + '', '', '', '', '<>', '', '/>', '', '', 'className="x"', "id='y'", + '{name}', '{`t${a}`}', 'text', '&', '', '// c', '/* c */', '"s"', '`', '${', 'name()', 'function f', + ' ', ' ', ' ', '\n', '\n', '\r\n', ' ', +] + +const markupSources = [ + 'const a =
\n {items.map(item =>\n )}\n
\n\nnext()\n', + 'return (\n <>\n x\n y\n \n)\nconst t = a < b\n', + 'const a = x\n\ny\n\nnext()\n', + 'if (a)\n/re/.test(b)\nclass A {\nmethod\n() {}\n}\n', + '<>', + '[{a\n(1)', +] + +for (const language of [jsx, tsx]) { + test(`stream patches match batch tokens for generated prefixes and partitions: ${language.name}`, () => { + checkStreams(language, markupFragments, markupSources) + }, timeout) +} diff --git a/packages/kirari/test/stream.test.ts b/packages/kirari/test/stream.test.ts index 11d4082..f2a16ff 100644 --- a/packages/kirari/test/stream.test.ts +++ b/packages/kirari/test/stream.test.ts @@ -5,6 +5,8 @@ import { render } from '../src/render.js' import { javascript } from '../src/languages/javascript.js' import { typescript } from '../src/languages/typescript.js' import { json } from '../src/languages/json.js' +import { jsonc } from '../src/languages/jsonc.js' +import { jsx } from '../src/languages/jsx.js' import { tsx } from '../src/languages/tsx.js' import { python } from '../src/languages/python.js' import { shell } from '../src/languages/shell.js' @@ -129,6 +131,58 @@ test('Python line checkpoints bound rescanning', () => { expect(createTokenStream({ ...python }).mode).toBe('full') }) +test('JSX, TSX, and JSONC streams checkpoint at root lines and copies use full mode', () => { + const component = 'export function Item({ name }: { name: string }) {\n return (\n
  • open(name)}>\n {name}\n
    • \n )\n}\n' + for (const [language, block] of [[jsx, component.replace(': { name: string }', '')], [tsx, component], [tsx, 'const identity = (value: T) => value;\n'], [jsonc, '{\n // note\n "key": [1, 2],\n}\n']] as const) { + const source = block.repeat(100) + const stream = createTokenStream(language) + expect(stream.mode).toBe('checkpoint') + expect(createTokenStream({ ...language }).mode).toBe('full') + let work = 0 + for (let start = 0; start < source.length; start += 4) { + const patch = stream.write(source.slice(start, start + 4)) + work += patch.length - patch.from + } + expect(work).toBeLessThan(source.length * 60) + } +}) + +for (const language of [jsx, tsx]) { + test(`${language.name} streams keep provisional extends tag rejections revisable`, () => { + for (const newline of ['\n', '\r\n', '\r', '\u2028', '\u2029']) { + for (const prefix of ['', 'const before = 1;\nconst view = ', '\n']) { + const source = `${prefix};\nnext()\n` + for (let split = 1; split < source.length; split++) { + const stream = createTokenStream(language) + const tokens: Token[] = [] + apply(tokens, stream.write(source.slice(0, split))) + expect(tokens).toEqual(tokenize(source.slice(0, split), language)) + for (let index = split; index < source.length; index++) { + apply(tokens, stream.write(source[index]!)) + expect(tokens).toEqual(tokenize(source.slice(0, index + 1), language)) + } + apply(tokens, stream.end()) + expect(tokens.filter(token => token.type === 'attribute').map(token => token.value)).toEqual(['extendsValue']) + } + } + } + }) +} + +test('script lookahead stays revisable while a slash on the last line may become a regex', () => { + for (const language of [typescript, tsx]) { + for (const newline of ['', '\n', '\r\n', '\r', '\u2028', '\u2029']) { + const source = `[{a${newline}(1)` + const stream = createTokenStream(language) + const tokens: Token[] = [] + for (let index = 0; index < source.length; index++) { + apply(tokens, stream.write(source[index]!)) + expect(tokens).toEqual(tokenize(source.slice(0, index + 1), language)) + } + } + } +}) + test('line checkpoints bound rescanning for ordinary multiline code', () => { const source = 'const value = 42;\n'.repeat(1000) const stream = createTokenStream(javascript) diff --git a/packages/react/README.md b/packages/react/README.md index 6d3296a..7c7ec1c 100644 --- a/packages/react/README.md +++ b/packages/react/README.md @@ -84,7 +84,7 @@ export const StreamExample = () => { This example adds one chunk per click. In your app, call `write(chunk)` as text arrives. The hook also returns `end()`, which closes the stream when it is complete. -- Built-in JavaScript, TypeScript, JSON, and Python reuse completed scanning work. Other languages rescan the full source. Long unfinished tokens can still be expensive; see [token streaming](https://github.com/vortexkit/kirari/blob/main/docs/streaming.md). +- Built-in JavaScript, TypeScript, JSX, TSX, JSON, JSONC, and Python reuse completed scanning work. Other languages rescan the full source. Long unfinished tokens can still be expensive; see [token streaming](https://github.com/vortexkit/kirari/blob/main/docs/streaming.md). - Changing `language` starts a new, empty stream. - After `end()`, call `reset()` before writing again. - `/stream` is a client module; `` is not. diff --git a/sites/docs/content/react.md b/sites/docs/content/react.md index 2a6d973..0a2bf93 100644 --- a/sites/docs/content/react.md +++ b/sites/docs/content/react.md @@ -43,4 +43,4 @@ export const StreamedCode = () => { } ``` -`useTokenStream` returns `{ tokens, write, end, reset }` and applies token patches as text arrives. Built-in JavaScript, TypeScript, JSON, and Python reuse completed scanning work; other languages rescan the full source. See [streaming limits](/docs/streaming/). It is client-only; `` is not. +`useTokenStream` returns `{ tokens, write, end, reset }` and applies token patches as text arrives. Built-in JavaScript, TypeScript, JSX, TSX, JSON, JSONC, and Python reuse completed scanning work; other languages rescan the full source. See [streaming limits](/docs/streaming/). It is client-only; `` is not. diff --git a/sites/docs/content/streaming.md b/sites/docs/content/streaming.md index 49ef88b..653bd92 100644 --- a/sites/docs/content/streaming.md +++ b/sites/docs/content/streaming.md @@ -39,6 +39,6 @@ For live output, render only `patch.tokens` and replace that tail of your view. **Know the streaming limits** -Only the unmodified built-in JavaScript, TypeScript, JSON, and Python use `'checkpoint'` mode (`stream.mode`), which rescans less. Everything else, including copies or extensions of those, uses `'full'`: every write retokenizes all source so far. +Only the unmodified built-in JavaScript, TypeScript, JSX, TSX, JSON, JSONC, and Python use `'checkpoint'` mode (`stream.mode`), which rescans less. Everything else, including copies or extensions of those, uses `'full'`: every write retokenizes all source so far. Even in checkpoint mode, a long line or unclosed multiline token can trigger repeated rescans. Both modes keep the full source in memory. Don’t mutate built-in definitions; copy them.