From 8da3031f5a109f2cf4f3b1283ccd5ff33267677d Mon Sep 17 00:00:00 2001 From: ugi-dev Date: Thu, 22 Oct 2026 01:48:08 +1100 Subject: [PATCH 1/3] fix(kirari): highlight decorators and reduce bundle overhead --- .changeset/decorator-highlighting.md | 7 + README.md | 2 +- docs/comparison.md | 8 +- packages/kirari/README.md | 10 +- packages/kirari/src/grammars/definitions.ts | 5 +- packages/kirari/src/grammars/functions.ts | 137 ++++++++-------- packages/kirari/src/languages/javascript.ts | 6 +- packages/kirari/src/languages/python.ts | 7 +- packages/kirari/src/render.ts | 14 +- packages/kirari/src/tokenize.ts | 4 +- packages/kirari/test/corpus/fixtures.ts | 28 ++++ packages/kirari/test/decorators.test.ts | 168 ++++++++++++++++++++ sites/docs/changelog/kirari/1.1.1.md | 5 + sites/docs/content/limitations.md | 2 +- sites/docs/src/lib/site.ts | 2 +- 15 files changed, 310 insertions(+), 95 deletions(-) create mode 100644 .changeset/decorator-highlighting.md create mode 100644 packages/kirari/test/decorators.test.ts diff --git a/.changeset/decorator-highlighting.md b/.changeset/decorator-highlighting.md new file mode 100644 index 0000000..e2f2551 --- /dev/null +++ b/.changeset/decorator-highlighting.md @@ -0,0 +1,7 @@ +--- +"@vortexlabs/kirari": patch +--- + +Highlight JavaScript and Python decorator markers as operators and bare decorator names as functions, while preserving qualified names and Python matrix multiplication. + +Reduce bundle overhead by consolidating function lookahead, simplifying equivalent JavaScript regexes, and sharing HTML escape replacements. diff --git a/README.md b/README.md index 8613f21..e42c83b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ TypeScript apps. It turns source code into escaped HTML or a token array on the server, at build time, or in the browser, without a DOM. It is a lexical highlighter, not a parser. -- **Small:** about 4.3 KB gzipped with JavaScript, about 23 KB with all 48 languages. +- **Small:** about 4.2 KB gzipped with JavaScript, about 23 KB with all 48 languages. - **Streaming:** emits token patches as code arrives, such as LLM output. - **Themeable:** CSS-variable themes, with 21 built in. - **Integrations:** a React `` component that works in Server Components, diff --git a/docs/comparison.md b/docs/comparison.md index c7b0019..57ebac9 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -14,11 +14,11 @@ bun packages/benchmarks/compare-size.ts "$comparison_dir" bun run size ``` -Measured on 2026-10-21: +Competitors measured on 2026-10-21; Kirari working tree measured on 2026-10-22: | Setup | Gzip bytes | | --- | ---: | -| Kirari working tree, `highlight()` wrapper | 4,190 | +| Kirari working tree, `highlight()` wrapper | 4,146 | | Shiki 4.5.0, synchronous core, JavaScript regex engine, JavaScript grammar, GitHub Dark | 73,530 | | Prism 1.30.0, core, clike dependency, JavaScript grammar | 5,278 | | highlight.js 11.12.0, core, JavaScript grammar | 11,056 | @@ -26,8 +26,8 @@ Measured on 2026-10-21: The script validates that each bundled function produces highlighted HTML and preserves an escaped source sample. It does not measure speed or grammar accuracy. -Kirari's full JavaScript entry is 4,292 bytes gzip; its full set of 48 language -definitions with core is 22,900 bytes. These come from `bun run size`. The wrapper +Kirari's full JavaScript entry is 4,247 bytes gzip; its full set of 48 language +definitions with core is 22,960 bytes. These come from `bun run size`. The wrapper above is smaller because unused exports can be removed. Kirari measurements use the current source, including unreleased changes; competitors use the named npm versions. Rerun both commands when the source or bundler changes. diff --git a/packages/kirari/README.md b/packages/kirari/README.md index cde06d9..bb0c927 100644 --- a/packages/kirari/README.md +++ b/packages/kirari/README.md @@ -2,7 +2,7 @@ A small, dependency-free syntax highlighter. Kirari turns source code into escaped HTML or a token array on the server, at build time, or in the browser, -without a DOM. The default bundle is about 4.3 KB gzipped; 48 language +without a DOM. The default bundle is about 4.2 KB gzipped; 48 language definitions and 21 CSS-variable themes are available as separate imports. It is a lexical highlighter, not a parser. @@ -329,13 +329,13 @@ The core fixture includes an exported API object; the all-language fixture also retains a language registry so that no language gets removed as unused. Theme CSS is excluded. These are bundle sizes, not throughput benchmarks. -Measured on 2026-10-17 using Bun 1.3.11: +Measured on 2026-10-22 using Bun 1.3.4: | Entry | Minified bytes | Gzip bytes | | --- | ---: | ---: | -| Core API | 3,439 | 1,717 | -| JavaScript default | 4,648 | 2,240 | -| All 48 languages | 51,964 | 16,044 | +| Core API | 8,505 | 3,728 | +| JavaScript default | 9,678 | 4,247 | +| All 48 languages | 68,465 | 22,960 | ## Development diff --git a/packages/kirari/src/grammars/definitions.ts b/packages/kirari/src/grammars/definitions.ts index a046040..c6032a5 100644 --- a/packages/kirari/src/grammars/definitions.ts +++ b/packages/kirari/src/grammars/definitions.ts @@ -195,7 +195,10 @@ export const classifyDefinitions = (tokens: Token[], language: string, open: rea const qualified = followsDeclaration && /^(\.|:)$/.test(value(qualifierEnd + 1)) if (qualified) qualifiers.add(qualifierEnd) const declared = followsDeclaration && !qualified - let isFunction = declared + // Decorators start a logical line outside brackets; matrix multiplication does not. + const decorated = language === 'python' && value(index - 1) === '@' && value(index + 1) !== '.' && !lineBreakBefore(index) + && !parents.has(index - 1) && (index === 1 || lineBreakBefore(index - 1)) + let isFunction = declared || decorated const member = /^(\.|\?\.|->|\?->|::)$/.test(value(index - 1)) let next = index + 1 if (!declared && token.type !== 'variable' && token.type !== 'property') continue diff --git a/packages/kirari/src/grammars/functions.ts b/packages/kirari/src/grammars/functions.ts index 91219fb..d2feae4 100644 --- a/packages/kirari/src/grammars/functions.ts +++ b/packages/kirari/src/grammars/functions.ts @@ -9,14 +9,14 @@ export interface FunctionLookupCache { lastLine?: number } export interface LookupOptions { - /** The name follows `const`, `let`, or `var` in TypeScript, so `:` starts a type annotation. */ - readonly annotation: boolean + /** A colon after the name may introduce a property value or a typed binding. */ + readonly colon: boolean /** The name may take type parameters, as after `function` or in a method position. */ readonly generic: boolean - /** The name is an object or class member. */ - readonly property: boolean /** The name is a bare `async` that may be an arrow-function modifier. */ readonly asyncModifier: boolean + /** The name follows `@`; a member-access suffix instead makes it a namespace. */ + readonly decorator: boolean } const closers: Record = { '(': ')', '[': ']', '{': '}', '${': '}', '<': '>' } @@ -32,16 +32,15 @@ const closers: Record = { '(': ')', '[': ']', '{': '}', '${': '} export const createFunctionLookup = (source: string, typed: boolean, scan: (start: number) => Generator, cache: FunctionLookupCache = {}) => { // Reuse nested groups, including unfinished ones, instead of rescanning each signature suffix. const groups = new Map() - return (start: number, { annotation, generic, property, asyncModifier }: LookupOptions): Result => { + return (start: number, { colon: acceptsColon, generic, asyncModifier, decorator }: LookupOptions): Result => { let reader = scan(start) let token: Token | undefined let limit = source.length let pending = false - // Wraps an answer, marking it pending when it relied on reaching the end of the source. - const result = (value: boolean): Result => ({ function: value, pending: pending || limit === source.length && (!token || token.end === limit && /^[=/]$/.test(token.value)) }) const is = (value: string): boolean => token?.value === value - // Advances `token` to the next significant token before `limit`. - const next = (): void => { + // Advances `token`, optionally restarting at a source offset. + const next = (position?: number): void => { + if (position !== undefined) reader = scan(position) do { const value = reader.next() token = value.done || value.value.start >= limit ? undefined : value.value @@ -52,13 +51,11 @@ export const createFunctionLookup = (source: string, typed: boolean, scan: (star } } while (token && !isSignificant(token.type)) } - // Restarts scanning at `position`. - const jump = (position: number): void => { reader = scan(position); next() } // Consumes the bracket group opening at `token`, including `<…>` and `>>` splits; results are cached. const group = (): Group => { const opening = token! const cached = groups.get(opening.start) - if (cached) { jump(cached.end); return cached } + if (cached) { next(cached.end); return cached } const angles = opening.value === '<' const stack: { token: Token; close: string }[] = [] while (token) { @@ -76,7 +73,7 @@ export const createFunctionLookup = (source: string, typed: boolean, scan: (star } const closed = { end: token.start + index + 1, complete: true } groups.set(frame.token.start, closed) - if (!stack.length) { jump(closed.end); return closed } + if (!stack.length) { next(closed.end); return closed } } } next() @@ -114,64 +111,74 @@ export const createFunctionLookup = (source: string, typed: boolean, scan: (star return false } - next() - // Optional or definite members: `name?()`, `name!: T`. - if (typed && generic && token && /^[?!]$/.test(token.value)) next() - // `name(` is a call or definition; for `async (x) => …` the name is a modifier, not a function. - if (is('(')) { - if (!asyncModifier) return result(true) - if (!group().complete) return result(false) - return result(!is('=>') && !(typed && is(':'))) - } - if (typed && is('<') && (generic || token!.start === start)) { - if (!group().complete) return result(false) - return result(is('(')) - } - // Otherwise the name must be bound: `name = …`, or `name: …` for properties and annotations. - if (!is('=') && !is(':')) return result(false) - const colon = is(':') - if (colon && !annotation && !property) return result(false) - next() - if (typed && (annotation || property) && colon && token && !/^(\(|<|async|function)$/.test(token.value)) { - if (!until('=')) return result(false) + const inspect = (): boolean => { next() - } - - // Inspect the bound value; parenthesized expressions are re-entered with a narrower limit. - while (token) { - if (is('async')) { - next() - if (is('=>')) return result(true) + if (decorator) { + // A trailing `?` or one/two dots may become a different operator in the next chunk. + pending ||= !!token && token.start >= source.length - 2 && (is('.') || is('?')) + return !is('.') && !is('?.') } - if (is('function')) { - next() - if (is('*')) next() - if (token && /^(variable|keyword|function)$/.test(token.type)) next() - if (typed && is('<') && !group().complete) return result(true) - if (!is('(') || !group().complete) return result(true) - if (typed && is(':')) { next(); if (!body()) return result(true) } - if (!is('{') || !group().complete) return result(true) - return result(continuation()) - } - if (typed && is('<') && !group().complete) return result(false) + // Optional or definite members: `name?()`, `name!: T`. + if (typed && generic && token && /^[?!]$/.test(token.value)) next() + // `name(` is a call or definition; for `async (x) => …` the name is a modifier, not a function. if (is('(')) { - const opening = token - const closed = group() - if (!closed.complete) return result(false) - if (is('=>')) return result(true) - if (typed && is(':')) { next(); return result(until('=>')) } - if (!continuation()) return result(false) - pending ||= !token && limit === source.length - limit = closed.end - 1 - jump(opening.end) - continue + if (!asyncModifier) return true + if (!group().complete) return false + return !is('=>') && !(typed && is(':')) + } + if (typed && is('<') && (generic || token!.start === start)) { + if (!group().complete) return false + return is('(') } - if (token && /^(variable|keyword|function)$/.test(token.type)) { + // Otherwise the name must be bound: `name = …`, or `name: …` for properties and annotations. + if (!is('=') && !is(':')) return false + const colon = is(':') + if (colon && !acceptsColon) return false + next() + if (typed && acceptsColon && colon && token && !/^(\(|<|async|function)$/.test(token.value)) { + if (!until('=')) return false next() - return result(is('=>')) } - return result(false) + + // Inspect the bound value; parenthesized expressions are re-entered with a narrower limit. + while (token) { + if (is('async')) { + next() + if (is('=>')) return true + } + if (is('function')) { + next() + if (is('*')) next() + if (token && /^(variable|keyword|function)$/.test(token.type)) next() + if (typed && is('<') && !group().complete) return true + if (!is('(') || !group().complete) return true + if (typed && is(':')) { next(); if (!body()) return true } + if (!is('{') || !group().complete) return true + return continuation() + } + if (typed && is('<') && !group().complete) return false + if (is('(')) { + const opening = token + const closed = group() + if (!closed.complete) return false + if (is('=>')) return true + if (typed && is(':')) { next(); return until('=>') } + if (!continuation()) return false + pending ||= !token && limit === source.length + limit = closed.end - 1 + next(opening.end) + continue + } + if (token && /^(variable|keyword|function)$/.test(token.type)) { + next() + return is('=>') + } + return false + } + return false } - return result(false) + const found = inspect() + // EOF-dependent answers remain provisional until more source arrives. + return { function: found, pending: pending || limit === source.length && (!token || token.end === limit && /^[=/]$/.test(token.value)) } } } diff --git a/packages/kirari/src/languages/javascript.ts b/packages/kirari/src/languages/javascript.ts index 49f8de7..def1e02 100644 --- a/packages/kirari/src/languages/javascript.ts +++ b/packages/kirari/src/languages/javascript.ts @@ -13,9 +13,9 @@ export const javascript: Language = { { type: 'comment', pattern: /^#![^\r\n\u2028\u2029]*/ }, { type: 'comment', pattern: /\/\/[^\r\n\u2028\u2029]*/ }, { type: 'comment', pattern: /\/\*[\s\S]*?(?:\*\/|$)/ }, - { type: 'string', pattern: /"(?:\\(?:\r\n|[\s\S]|$)|[^"\\\r\n])*(?:"|(?=[\r\n])|$)|'(?:\\(?:\r\n|[\s\S]|$)|[^'\\\r\n])*(?:'|(?=[\r\n])|$)/ }, - { type: 'number', pattern: /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[bB][01](?:_?[01])*|0[oO][0-7](?:_?[0-7])*|\d(?:_?\d)*(?:\.(?:\d(?:_?\d)*)?)?(?:[eE][+-]?\d(?:_?\d)*)?|\.\d+(?:[eE][+-]?\d+)?)(?:n)?/ }, - { type: 'operator', pattern: /\?\.|=>|\*\*=?|===?|!==?|>>>?=?|<<=?|&&=?|\|\|=?|\?\?=?|\+\+|--|\.\.\.|[+*/%&|^!=<>-]=?|[?~]/ }, + { type: 'string', pattern: /"(?:\\(?:\r\n|[\s\S]|$)|[^"\\\r\n])*"?|'(?:\\(?:\r\n|[\s\S]|$)|[^'\\\r\n])*'?/ }, + { type: 'number', pattern: /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[bB][01](?:_?[01])*|0[oO][0-7](?:_?[0-7])*|\d(?:_?\d)*(?:\.(?:\d(?:_?\d)*)?)?(?:[eE][+-]?\d(?:_?\d)*)?|\.\d+(?:[eE][+-]?\d+)?)n?/ }, + { type: 'operator', pattern: /\?\.|=>|\*\*=?|===?|!==?|>>>?=?|<<=?|&&=?|\|\|=?|\?\?=?|\+\+|--|\.\.\.|[+*/%&|^!=<>-]=?|[?~@]/ }, { type: 'punctuation', pattern: /[{}()[\],;:.]/ }, ], } diff --git a/packages/kirari/src/languages/python.ts b/packages/kirari/src/languages/python.ts index 6de8507..2e49b42 100644 --- a/packages/kirari/src/languages/python.ts +++ b/packages/kirari/src/languages/python.ts @@ -58,7 +58,7 @@ const scanLanguage = (language: Language): Language => { return builtIn } -// Resume points are whitespace tokens containing a newline, outside f-strings, with more source after them. +// Resume points are whitespace at unescaped line breaks, outside f-strings, with more source after them. const scanPython: ResumableScan = (source, language, from, checkpoint) => { const tokens: Token[] = [] const frames: PythonFrame[] = [{ kind: 'expression', depth: 0, root: true }] @@ -105,7 +105,10 @@ const scanPython: ResumableScan = (source, language, from, checkpoint) => { if (checkpoint && frames.length === 1 && token.type === 'text' && /[\r\n]/.test(token.value) && token.end < source.length) { const last = tokens[tokens.length - 1] const merged = last?.type === 'text' - checkpoint({ offset: merged ? last.start : position, previous }, merged ? tokens.length - 1 : tokens.length) + // A backslash in source text continues the line; one inside a comment does not. + if (!merged || !last.value.endsWith('\\')) { + checkpoint({ offset: merged ? last.start : position, previous }, merged ? tokens.length - 1 : tokens.length) + } } const opening = token.type === fstringOpening.type && /^(f|fr|rf)("""|'''|"|')$/i.exec(token.value) if (opening) { diff --git a/packages/kirari/src/render.ts b/packages/kirari/src/render.ts index 1c223ea..710acc9 100644 --- a/packages/kirari/src/render.ts +++ b/packages/kirari/src/render.ts @@ -1,17 +1,11 @@ import type { RenderOptions, Token } from './types.js' -export const escape = (value: string): string => { - return value.replace(/[&<>"']/g, char => { - switch (char) { - case '&': return '&' - case '<': return '<' - case '>': return '>' - case '"': return '"' - default: return ''' - } - }) +const entities: Readonly> = { + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', } +export const escape = (value: string): string => value.replace(/[&<>"']/g, char => entities[char]!) + /** Extra classes for a token from the `classes` map and the `className` callback. */ export const tokenClasses = (token: Token, options: RenderOptions): string => { const mapped = options.classes && Object.hasOwn(options.classes, token.type) diff --git a/packages/kirari/src/tokenize.ts b/packages/kirari/src/tokenize.ts index 5386289..6bd1300 100644 --- a/packages/kirari/src/tokenize.ts +++ b/packages/kirari/src/tokenize.ts @@ -243,10 +243,10 @@ export function* scan(source: string, language: Language, start = 0, previous?: // `pending` means the answer depends on source that has not arrived yet (streaming). if (lookup && !member && (type === 'variable' || type === 'property' || context.functionName || method || /^(async|get|set|static)$/.test(name))) { const found = lookup(end, { - annotation: typed && variableKeyword.test(previous?.value ?? ''), + colon: type === 'property' || method || typed && variableKeyword.test(previous?.value ?? ''), generic: !!context.functionName || method, - property: type === 'property' || method, asyncModifier: name === 'async' && !context.functionName && !method, + decorator: previous?.value === '@', }) if (found.function) type = 'function' if (found.pending) context.functionPending = true diff --git a/packages/kirari/test/corpus/fixtures.ts b/packages/kirari/test/corpus/fixtures.ts index cc087d9..9572e99 100644 --- a/packages/kirari/test/corpus/fixtures.ts +++ b/packages/kirari/test/corpus/fixtures.ts @@ -400,4 +400,32 @@ export const corpus: readonly CorpusFixture[] = [ { text: '${PORT:-${FALLBACK}}', type: 'variable' }, ], }, + { + id: 'javascript-decorators', language: 'javascript', + source: '@sealed\n@factory()\nclass Parcel {}', + assertions: [ + { text: '@', type: 'operator', occurrence: 0 }, { text: '@', type: 'operator', occurrence: 1 }, + { text: 'sealed', type: 'function' }, { text: 'factory', type: 'function' }, + { text: 'class', type: 'keyword' }, { text: 'Parcel', type: 'type' }, + ], + }, + { + id: 'typescript-qualified-decorator', language: 'typescript', + source: '@registry /* namespace */ .sealed\nclass Parcel {}', + assertions: [ + { text: '@', type: 'operator' }, { text: 'registry', type: 'variable' }, + { text: '/* namespace */', type: 'comment' }, { text: 'sealed', type: 'property' }, + { text: 'Parcel', type: 'type' }, + ], + }, + { + id: 'python-decorators-and-matrix-multiplication', language: 'python', + source: '@dataclass\n@registry.sealed()\nclass Parcel:\n result = left @ right', + assertions: [ + { text: '@', type: 'operator', occurrence: 0 }, { text: '@', type: 'operator', occurrence: 1 }, + { text: '@', type: 'operator', occurrence: 2 }, { text: 'dataclass', type: 'function' }, + { text: 'registry', type: 'variable' }, { text: 'sealed', type: 'property' }, + { text: 'Parcel', type: 'type' }, { text: 'left', type: 'variable' }, { text: 'right', type: 'variable' }, + ], + }, ] diff --git a/packages/kirari/test/decorators.test.ts b/packages/kirari/test/decorators.test.ts new file mode 100644 index 0000000..ba0e447 --- /dev/null +++ b/packages/kirari/test/decorators.test.ts @@ -0,0 +1,168 @@ +import { expect, test } from 'bun:test' +import { tokenize } from '../src/core.js' +import { javascript } from '../src/languages/javascript.js' +import { jsx } from '../src/languages/jsx.js' +import { python } from '../src/languages/python.js' +import { tsx } from '../src/languages/tsx.js' +import { typescript } from '../src/languages/typescript.js' +import { createTokenStream } from '../src/stream.js' +import type { Language, Token } from '../src/types.js' + +const pairs = (source: string, language: Language) => tokenize(source, language).map(token => [token.type, token.value]) + +for (const language of [javascript, typescript, jsx, tsx]) { + test(`${language.name} highlights bare decorators and decorator factories`, () => { + for (const source of ['@sealed\nclass Parcel {}', '@sealed()\nclass Parcel {}', 'class Parcel { @sealed field = 1 }']) { + const result = pairs(source, language) + expect(result, source).toContainEqual(['operator', '@']) + expect(result, source).toContainEqual(['function', 'sealed']) + } + }) + + test(`${language.name} retains namespace and member classifications in qualified decorators`, () => { + for (const source of ['@registry.sealed\nclass Parcel {}', '@registry.sealed()\nclass Parcel {}', '@registry /* namespace */ .sealed\nclass Parcel {}']) { + const result = pairs(source, language) + expect(result, source).toContainEqual(['operator', '@']) + expect(result, source).toContainEqual(['variable', 'registry']) + expect(result, source).toContainEqual(['property', 'sealed']) + } + }) + + test(`${language.name} keeps decorator-looking strings and comments opaque`, () => { + const result = pairs('const label = "@sealed"; // @comment\n/* @other */', language) + expect(result.filter(([type]) => type === 'operator' || type === 'function')).toEqual([['operator', '=']]) + expect(result).toContainEqual(['string', '"@sealed"']) + expect(result).toContainEqual(['comment', '// @comment']) + expect(result).toContainEqual(['comment', '/* @other */']) + }) + + test(`${language.name} distinguishes decorator member access from a spread operator`, () => { + expect(pairs('@ sealed\n?.', language)).toContainEqual(['variable', 'sealed']) + expect(pairs('@sealed\n...', language)).toContainEqual(['function', 'sealed']) + const qualified = pairs('@registry /*c*/ .sealed', language) + expect(qualified).toContainEqual(['variable', 'registry']) + expect(qualified).toContainEqual(['property', 'sealed']) + }) +} + +test('Python highlights bare decorators, stacked decorators, and decorator factories', () => { + for (const source of [ + '@dataclass\nclass Parcel: pass', + '@dataclass()\nclass Parcel: pass', + '@cached\n@dataclass\nclass Parcel: pass', + 'class Parcel:\n @cached\n def label(self): pass', + ]) { + const result = pairs(source, python) + expect(result, source).toContainEqual(['operator', '@']) + for (const name of ['dataclass', 'cached']) { + if (source.includes(name)) expect(result, source).toContainEqual(['function', name]) + } + } +}) + +test('Python retains qualified decorators as a namespace and member', () => { + for (const source of ['@registry.sealed\nclass Parcel: pass', '@registry.sealed()\nclass Parcel: pass']) { + const result = pairs(source, python) + expect(result, source).toContainEqual(['operator', '@']) + expect(result, source).toContainEqual(['variable', 'registry']) + expect(result, source).toContainEqual(['property', 'sealed']) + } +}) + +test('Python decorator continuations stay on the same logical line', () => { + for (const newline of ['\n', '\r\n']) { + const bare = pairs(`@ \\${newline}cached${newline}def label(): pass`, python) + expect(bare).toContainEqual(['operator', '@']) + expect(bare).toContainEqual(['function', 'cached']) + + const qualified = pairs(`@registry \\${newline} .sealed${newline}def label(): pass`, python) + expect(qualified).toContainEqual(['variable', 'registry']) + expect(qualified).toContainEqual(['property', 'sealed']) + + const interrupted = pairs(`@${newline}cached${newline}def label(): pass`, python) + expect(interrupted).toContainEqual(['variable', 'cached']) + } +}) + +test('Python matrix multiplication operands remain variables across continued lines', () => { + for (const newline of ['\n', '\r\n']) { + for (const source of [ + 'result = left @ right', + `result = left \\${newline} @ right`, + `result = left @ \\${newline} right`, + `result = (${newline} left${newline} @ right${newline})`, + `result = (${newline} left @${newline} right${newline})`, + `result = (${newline} left # matrix${newline} @ right${newline})`, + `@cached${newline}def multiply():${newline} return left @ right`, + ]) { + const result = pairs(source, python) + expect(result, source).toContainEqual(['operator', '@']) + expect(result, source).toContainEqual(['variable', 'left']) + expect(result, source).toContainEqual(['variable', 'right']) + } + } +}) + +test('Python keeps decorator-looking strings and comments opaque', () => { + const result = pairs('label = "@dataclass"\n# @comment\ntext = """@other"""', python) + expect(result.filter(([type]) => type === 'function')).toEqual([]) + expect(result).toContainEqual(['string', '"@dataclass"']) + expect(result).toContainEqual(['comment', '# @comment']) + expect(result).toContainEqual(['string', '"""@other"""']) +}) + +test('Python retains a checkpoint after a comment ending with a backslash', () => { + const stream = createTokenStream(python) + stream.write('# comment \\\nvalue') + expect(stream.write('\nnext').from).toBeGreaterThan(0) +}) + +for (const original of [javascript, typescript, jsx, tsx, python]) { + test(`${original.name} decorators retain classifications in renamed language copies`, () => { + const language = { ...original, name: `custom-${original.name}` } + const source = '@sealed\n' + (original === python ? 'class Parcel: pass' : 'class Parcel {}') + expect(pairs(source, language)).toContainEqual(['operator', '@']) + expect(pairs(source, language)).toContainEqual(['function', 'sealed']) + }) + + test(`${original.name} decorator prefixes preserve offsets and streaming patch parity`, () => { + const sources = original === python ? [ + '@dataclass\nclass Café: pass\n@registry.sealed()\ndef label(): pass\n', + 'result = (\n left\n @ right\n)\n@cached\ndef multiply(): return left @ right\n', + 'result = left \\\r\n @ right\r\n# @opaque 🚉\r\n', + '@ \\\n cached\ndef label(): pass\n', + '@registry \\\r\n .sealed\r\ndef label(): pass\r\n', + '@\ncached\ndef label(): pass\n', + ] : [ + '@sealed\nclass Café {}\n@factory()\nclass Parcel {}\n', + '@registry.sealed\nclass Parcel { @cached method() {} }\nconst label = "@opaque 🚉";', + '@ sealed\n?.', + '@sealed\n...', + '@registry /*c*/ .sealed', + ] + for (const language of [original, { ...original, name: `custom-${original.name}` }]) { + for (const source of sources) { + const stream = createTokenStream(language) + let assembled: Token[] = [] + for (let end = 1; end <= source.length; end++) { + const prefix = source.slice(0, end) + const tokens = tokenize(prefix, language) + let offset = 0 + for (const token of tokens) { + expect(token.start).toBe(offset) + expect(token.end).toBeGreaterThan(token.start) + expect(token.value).toBe(prefix.slice(token.start, token.end)) + offset = token.end + } + expect(offset).toBe(end) + const patch = stream.write(source[end - 1]!) + assembled = [...assembled.filter(token => token.start < patch.from), ...patch.tokens] + expect(assembled, `${language.name}: ${prefix}`).toEqual(tokens) + } + const patch = stream.end() + assembled = [...assembled.filter(token => token.start < patch.from), ...patch.tokens] + expect(assembled).toEqual(tokenize(source, language)) + } + } + }) +} diff --git a/sites/docs/changelog/kirari/1.1.1.md b/sites/docs/changelog/kirari/1.1.1.md index 9898401..370ff8b 100644 --- a/sites/docs/changelog/kirari/1.1.1.md +++ b/sites/docs/changelog/kirari/1.1.1.md @@ -4,6 +4,11 @@ version: 1.1.1 date: 2026-10-21 --- +### Improvements + +- Reduce bundle overhead by consolidating function lookahead, simplifying equivalent JavaScript regexes, and sharing HTML escape replacements. + ### Bug fixes - TypeScript enum declaration names are now highlighted as types, like class, interface, and type alias names. +- JavaScript, TypeScript, and Python decorator markers are highlighted as operators and bare decorator names as functions, while qualified names and Python matrix multiplication retain their classifications. diff --git a/sites/docs/content/limitations.md b/sites/docs/content/limitations.md index 0dcfe84..d16d432 100644 --- a/sites/docs/content/limitations.md +++ b/sites/docs/content/limitations.md @@ -16,7 +16,7 @@ Modern ESM runtimes with ES2022 features are required. The library has no DOM, N ## Bundle measurements -The JavaScript entry measures 4,292 bytes gzip; all 48 definitions together measure 22,900 bytes gzip. Measured with Bun 1.3.11 minification and gzip level 9, excluding CSS. These are library bundle sizes, not website sizes or throughput benchmarks. +The JavaScript entry measures 4,247 bytes gzip; all 48 definitions together measure 22,960 bytes gzip. Measured on 2026-10-22 with Bun 1.3.4 minification and gzip level 9, excluding CSS. These are library bundle sizes, not website sizes or throughput benchmarks. To reproduce, run `bun run size` in the Kirari repository. diff --git a/sites/docs/src/lib/site.ts b/sites/docs/src/lib/site.ts index 937c8d9..3f5055e 100644 --- a/sites/docs/src/lib/site.ts +++ b/sites/docs/src/lib/site.ts @@ -12,6 +12,6 @@ export const site = { x: 'https://x.com/intent/follow?screen_name=ugi_dev', version: kirariPackage.version, // Tree-shaken highlight() setup; measured by packages/benchmarks/compare-size.ts. - javascriptHighlightGzipBytes: 4190, + javascriptHighlightGzipBytes: 4146, install: 'npm install @vortexlabs/kirari', } as const From 532aefa45f6fa5de6005b288df2ffd08b362cbc3 Mon Sep 17 00:00:00 2001 From: ugi-dev Date: Thu, 22 Oct 2026 02:04:35 +1100 Subject: [PATCH 2/3] chore: configure CodeRabbit reviews for Kirari --- .coderabbit.yaml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000..a175934 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +inheritance: true +language: "en-US" +early_access: false +tone_instructions: "Review Kirari as a TypeScript syntax-highlighting library. Keep feedback concise and actionable, prioritizing correctness, streaming consistency, public API compatibility, and bundle size." +reviews: + profile: "assertive" + request_changes_workflow: false + high_level_summary: true + poem: false + review_status: true + collapse_walkthrough: false + auto_review: + enabled: true + drafts: false + base_branches: ["main"] + auto_pause_after_reviewed_commits: 0 + path_filters: + - "!**/.zed/**" + path_instructions: + - path: "packages/kirari/src/**/*.ts" + instructions: | + Review Kirari's tokenizer, grammars, renderer, and streaming API. Ensure that: + - Preserve exact source text, contiguous UTF-16 token offsets, custom token categories, and HTML escaping. + - Keep streamed patches equivalent to full tokenization across chunk boundaries, incomplete source, and reset/end calls. + - Preserve safe resume checkpoints and lookahead caches; check adversarial inputs for repeated rescanning. + - Preserve public exports, language coverage, tree shaking, and dependency-free browser/server support. + - Verify bundle-size claims with `bun run size` and keep existing size budgets. + - path: "packages/react/src/**/*.{ts,tsx}" + instructions: | + Review the React rendering and streaming integration. Ensure that: + - Keep static Code rendering compatible with Server Components and hooks in the client entry point. + - Reuse core tokenization and decoration logic, and render source as React text children. + - Preserve token-patch application, language-change resets, and protection against stale stream callbacks. + - path: "packages/rehype/src/**/*.ts" + instructions: | + Review the Rehype integration and code-block metadata handling. Ensure that: + - Preserve source text and existing pre/code properties when transforming HAST nodes. + - Keep code content as text nodes and attributes as HAST properties for safe serialization. + - Preserve language aliases, unknown-language fallback, metadata parsing, and line decorations. +chat: + auto_reply: true From 3d6dadf1745a38eaf64f1f6897f4fa0094d53c19 Mon Sep 17 00:00:00 2001 From: ugi-dev Date: Thu, 22 Oct 2026 02:14:13 +1100 Subject: [PATCH 3/3] docs: name TypeScript in decorator release note --- .changeset/decorator-highlighting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/decorator-highlighting.md b/.changeset/decorator-highlighting.md index e2f2551..545ee5b 100644 --- a/.changeset/decorator-highlighting.md +++ b/.changeset/decorator-highlighting.md @@ -2,6 +2,6 @@ "@vortexlabs/kirari": patch --- -Highlight JavaScript and Python decorator markers as operators and bare decorator names as functions, while preserving qualified names and Python matrix multiplication. +Highlight JavaScript, TypeScript, and Python decorator markers as operators and bare decorator names as functions, while preserving qualified names and Python matrix multiplication. Reduce bundle overhead by consolidating function lookahead, simplifying equivalent JavaScript regexes, and sharing HTML escape replacements.