grammar(ts): vendor our tree-sitter-typescript fork (84ffdb8)

README.md

@@ -10,12 +10,14 @@
 > - **Much more TypeScript parses.** The TypeScript grammar is built from
 >   [marionebl/tree-sitter-typescript](https://github.com/marionebl/tree-sitter-typescript),
 >   a maintained fork of the stalled upstream grammar, with two fixes from
->   [dokimasia's fork](https://github.com/dokimasia/tree-sitter-typescript) and our own patch.
->   Variance annotations (`interface Stream<out A, in E>`), generic overloads after a member with
->   no `;`, `export type *`, `import("m").X[]` and template-literal escapes no longer turn a
->   declaration, and in large files the whole file, into a parse error that indexes as almost
->   nothing. On Effect, Zod, Excalidraw and VS Code, the files that tsc accepts but the grammar
->   rejects drop from 201 to 32 (upstream [#3226](https://github.com/colbymchenry/codegraph/pull/3226)
+>   [dokimasia's fork](https://github.com/dokimasia/tree-sitter-typescript) and our own, kept in
+>   our grammar fork (checked against tsc in its CI). Variance annotations
+>   (`interface Stream<out A, in E>`), generic overloads after a member with no `;`, tagged
+>   templates with type arguments (``sql<Row>`...` ``), `export type *`, `import("m").X[]`,
+>   keyword-named tuple labels and template-literal escapes no longer turn a declaration, and in
+>   large files the whole file, into a parse error that indexes as almost nothing. On Effect, Zod,
+>   Excalidraw and VS Code, the files that tsc accepts but the grammar rejects drop from 201 to 5
+>   (upstream [#3226](https://github.com/colbymchenry/codegraph/pull/3226)
 >   covers variance only; see `docs/grammars/tree-sitter-typescript.md`). Re-index TypeScript
 >   projects after updating.
 >

__tests__/fixtures/ts-syntax/gap-unique-in-comparison.ts

@@ -0,0 +1,5 @@
+// Gap: `unique` lexes as the keyword of `unique symbol` wherever type
+// arguments can start, so `i < unique.length` fails. Fixing it by making
+// `unique` a reserved identifier broke `f<typeof unique>(x)` instead.
+declare const unique: string[];
+for (let i = 0; i < unique.length; i++) {}

__tests__/fixtures/ts-syntax/gap-unique-in-comparison.tsx

@@ -0,0 +1,5 @@
+// Gap: `unique` lexes as the keyword of `unique symbol` wherever type
+// arguments can start, so `i < unique.length` fails. Fixing it by making
+// `unique` a reserved identifier broke `f<typeof unique>(x)` instead.
+declare const unique: string[];
+for (let i = 0; i < unique.length; i++) {}

__tests__/fixtures/ts-syntax/tagged-template-type-args.ts

@@ -0,0 +1,9 @@
+// Must keep parsing: a tagged template with type arguments (Effect's sql<Row>`...`).
+declare function sql<T>(s: TemplateStringsArray, ...v: unknown[]): T;
+declare const table: string;
+export const rows = sql<{ id: number; name: string }>`SELECT * FROM ${table}`;
+export const one = sql<number>`SELECT 1`.valueOf();
+export const nested = sql<Array<{ readonly count: number }>>`
+  SELECT COUNT(*) as count FROM ${table}
+`;
+export const lt = 1 < 2;

__tests__/fixtures/ts-syntax/tagged-template-type-args.tsx

@@ -0,0 +1,9 @@
+// Must keep parsing: a tagged template with type arguments (Effect's sql<Row>`...`).
+declare function sql<T>(s: TemplateStringsArray, ...v: unknown[]): T;
+declare const table: string;
+export const rows = sql<{ id: number; name: string }>`SELECT * FROM ${table}`;
+export const one = sql<number>`SELECT 1`.valueOf();
+export const nested = sql<Array<{ readonly count: number }>>`
+  SELECT COUNT(*) as count FROM ${table}
+`;
+export const lt = 1 < 2;

__tests__/fixtures/ts-syntax/tuple-keyword-type-names.ts

@@ -0,0 +1,12 @@
+// Must keep parsing: tuple elements that are types named like contextual
+// keywords, and `typeof unique` in type arguments.
+type type = 1; type get = 2; type set = 3; type declare = 4;
+export type A = [type];
+export type B = [get, set];
+export type C = [declare?];
+export type D = [type[]];
+export function f(a: [get]): void {}
+declare const unique: number;
+declare function g<T>(x: T): T;
+export const r = g<typeof unique>(1);
+export type E = (typeof unique)[];

__tests__/fixtures/ts-syntax/tuple-keyword-type-names.tsx

@@ -0,0 +1,12 @@
+// Must keep parsing: tuple elements that are types named like contextual
+// keywords, and `typeof unique` in type arguments.
+type type = 1; type get = 2; type set = 3; type declare = 4;
+export type A = [type];
+export type B = [get, set];
+export type C = [declare?];
+export type D = [type[]];
+export function f(a: [get]): void {}
+declare const unique: number;
+declare function g<T>(x: T): T;
+export const r = g<typeof unique>(1);
+export type E = (typeof unique)[];

__tests__/fixtures/ts-syntax/tuple-label-keyword.ts

@@ -0,0 +1,7 @@
+// Must keep parsing: a tuple label named like a type keyword (VS Code's terminalVoice.ts).
+export type A = [spoken: string, symbol: string][];
+export type B = [string: number, number?: string];
+export type C = [any: 1, ...object: unknown[]];
+export type D = [any?];
+export type E = [readonly string[]];
+export type F = [readonly: string];

__tests__/fixtures/ts-syntax/tuple-label-keyword.tsx

@@ -0,0 +1,7 @@
+// Must keep parsing: a tuple label named like a type keyword (VS Code's terminalVoice.ts).
+export type A = [spoken: string, symbol: string][];
+export type B = [string: number, number?: string];
+export type C = [any: 1, ...object: unknown[]];
+export type D = [any?];
+export type E = [readonly string[]];
+export type F = [readonly: string];

__tests__/typescript-grammar-fork-shapes.test.ts

@@ -0,0 +1,89 @@
+/**
+ * Node shapes and extraction for the constructs our tree-sitter-typescript fork
+ * added (docs/grammars/tree-sitter-typescript.md). The tsc differential only
+ * checks that a file parses; these pin what it parses into, and the call edges
+ * codegraph derives from it, so a later grammar move cannot regress them silently.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Node } from 'web-tree-sitter';
+import { extractFromSource } from '../src/extraction';
+import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
+
+beforeAll(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['typescript', 'tsx']);
+});
+
+const LANGS = ['typescript', 'tsx'] as const;
+
+function parse(language: 'typescript' | 'tsx', src: string): Node {
+  return getParser(language)!.parse(src)!.rootNode;
+}
+
+/** Names this file's code calls, as codegraph records them. */
+function calls(language: 'typescript' | 'tsx', src: string): string[] {
+  const file = language === 'tsx' ? 'src/a.tsx' : 'src/a.ts';
+  return extractFromSource(file, src, language)
+    .unresolvedReferences.filter((r) => r.referenceKind === 'calls')
+    .map((r) => r.referenceName);
+}
+
+describe('tagged templates with type arguments', () => {
+  it.each(LANGS)('%s: sql<Row>`...` is a call of sql with type arguments', (language) => {
+    const root = parse(language, 'sql<{ id: number }>`SELECT ${x}`;\nconst lt = a < b;');
+    expect(root.hasError).toBe(false);
+    const call = root.descendantsOfType('call_expression')[0]!;
+    expect(call.childForFieldName('function')!.text).toBe('sql');
+    expect(call.childForFieldName('type_arguments')!.type).toBe('type_arguments');
+    expect(call.childForFieldName('arguments')!.type).toBe('template_string');
+    expect(root.descendantsOfType('binary_expression')).toHaveLength(1);
+  });
+
+  it.each(LANGS)('%s: the tag is a call edge, also behind await and on a member', (language) => {
+    const src = [
+      'export async function load() {',
+      '  const a = sql<Row>`SELECT 1`;',
+      '  const b = await query<Row[]>`SELECT 2`;',
+      '  const c = db.tag<Row>`SELECT 3`;',
+      '}',
+    ].join('\n');
+    expect(calls(language, src)).toEqual(expect.arrayContaining(['sql', 'query', 'db.tag']));
+  });
+});
+
+describe('tuple labels named like type keywords', () => {
+  it.each(LANGS)('%s: keyword labels are names, keyword-named types stay types', (language) => {
+    const cases: Array<[string, Record<string, number>]> = [
+      ['type A = [symbol: string, number?: string];', { required_parameter: 1, optional_parameter: 1 }],
+      ['type B = [readonly: string, new: number];', { required_parameter: 2 }],
+      ['type C = [any?];', { optional_type: 1 }],
+      ['type D = [readonly string[]];', { readonly_type: 1 }],
+      ['type E = [type, get?];', { type_identifier: 3, optional_type: 1 }],
+      ['function f(a: [declare]) {}', { type_identifier: 1 }],
+    ];
+    for (const [src, want] of cases) {
+      const root = parse(language, src);
+      expect(root.hasError, src).toBe(false);
+      for (const [kind, n] of Object.entries(want)) {
+        expect(root.descendantsOfType(kind), `${src} ${kind}`).toHaveLength(n);
+      }
+    }
+  });
+});
+
+describe('typeof unique in type arguments', () => {
+  it.each(LANGS)('%s: f<typeof unique>(x) stays a generic call', (language) => {
+    const src = 'export function g() { return f<typeof unique>(1); }\ntype E = (typeof unique)[];';
+    expect(parse(language, src).hasError).toBe(false);
+    expect(calls(language, src)).toContain('f');
+    for (const more of [
+      'f<typeof unique.a>(x);',
+      'f<typeof unique[]>(x);',
+      'f<(typeof unique)[]>(x);',
+      'type A = (typeof unique);',
+      'type T = (typeof unique)["x"];',
+    ]) {
+      expect(parse(language, more).hasError, more).toBe(false);
+    }
+  });
+});