feat(mql-typescript): add MongoDB 8.3 arrayIndexAs/as/valueAs to $filter and $reduce MONGOSH-3244

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "packages/mql-typescript/mongo-php-library"]
-	path = packages/mql-typescript/mongo-php-library
-	url = https://github.com/mongodb-js/mongo-php-library
+[submodule "packages/mql-typescript/mql-specifications"]
+	path = packages/mql-typescript/mql-specifications
+	url = https://github.com/mongodb/mql-specifications.git

packages/mql-typescript/.depcheckrc

@@ -4,7 +4,6 @@ ignores:
  - '@types/chai'
  - '@types/sinon-chai'
  - 'sinon'
- - 'json-schema-to-zod'
 ignore-patterns:
  - 'dist'
  - 'mongo-php-library'

packages/mql-typescript/.eslintignore

@@ -1,3 +1,4 @@
 .nyc-output
 dist
 mongo-php-library
+mql-specifications

packages/mql-typescript/.prettierignore

@@ -2,3 +2,4 @@
 dist
 coverage
 mongo-php-library
+mql-specifications

packages/mql-typescript/package.json

@@ -57,7 +57,7 @@
     "check-ci": "npm run check",
     "test": "tsc --noEmit tests/**/*.ts",
     "reformat": "npm run prettier -- --write .",
-    "extract-metaschema": "json-refs resolve mongo-php-library/generator/config/schema.json | json-schema-to-zod -n Operator -o src/metaschema.ts && npm run prettier -- --write src/metaschema.ts",
+    "extract-metaschema": "ts-node src/build-metaschema.ts && npm run prettier -- --write src/metaschema.ts",
     "pregenerate-schema": "npm run extract-metaschema",
     "generate-schema": "ts-node src/cli.ts schema",
     "postgenerate-schema": "npm run prettier -- --write out/*",
@@ -66,7 +66,7 @@
     "postgenerate-tests": "npm run prettier -- --write tests/**/*.ts",
     "pregenerate-driver-schema": "npm run extract-metaschema",
     "generate-driver-schema": "ts-node src/cli.ts driver-schema",
-    "postgenerate-driver-schema": "yamlfix -c mongo-php-library/generator/config/.yamlfix.toml mongo-php-library/generator/config",
+    "postgenerate-driver-schema": "yamlfix -c mql-specifications/.yamlfix.toml mql-specifications/definitions",
     "format-generated-files": "npm run postgenerate-schema && npm run postgenerate-tests && npm run postgenerate-driver-schema"
   },
   "devDependencies": {
@@ -83,7 +83,6 @@
     "gen-esm-wrapper": "^1.1.3",
     "js-yaml": "^4.1.1",
     "jsdom": "^24.1.3",
-    "json-refs": "^3.0.15",
     "json-schema-to-zod": "^2.6.0",
     "prettier": "^3.8.1",
     "ts-node": "^10.9.2",

packages/mql-typescript/src/build-metaschema.ts

@@ -0,0 +1,77 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { jsonSchemaToZod } from 'json-schema-to-zod';
+
+/**
+ * Generates `src/metaschema.ts` (the zod schema used to parse the operator
+ * definitions) directly from the JSON metaschema shipped in the
+ * `mql-specifications` submodule, so the zod definitions don't have to be
+ * maintained by hand.
+ *
+ * `json-schema-to-zod` expects a fully dereferenced schema and emits `z.any()`
+ * for any `$ref` it doesn't understand, while `json-refs` cannot inline the
+ * recursive `Argument.arguments` reference. We therefore dereference the
+ * `#/definitions/*` references ourselves, breaking reference cycles by
+ * replacing the recursive reference with an empty schema (which becomes
+ * `z.any()`).
+ */
+
+type JsonSchema = Record<string, unknown>;
+
+const schemaFile = path.resolve(
+  __dirname,
+  '..',
+  'mql-specifications',
+  'schemas',
+  'operator.json',
+);
+const outputFile = path.resolve(__dirname, 'metaschema.ts');
+
+const root = JSON.parse(fs.readFileSync(schemaFile, 'utf8')) as {
+  definitions: Record<string, JsonSchema>;
+};
+const definitions = root.definitions;
+
+function dereference(node: unknown, stack: readonly string[]): unknown {
+  if (Array.isArray(node)) {
+    return node.map((item) => dereference(item, stack));
+  }
+
+  if (node && typeof node === 'object') {
+    const ref = (node as JsonSchema).$ref;
+    if (typeof ref === 'string') {
+      const match = /^#\/definitions\/(.+)$/.exec(ref);
+      if (!match) {
+        throw new Error(`Unsupported $ref: ${ref}`);
+      }
+      const name = match[1];
+      if (stack.includes(name)) {
+        // Break the reference cycle. An empty schema is converted to
+        // `z.any()`, which accepts the recursive structure at runtime.
+        return {};
+      }
+      if (!definitions[name]) {
+        throw new Error(`Unknown definition: ${name}`);
+      }
+      return dereference(definitions[name], [...stack, name]);
+    }
+
+    return Object.fromEntries(
+      Object.entries(node).map(([key, value]) => [
+        key,
+        dereference(value, stack),
+      ]),
+    );
+  }
+
+  return node;
+}
+
+const operatorSchema = dereference(definitions.Operator, ['Operator']);
+
+const code = jsonSchemaToZod(operatorSchema as JsonSchema, {
+  name: 'Operator',
+  module: 'esm',
+});
+
+fs.writeFileSync(outputFile, code, 'utf8');

packages/mql-typescript/src/cli.ts

@@ -6,28 +6,31 @@ import { DriverSchemaGenerator } from './driverSchema/driver-schema-generator';
 import type { GeneratorBase } from './generator';
 
 async function main() {
+  const filterOptions = {
+    category: {
+      type: 'string',
+      choices: ['accumulator', 'expression', 'query', 'search', 'stage'],
+      description:
+        'The category of the operator to generate. If not provided, all categories will be used.',
+    },
+    operator: {
+      type: 'string',
+      description:
+        'The operator to generate. If not provided, all operators will be used.',
+    },
+  } as const;
+
   const argv = await yargs(hideBin(process.argv))
     .command('schema', 'Generates schema from the php driver definitions')
     .command(
       'tests',
       'Generates tests from the php driver definitions and the docs examples',
+      filterOptions,
     )
     .command(
       'driver-schema',
       'Updates the php driver definitions with the schema for the tests',
-      {
-        category: {
-          type: 'string',
-          choices: ['accumulator', 'expression', 'query', 'search', 'stage'],
-          description:
-            'The category of the operator to update. If not provided, all categories will be updated.',
-        },
-        operator: {
-          type: 'string',
-          description:
-            'The operator to update the schema for. If not provided, all operators will be updated.',
-        },
-      },
+      filterOptions,
     )
     .demandCommand(1, 'A command must be provided')
     .help().argv;

packages/mql-typescript/src/driverSchema/driver-schema-generator.ts

@@ -11,11 +11,13 @@ type TestType = NonNullable<typeof Operator._output.tests>[number];
 export class DriverSchemaGenerator extends GeneratorBase {
   private async getSchemaFromDocs(test: TestType): Promise<object | string> {
     if (!test.link) {
-      console.error(`No docs reference found for ${test.name}`);
+      console.error(`No docs reference found for ${String(test.name)}`);
       return '// TODO: No docs reference found';
     }
-    if (!test.pipeline) {
-      console.error(`No pipeline found for ${test.name} at ${test.link}`);
+    if (!('pipeline' in test) || !test.pipeline) {
+      console.error(
+        `No pipeline found for ${String(test.name)} at ${test.link}`,
+      );
       return '// TODO: No pipeline found';
     }
 
@@ -24,7 +26,7 @@ export class DriverSchemaGenerator extends GeneratorBase {
 
     if (!schema) {
       console.error(
-        `Could not extract schema for ${test.name} at ${test.link}`,
+        `Could not extract schema for ${String(test.name)} at ${test.link}`,
       );
       return '// TODO: No schema found in docs';
     }
@@ -45,8 +47,13 @@ export class DriverSchemaGenerator extends GeneratorBase {
     test: TestType;
     rawYaml: { tests: { name: string; schema: object | string }[] };
   }): Promise<void> {
+    if (test.name === undefined) {
+      return;
+    }
+
+    const testName = test.name;
     const yamlTest = rawYaml.tests.find(
-      (t: { name: string }) => t.name === test.name,
+      (t: { name: string }) => t.name === testName,
     );
 
     if (!yamlTest) {
@@ -57,7 +64,7 @@ export class DriverSchemaGenerator extends GeneratorBase {
     }
 
     yamlTest.schema =
-      getStaticSchema({ category, operator, test: test.name }) ??
+      getStaticSchema({ category, operator, test: testName }) ??
       (await this.getSchemaFromDocs(test));
   }
 
@@ -86,7 +93,7 @@ export class DriverSchemaGenerator extends GeneratorBase {
           lineWidth: -1,
         });
 
-        updatedYaml = `# $schema: ../schema.json\n${updatedYaml}`;
+        updatedYaml = `# $schema: ../../schemas/operator.json\n${updatedYaml}`;
 
         await fs.writeFile(operator.path, updatedYaml, 'utf8');
       }

packages/mql-typescript/src/driverSchema/static-schemas.ts

@@ -92,6 +92,35 @@ const inventorySchema: SchemaInfo = {
   },
 };
 
+// The docs examples for these operators live on pages with several sibling
+// examples, so the crawler cannot reliably infer their collection schema.
+const zipCodeSchema: SchemaInfo = {
+  collectionName: 'addressBook',
+  schema: {
+    _id: { types: [{ bsonType: 'Int32' }] },
+    address: { types: [{ bsonType: 'String' }] },
+    zipCode: {
+      types: [
+        { bsonType: 'String' },
+        { bsonType: 'Int32' },
+        { bsonType: 'Double' },
+        { bsonType: 'Array', types: [{ bsonType: 'String' }] },
+      ],
+    },
+  },
+};
+
+const salesByDateSchema: SchemaInfo = {
+  collectionName: 'sales',
+  schema: {
+    _id: { types: [{ bsonType: 'Int32' }] },
+    item: { types: [{ bsonType: 'String' }] },
+    price: { types: [{ bsonType: 'Decimal128' }] },
+    quantity: { types: [{ bsonType: 'Int32' }] },
+    date: { types: [{ bsonType: 'Date' }] },
+  },
+};
+
 const mflixMoviesSchema: SchemaInfo = {
   collectionName: 'movies',
   schema: {
@@ -1415,7 +1444,62 @@ type SchemaMap = {
 };
 
 const staticSchemas: SchemaMap = {
+  accumulator: {
+    concatArrays: {
+      ['Warehouse collection']: {
+        collectionName: 'warehouses',
+        schema: {
+          instock: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+          ordered: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+        },
+      },
+    },
+    setUnion: {
+      ['Flowers collection']: {
+        collectionName: 'flowers',
+        schema: {
+          flowerFieldA: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+          flowerFieldB: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+        },
+      },
+    },
+  },
   expression: {
+    concatArrays: {
+      Example: {
+        collectionName: 'warehouses',
+        schema: {
+          instock: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+          ordered: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+        },
+      },
+    },
+    setUnion: {
+      Example: {
+        collectionName: 'flowers',
+        schema: {
+          flowerFieldA: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+          flowerFieldB: {
+            types: [{ bsonType: 'Array', types: [{ bsonType: 'String' }] }],
+          },
+        },
+      },
+    },
+    serializeEJSON: mflixMoviesSchema,
     dateFromParts: {
       Example: {
         collectionName: 'sales',
@@ -1496,6 +1580,15 @@ const staticSchemas: SchemaMap = {
       },
     },
     exists: inventorySchema,
+    gt: inventorySchema,
+    gte: inventorySchema,
+    lt: inventorySchema,
+    lte: inventorySchema,
+    nin: inventorySchema,
+    type: {
+      ['Querying by Data Type']: zipCodeSchema,
+      ['Querying by Array Type']: zipCodeSchema,
+    },
     geoIntersects: geoPolygonSchema,
     geoWithin: geoPolygonSchema,
     jsonSchema: {
@@ -1563,6 +1656,22 @@ const staticSchemas: SchemaMap = {
     wildcard: mflixMoviesSchema,
   },
   stage: {
+    project: {
+      // The docs page for this example shares its insertion code with sibling
+      // examples, so the crawler cannot reliably infer the {x, y} collection.
+      'Project New Array Fields': {
+        collectionName: 'TestCollection',
+        schema: {
+          _id: { types: [{ bsonType: 'ObjectId' }] },
+          x: { types: [{ bsonType: 'Int32' }] },
+          y: { types: [{ bsonType: 'Int32' }] },
+        },
+      },
+    },
+    group: {
+      ['Calculate Count Sum and Average']: salesByDateSchema,
+      ['Group by null']: salesByDateSchema,
+    },
     sort: {
       collectionName: 'users',
       schema: {