fix(LlamaJsonSchemaGrammar.parse): count Unicode code points for minLength and maxLength checks

src/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.ts

@@ -244,16 +244,21 @@ function validateBasicString<T extends GbnfJsonBasicStringSchema>(object: any, s
         ? undefined
         : Math.max(schema.maxLength, minLength);
 
-    if (object.length < minLength) {
+    // return early when the code points cannot possibly violate the length constraints
+    if (object.length >= minLength * 2 && (maxLength == null || object.length <= maxLength))
+        return true;
+
+    const length = getCodePointLength(object);
+    if (length < minLength) {
         if (minLength === maxLength)
-            throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${object.length}`);
+            throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${length}`);
 
-        throw new TechnicalValidationError(`Expected at least ${minLength} characters but got ${object.length}`);
-    } else if (maxLength != null && object.length > maxLength) {
+        throw new TechnicalValidationError(`Expected at least ${minLength} characters but got ${length}`);
+    } else if (maxLength != null && length > maxLength) {
         if (minLength === maxLength)
-            throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${object.length}`);
+            throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${length}`);
 
-        throw new TechnicalValidationError(`Expected at most ${maxLength} characters but got ${object.length}`);
+        throw new TechnicalValidationError(`Expected at most ${maxLength} characters but got ${length}`);
     }
 
     return true;
@@ -325,3 +330,23 @@ function validateImmutableType(value: any, type: GbnfJsonSchemaImmutableType) {
 
     throw new TechnicalValidationError(`Unknown immutable type ${JSON.stringify(type)}`);
 }
+
+
+const highSurrogateRegex = /[\uD800-\uDBFF]/;
+function getCodePointLength(value: string): number {
+    const units = value.length;
+
+    if (!highSurrogateRegex.test(value))
+        return units;
+
+    let length = units;
+    for (let i = 0; i < units - 1; i++) {
+        if ((value.charCodeAt(i) & 0xFC00) !== 0xD800 || (value.charCodeAt(i + 1) & 0xFC00) !== 0xDC00)
+            continue;
+
+        length--;
+        i++;
+    }
+
+    return length;
+}

test/standalone/llamaEvaluator/LlamaGrammar.test.ts

@@ -1665,6 +1665,74 @@ describe("grammar for JSON schema", () => {
     });
 
     describe("string options", () => {
+        test.each([
+            ["a", 1],
+            ["中", 1],
+            ["😀", 1],
+            ["e\u0301", 2],
+            ["👩\u200d💻", 3],
+            ["a😀", 2]
+        ] as const)("length counts Unicode code points for %s", async (value, length) => {
+            const llama = await getTestLlama();
+            const grammar = new LlamaJsonSchemaGrammar(llama, {
+                type: "string",
+                minLength: length,
+                maxLength: length
+            });
+
+            expect(grammar.parse(JSON.stringify(value))).to.equal(value);
+            expect(testGrammar(grammar, value)).to.equal(true);
+
+            const minGrammar = new LlamaJsonSchemaGrammar(llama, {
+                type: "string",
+                minLength: length + 1
+            });
+            expect(() => minGrammar.parse(JSON.stringify(value)))
+                .to.throw(`Expected at least ${length + 1} characters but got ${length}`);
+            expect(testGrammar(minGrammar, value)).to.equal(false);
+
+            const maxGrammar = new LlamaJsonSchemaGrammar(llama, {
+                type: "string",
+                maxLength: length - 1
+            });
+            expect(() => maxGrammar.parse(JSON.stringify(value)))
+                .to.throw(length === 1
+                    ? "Expected exactly 0 characters but got 1"
+                    : `Expected at most ${length - 1} characters but got ${length}`);
+            expect(testGrammar(maxGrammar, value)).to.equal(false);
+        });
+
+        test("length counts decoded Unicode escape pairs", async () => {
+            const llama = await getTestLlama();
+            const grammar = new LlamaJsonSchemaGrammar(llama, {
+                type: "string",
+                maxLength: 1
+            });
+
+            expect(grammar.parse('"\\ud83d\\ude00"')).to.equal("😀");
+            expect(() => grammar.parse('"\\ud83d\\ude00a"'))
+                .to.throw("Expected at most 1 characters but got 2");
+        });
+
+        test("length counts Unicode code points in object properties", async () => {
+            const llama = await getTestLlama();
+            const grammar = new LlamaJsonSchemaGrammar(llama, {
+                type: "object",
+                properties: {
+                    text: {
+                        type: "string",
+                        maxLength: 1
+                    }
+                }
+            });
+            const value = {text: "😀"};
+
+            expect(grammar.parse(JSON.stringify(value))).to.eql(value);
+            expect(testGrammar(grammar, value)).to.equal(true);
+            expect(() => grammar.parse(JSON.stringify({text: "😀a"})))
+                .to.throw("Expected at most 1 characters but got 2");
+        });
+
         test("minLength", async () => {
             const llama = await getTestLlama();
             const grammar = new LlamaJsonSchemaGrammar(llama, {