diff --git a/src/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.ts b/src/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.ts index 1bfca85f..5a207a5a 100644 --- a/src/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.ts +++ b/src/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.ts @@ -244,16 +244,21 @@ function validateBasicString(object: any, s ? undefined : Math.max(schema.maxLength, minLength); - if (object.length < minLength) { + // return early when the code points cannot possibly violate the length constraints + if (object.length >= minLength * 2 && (maxLength == null || object.length <= maxLength)) + return true; + + const length = getCodePointLength(object); + if (length < minLength) { if (minLength === maxLength) - throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${object.length}`); + throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${length}`); - throw new TechnicalValidationError(`Expected at least ${minLength} characters but got ${object.length}`); - } else if (maxLength != null && object.length > maxLength) { + throw new TechnicalValidationError(`Expected at least ${minLength} characters but got ${length}`); + } else if (maxLength != null && length > maxLength) { if (minLength === maxLength) - throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${object.length}`); + throw new TechnicalValidationError(`Expected exactly ${minLength} characters but got ${length}`); - throw new TechnicalValidationError(`Expected at most ${maxLength} characters but got ${object.length}`); + throw new TechnicalValidationError(`Expected at most ${maxLength} characters but got ${length}`); } return true; @@ -325,3 +330,23 @@ function validateImmutableType(value: any, type: GbnfJsonSchemaImmutableType) { throw new TechnicalValidationError(`Unknown immutable type ${JSON.stringify(type)}`); } + + +const highSurrogateRegex = /[\uD800-\uDBFF]/; +function getCodePointLength(value: string): number { + const units = value.length; + + if (!highSurrogateRegex.test(value)) + return units; + + let length = units; + for (let i = 0; i < units - 1; i++) { + if ((value.charCodeAt(i) & 0xFC00) !== 0xD800 || (value.charCodeAt(i + 1) & 0xFC00) !== 0xDC00) + continue; + + length--; + i++; + } + + return length; +} diff --git a/test/standalone/llamaEvaluator/LlamaGrammar.test.ts b/test/standalone/llamaEvaluator/LlamaGrammar.test.ts index cab7d15c..142bb115 100644 --- a/test/standalone/llamaEvaluator/LlamaGrammar.test.ts +++ b/test/standalone/llamaEvaluator/LlamaGrammar.test.ts @@ -1665,6 +1665,74 @@ describe("grammar for JSON schema", () => { }); describe("string options", () => { + test.each([ + ["a", 1], + ["δΈ­", 1], + ["πŸ˜€", 1], + ["e\u0301", 2], + ["πŸ‘©\u200dπŸ’»", 3], + ["aπŸ˜€", 2] + ] as const)("length counts Unicode code points for %s", async (value, length) => { + const llama = await getTestLlama(); + const grammar = new LlamaJsonSchemaGrammar(llama, { + type: "string", + minLength: length, + maxLength: length + }); + + expect(grammar.parse(JSON.stringify(value))).to.equal(value); + expect(testGrammar(grammar, value)).to.equal(true); + + const minGrammar = new LlamaJsonSchemaGrammar(llama, { + type: "string", + minLength: length + 1 + }); + expect(() => minGrammar.parse(JSON.stringify(value))) + .to.throw(`Expected at least ${length + 1} characters but got ${length}`); + expect(testGrammar(minGrammar, value)).to.equal(false); + + const maxGrammar = new LlamaJsonSchemaGrammar(llama, { + type: "string", + maxLength: length - 1 + }); + expect(() => maxGrammar.parse(JSON.stringify(value))) + .to.throw(length === 1 + ? "Expected exactly 0 characters but got 1" + : `Expected at most ${length - 1} characters but got ${length}`); + expect(testGrammar(maxGrammar, value)).to.equal(false); + }); + + test("length counts decoded Unicode escape pairs", async () => { + const llama = await getTestLlama(); + const grammar = new LlamaJsonSchemaGrammar(llama, { + type: "string", + maxLength: 1 + }); + + expect(grammar.parse('"\\ud83d\\ude00"')).to.equal("πŸ˜€"); + expect(() => grammar.parse('"\\ud83d\\ude00a"')) + .to.throw("Expected at most 1 characters but got 2"); + }); + + test("length counts Unicode code points in object properties", async () => { + const llama = await getTestLlama(); + const grammar = new LlamaJsonSchemaGrammar(llama, { + type: "object", + properties: { + text: { + type: "string", + maxLength: 1 + } + } + }); + const value = {text: "πŸ˜€"}; + + expect(grammar.parse(JSON.stringify(value))).to.eql(value); + expect(testGrammar(grammar, value)).to.equal(true); + expect(() => grammar.parse(JSON.stringify({text: "πŸ˜€a"}))) + .to.throw("Expected at most 1 characters but got 2"); + }); + test("minLength", async () => { const llama = await getTestLlama(); const grammar = new LlamaJsonSchemaGrammar(llama, {