From 8bc309f2d08bc46e1dc9c417a07a7f24ad153e20 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Tue, 3 Nov 2026 09:52:03 -0800 Subject: [PATCH 1/3] Add dependency-free TypeScript token providers and client adapters Preserve legacy HTTP factories while adding opt-in token providers and configured Chat Completions/Responses clients with multimodal prompts and shared metadata/filtering behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- site/src/docs/typescript/basic-usage.md | 85 +++++++- typescript/README.md | 12 ++ typescript/metadata.md | 6 + typescript/src/model.ts | 230 +++++++++++++++++--- typescript/test/metadata.test.ts | 86 +++++++- typescript/tests/model.test.mjs | 274 +++++++++++++++++++++++- 6 files changed, 658 insertions(+), 35 deletions(-) diff --git a/site/src/docs/typescript/basic-usage.md b/site/src/docs/typescript/basic-usage.md index d83db7d4..b77874f4 100644 --- a/site/src/docs/typescript/basic-usage.md +++ b/site/src/docs/typescript/basic-usage.md @@ -53,24 +53,30 @@ export interface TypeChatLanguageModel { retryPauseMs?: number; /** * Obtains a completion from the language model for the given prompt. - * @param prompt The prompt string. + * @param prompt The prompt string or role-tagged prompt sections. */ - complete(prompt: string): Promise>; + complete(prompt: string | PromptSection[]): Promise>; } ``` then you should be able to try TypeChat out with such a model. The key thing here is that only `complete` is required. -`complete` is just a function that takes a `string` and eventually returns a `string` if all goes well. +`complete` takes a string or an array of prompt sections and eventually returns text wrapped in a +`Result` if all goes well. For convenience, TypeChat provides two functions out of the box to connect to the OpenAI API and Azure's OpenAI Services. You can call these directly. ```ts -export function createOpenAILanguageModel(apiKey: string, model: string, endPoint?: string): TypeChatLanguageModel; - -export function createAzureOpenAILanguageModel(apiKey: string, endPoint: string): TypeChatLanguageModel; +export function createOpenAILanguageModel( + apiKey: string | undefined, model: string, endPoint?: string, + org?: string, options?: LanguageModelOptions +): TypeChatLanguageModel; + +export function createAzureOpenAILanguageModel( + apiKey: string | undefined, endPoint: string, options?: LanguageModelOptions +): TypeChatLanguageModel; ``` For even more convenience, TypeChat also provides a function to infer whether you're using OpenAI or Azure OpenAI. @@ -82,6 +88,73 @@ export function createLanguageModel(env: Record): Ty With `createLanguageModel`, you can populate your environment variables and pass them in. Based on whether `OPENAI_API_KEY` or `AZURE_OPENAI_API_KEY` is set, you'll get a model of the appropriate type. +#### Using a bearer-token provider + +Pass `undefined` for the API key and supply your existing synchronous or asynchronous +token provider in the options: + +```ts +const model = createAzureOpenAILanguageModel(undefined, endpoint, { + tokenProvider +}); +``` + +`TokenProvider` is `() => string | Promise`. Return the raw access-token string, +without a `Bearer ` prefix. The provider owns credential acquisition, caching, renewal, +and cleanup; TypeChat adds no Azure Identity dependency. It invokes the provider for each +HTTP attempt, including retries, and sends bearer authorization without an `api-key` +header. API keys and providers are mutually exclusive. Empty or invalid tokens return +an error without sending a request. Provider exceptions follow the HTTP model's existing +retry/error policy. + +The same option works with `createOpenAILanguageModel`, including its Responses mode: + +```ts +const model = createOpenAILanguageModel(undefined, deployment, responsesEndpoint, "", { + tokenProvider, + useResponsesApi: true +}); +``` + +The HTTP factories retain their proxy, timeout, response-size, and retry controls. +The HTTP request timeout does not bound credential acquisition inside your provider. +Existing string-key calls, environment handling, defaults, and legacy headers are unchanged; +`AZURE_OPENAI_API_KEY=identity` does not activate a provider. + +#### Reusing a configured client + +Adapt an existing OpenAI/AzureOpenAI SDK client, or a structurally compatible asynchronous +client, without importing an SDK into TypeChat: + +```ts +const model = createOpenAILanguageModelFromClient(client, modelOrDeployment); +const responsesModel = createOpenAILanguageModelFromClient(client, modelOrDeployment, { + useResponsesApi: true +}); +``` + +Chat Completions is the default and calls `client.chat.completions.create`. Responses mode +is explicit and calls `client.responses.create`; a Chat-only client does not need a +`responses` property. TypeChat requests non-streaming output at temperature zero, with +`n=1` only for Chat Completions. + +The adapter exposes only `complete`, retaining the supplied client's endpoint, API version, +authentication, retry/timeout settings, and ownership. It does not configure or close the +client or add another network-control layer. Configure proxies, timeouts, retries, and +response-size safeguards on that client, not on the adapter. Client exceptions and +cancellation propagate unchanged; missing/malformed output and filtered/refused content +return TypeChat errors. + +String prompts and text/image user messages are supported. Bare strings in content arrays +become text parts; images retain URLs/data URLs and detail levels. Responses mode maps them +to `input_text`/`input_image` parts, defaulting omitted image detail to `"auto"`. System and +assistant messages support text only; unsupported content is reported rather than dropped. +Neither path fetches image references separately. + +Pass either model to `createJsonTranslator` as usual. Schema validation, repair, and +reported response metadata/filtering behavior are preserved. SDK metadata covers the +resolved responses visible to TypeChat, not HTTP attempts hidden inside the client's retries. + #### Using OpenAI-compatible gateways TypeChat works with any service that exposes the OpenAI Chat Completions API. diff --git a/typescript/README.md b/typescript/README.md index defb1fcd..c1b46718 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -45,6 +45,18 @@ To see TypeChat in action, we recommend exploring the [TypeChat example projects To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started. +## Reusing model authentication + +The HTTP model factories accept a synchronous or asynchronous `tokenProvider` option +with `undefined` for the API key. Existing string-key and environment-based calls are +unchanged. For an already configured OpenAI/AzureOpenAI or compatible client, use +`createOpenAILanguageModelFromClient`, optionally with `{ useResponsesApi: true }`. + +These paths add no SDK dependency to TypeChat. See the +[token-provider example](../site/src/docs/typescript/basic-usage.md#using-a-bearer-token-provider) +and [configured-client example](../site/src/docs/typescript/basic-usage.md#reusing-a-configured-client) +for multimodal prompts, network-control ownership, and error behavior. + ## Accessing failed output Failed results include a `message` and may also include `data: unknown` and `responseText: string`. diff --git a/typescript/metadata.md b/typescript/metadata.md index a5c89b3e..e61e1403 100644 --- a/typescript/metadata.md +++ b/typescript/metadata.md @@ -28,6 +28,12 @@ response that is filtered or refused returns a concise error rather than putting provider body or refusal text in the error message. Filtering details remain in metadata, and available text remains in `responseText`. +Models adapted by `createOpenAILanguageModelFromClient` use the same reported-usage and +filtering parser for Chat Completions and Responses output. Their records cover resolved +SDK responses visible to TypeChat, including translator repair, not requests/retries hidden +inside the supplied client. Client exceptions propagate instead of creating fabricated +response records. HTTP token-provider models retain the existing per-HTTP-response accounting. + `metadata.usage` sums **reported counts only**, once per record: input/output/total 13/7/20 plus 5/3/8 yields 18/10/28. Each corresponding `metadata.usageReporting` field is `"complete"` if every record reports that count, otherwise `"partial"`. If no record reports a count, its total and diff --git a/typescript/src/model.ts b/typescript/src/model.ts index e550f66d..03d07be1 100644 --- a/typescript/src/model.ts +++ b/typescript/src/model.ts @@ -57,7 +57,8 @@ export type ImageUrl = { * Represents a AI language model that can complete prompts. TypeChat uses an implementation of this * interface to communicate with an AI service that can translate natural language requests to JSON * instances according to a provided schema. The `createLanguageModel`, `createOpenAILanguageModel`, - * and `createAzureOpenAILanguageModel` functions create instances of this interface. + * `createAzureOpenAILanguageModel`, and `createOpenAILanguageModelFromClient` functions create + * instances of this interface. */ export interface TypeChatLanguageModel { /** @@ -92,6 +93,9 @@ export interface TypeChatLanguageModel { complete(prompt: string | PromptSection[]): Promise>; } +/** Supplies a bearer token; credential acquisition, caching, and renewal remain caller-owned. */ +export type TokenProvider = () => string | Promise; + /** * Optional settings accepted by the language model factory functions. */ @@ -115,9 +119,52 @@ export interface LanguageModelOptions { * Selects the OpenAI API variant. When `true`, the Responses API (`/v1/responses`) is used * regardless of the endpoint URL; when `false`, the Chat Completions API is used. When omitted * (default), the variant is inferred from the endpoint URL (a path ending in `/responses` - * selects the Responses API). Only applies to `createOpenAILanguageModel`. + * selects the Responses API). Applies to `createOpenAILanguageModel`; configured-client adapters + * also accept this option but default to Chat Completions without endpoint inference. */ useResponsesApi?: boolean; + + /** + * Supplies a bearer token for each HTTP attempt, including retries. Pass `undefined` for the + * factory's `apiKey` argument when using this option. API keys and providers are mutually exclusive. + */ + tokenProvider?: TokenProvider; +} + +type ClientMessage = + | { role: "user"; content: string | (TextPromptContent | ImagePromptContent)[] } + | { role: "system"; content: string | TextPromptContent[] } + | { role: "assistant"; content: string | TextPromptContent[] }; + +type ClientResponseContent = + | { type: "input_text"; text: string } + | { type: "input_image"; image_url: string; detail: NonNullable }; + +/** Structural contract for a caller-owned asynchronous Chat Completions client. */ +export interface ChatCompletionClient { + chat: { + completions: { + create(request: { + model: string; + messages: ClientMessage[]; + temperature: number; + n: number; + stream: false; + }): PromiseLike; + }; + }; +} + +/** Structural contract for a caller-owned asynchronous Responses client. */ +export interface ResponsesClient { + responses: { + create(request: { + model: string; + input: string | { role: PromptSection["role"]; content: string | ClientResponseContent[] }[]; + temperature: number; + stream: false; + }): PromiseLike; + }; } /** @@ -180,7 +227,7 @@ export function createLanguageModel(env: Record): Ty * the `/v1/responses` request/response format; otherwise the Chat Completions format is used. * The Responses API is auto-detected when the endpoint URL path ends with `/responses` * (e.g. `https://api.openai.com/v1/responses`). - * @param apiKey The OpenAI API key. + * @param apiKey The OpenAI API key, or `undefined` when using `options.tokenProvider`. * @param model The model name (e.g. `"gpt-4o"`). * @param endPoint The URL of the OpenAI REST API endpoint. Defaults to * `"https://api.openai.com/v1/chat/completions"`. Supply a `/responses` URL to use the @@ -190,7 +237,13 @@ export function createLanguageModel(env: Record): Ty * See {@link LanguageModelOptions}. * @returns An instance of `TypeChatLanguageModel`. */ -export function createOpenAILanguageModel(apiKey: string, model: string, endPoint = "https://api.openai.com/v1/chat/completions", org = "", options?: LanguageModelOptions): TypeChatLanguageModel { +export function createOpenAILanguageModel(apiKey: string | undefined, model: string, endPoint = "https://api.openai.com/v1/chat/completions", org = "", options?: LanguageModelOptions): TypeChatLanguageModel { + const tokenProvider = options?.tokenProvider; + validateAuthentication(apiKey, tokenProvider); + if (apiKey === undefined) { + return createFetchLanguageModel(endPoint, { "OpenAI-Organization": org }, { model }, + resolveProxySettings(options), options?.useResponsesApi ?? isResponsesApiUrl(endPoint), tokenProvider); + } const headers = { "Authorization": `Bearer ${apiKey}`, "OpenAI-Organization": org @@ -204,11 +257,16 @@ export function createOpenAILanguageModel(apiKey: string, model: string, endPoin * @param endPoint The URL of the OpenAI REST API endpoint. The URL must be in the format * "https://{your-resource-name}.openai.azure.com/openai/deployments/{your-deployment-name}/chat/completions?api-version={API-version}". * Example deployment names are "gpt-35-turbo" and "gpt-4". An example API versions is "2023-05-15". - * @param apiKey The Azure OpenAI API key. + * @param apiKey The Azure OpenAI API key, or `undefined` when using `options.tokenProvider`. * @param options Optional settings such as a proxy URL. See {@link LanguageModelOptions}. * @returns An instance of `TypeChatLanguageModel`. */ -export function createAzureOpenAILanguageModel(apiKey: string, endPoint: string, options?: LanguageModelOptions): TypeChatLanguageModel { +export function createAzureOpenAILanguageModel(apiKey: string | undefined, endPoint: string, options?: LanguageModelOptions): TypeChatLanguageModel { + const tokenProvider = options?.tokenProvider; + validateAuthentication(apiKey, tokenProvider); + if (apiKey === undefined) { + return createFetchLanguageModel(endPoint, {}, {}, resolveProxySettings(options), false, tokenProvider); + } const headers = { // Needed when using managed identity "Authorization": `Bearer ${apiKey}`, @@ -218,6 +276,117 @@ export function createAzureOpenAILanguageModel(apiKey: string, endPoint: string, return createFetchLanguageModel(endPoint, headers, {}, resolveProxySettings(options)); } +/** + * Adapts a configured asynchronous client without importing an SDK, changing its settings, closing + * it, or adding retries/timeouts. Client exceptions propagate. Only `complete` is exposed; configure + * network controls on the supplied client. Responses mode is explicit, not inferred from the client. + */ +export function createOpenAILanguageModelFromClient(client: ChatCompletionClient, model: string, options?: { useResponsesApi?: false }): Pick; +export function createOpenAILanguageModelFromClient(client: ResponsesClient, model: string, options: { useResponsesApi: true }): Pick; +export function createOpenAILanguageModelFromClient(client: ChatCompletionClient & ResponsesClient, model: string, options: Pick): Pick; +export function createOpenAILanguageModelFromClient(client: ChatCompletionClient | ResponsesClient, model: string, options?: Pick): Pick { + if (!model.trim()) { + throw new Error("The client model or deployment name must not be empty."); + } + const responsesApi = options?.useResponsesApi ?? false; + const finish = (json: unknown) => { + const metadata = responseMetadata(json, responsesApi); + return withResponseMetadata(completionResult(json, responsesApi, metadata), [metadata]); + }; + if (responsesApi) { + if (!("responses" in client)) { + throw new Error("Responses mode requires a client with responses.create."); + } + return { + async complete(prompt) { + const messages = clientMessages(prompt); + if (!messages.success) return messages; + const input = typeof prompt === "string" ? prompt : messages.data.map(message => ({ + role: message.role, + content: typeof message.content === "string" ? message.content : + message.content.map((part): ClientResponseContent => part.type === "text" ? + { type: "input_text", text: part.text } : + { type: "input_image", image_url: part.image_url.url, detail: part.image_url.detail ?? "auto" }) + })); + return finish(await client.responses.create({ model, input, temperature: 0, stream: false })); + } + }; + } + if (!("chat" in client)) { + throw new Error("Chat Completions mode requires a client with chat.completions.create."); + } + return { + async complete(prompt) { + const messages = clientMessages(prompt); + if (!messages.success) return messages; + return finish(await client.chat.completions.create({ + model, messages: messages.data, temperature: 0, n: 1, stream: false + })); + } + }; +} + +function validateAuthentication(apiKey: string | undefined, provider: TokenProvider | undefined): void { + if (provider !== undefined && typeof provider !== "function") { + throw new TypeError("Token provider must be a function."); + } + if (apiKey !== undefined && provider !== undefined) { + throw new Error("Specify either an API key or a token provider, not both."); + } + if (apiKey === undefined && provider === undefined) { + throw new Error("Specify an API key or a token provider."); + } +} + +function clientMessages(prompt: string | PromptSection[]): Result { + const sections: PromptSection[] = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt; + const messages: ClientMessage[] = []; + for (const { role, content } of sections) { + if (role !== "user" && role !== "system" && role !== "assistant") { + return error("Unsupported prompt role."); + } + if (typeof content === "string") { + messages.push({ role, content }); + continue; + } + if (!Array.isArray(content)) { + return error("Prompt content must be a string or an array of text/image parts."); + } + const parts: (TextPromptContent | ImagePromptContent)[] = []; + for (const part of content) { + if (typeof part === "string") { + parts.push({ type: "text", text: part }); + continue; + } + const value = record(part); + if (value?.type === "text" && typeof value.text === "string") { + parts.push({ type: "text", text: value.text }); + continue; + } + const image = record(value?.image_url); + const detail = image?.detail; + if (value?.type !== "image_url" || typeof image?.url !== "string" || + detail !== undefined && detail !== "auto" && detail !== "low" && detail !== "high") { + return error("Unsupported or malformed text/image prompt part."); + } + parts.push({ type: "image_url", image_url: { url: image.url, ...(detail !== undefined ? { detail } : {}) } }); + } + if (role === "user") { + messages.push({ role, content: parts }); + } else { + const textParts: TextPromptContent[] = []; + for (const part of parts) { + if (part.type !== "text") { + return error("Image prompt parts are supported only in user messages."); + } + textParts.push(part); + } + messages.push({ role, content: textParts }); + } + } + return success(messages); +} + /** * Default per-request timeout applied when a model does not set {@link TypeChatLanguageModel.timeoutMs}. * Ten minutes matches common LLM client defaults, allowing slow completions while still preventing a @@ -235,7 +404,7 @@ const defaultMaxResponseBytes = 100 * 1024 * 1024; /** * Common OpenAI REST API endpoint encapsulation using the fetch API. */ -function createFetchLanguageModel(url: string, headers: object, defaultParams: object, proxy?: ProxySettings, responsesApi = false) { +function createFetchLanguageModel(url: string, headers: object, defaultParams: object, proxy?: ProxySettings, responsesApi = false, tokenProvider?: TokenProvider) { let dispatcherPromise: Promise | undefined; const model: TypeChatLanguageModel = { complete @@ -281,6 +450,13 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o } let response: Response; try { + if (tokenProvider) { + const token = await tokenProvider(); + if (typeof token !== "string" || !token.trim()) { + return finish(error("Token provider must return a nonempty string.")); + } + options.headers = { ...headers, "content-type": "application/json", "Authorization": `Bearer ${token}` }; + } response = await fetchWithTimeout(url, options, timeoutMs); } catch (e) { if (retryCount >= retryMaxAttempts) { @@ -296,26 +472,7 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o } const json = parsed.success ? parsed.data : undefined; if (response.ok) { - let content: unknown; - if (responsesApi) { - const output = record(json)?.output; - const message = Array.isArray(output) ? record(output.find(o => record(o)?.type === "message")) : undefined; - const parts = message?.content; - content = Array.isArray(parts) ? record(parts.find(c => record(c)?.type === "output_text"))?.text : undefined; - } else { - const choices = record(json)?.choices; - const choice = record(Array.isArray(choices) ? choices[0] : undefined); - content = record(choice?.message)?.content; - } - if (responses[responses.length - 1].filtering?.blocked) { - return finish(error("REST API response was filtered or refused.", - typeof content === "string" ? { responseText: content } : undefined)); - } - if (typeof content === "string") { - return finish(success(content)); - } else { - return finish(error(`REST API unexpected response format: ${JSON.stringify(json)}`)); - } + return finish(completionResult(json, responsesApi, responses[responses.length - 1])); } if (!isTransientHttpError(response.status) || retryCount >= retryMaxAttempts) { return finish(error(`REST API error ${response.status}: ${response.statusText}`)); @@ -326,6 +483,25 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o } } +function completionResult(json: unknown, responsesApi: boolean, metadata: TypeChatResponseMetadata): Result { + let content: unknown; + if (responsesApi) { + const output = record(json)?.output; + const message = Array.isArray(output) ? record(output.find(o => record(o)?.type === "message")) : undefined; + const parts = message?.content; + content = Array.isArray(parts) ? record(parts.find(c => record(c)?.type === "output_text"))?.text : undefined; + } else { + const choices = record(json)?.choices; + const choice = record(Array.isArray(choices) ? choices[0] : undefined); + content = record(choice?.message)?.content; + } + if (metadata.filtering?.blocked) { + return error("REST API response was filtered or refused.", + typeof content === "string" ? { responseText: content } : undefined); + } + return typeof content === "string" ? success(content) : error(`REST API unexpected response format: ${JSON.stringify(json)}`); +} + function record(value: unknown): Record | undefined { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; } diff --git a/typescript/test/metadata.test.ts b/typescript/test/metadata.test.ts index af63dc84..342cbdd8 100644 --- a/typescript/test/metadata.test.ts +++ b/typescript/test/metadata.test.ts @@ -3,7 +3,8 @@ import { test } from "node:test"; import { createOpenAILanguageModel, createAzureOpenAILanguageModel, createJsonTranslator, success, error, getData, withResponseMetadata, - Result, TypeChatLanguageModel, TypeChatJsonValidator, TypeChatResponseMetadata, PromptSection + Result, TypeChatLanguageModel, TypeChatJsonValidator, TypeChatResponseMetadata, PromptSection, + TokenProvider, ChatCompletionClient, ResponsesClient, createOpenAILanguageModelFromClient } from "../dist/index.js"; import { createTypeScriptJsonValidator } from "../dist/ts/index.js"; @@ -35,6 +36,89 @@ function translatorFor(results: Result[], schema = validator) { return { translator: createJsonTranslator(model, schema), calls: () => calls, prompts }; } +test("token-provider retries retain reported metadata without retaining credentials", async t => { + let calls = 0; + const tokens: (string | null)[] = []; + t.mock.method(globalThis, "fetch", async (_url: unknown, init?: RequestInit) => { + tokens.push(new Headers(init?.headers).get("Authorization")); + return ++calls === 1 ? + Response.json({ usage: { prompt_tokens: 13, completion_tokens: 7, total_tokens: 20 } }, { status: 429 }) : + Response.json({ choices: [{ message: { content: valid } }], usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } }); + }); + let tokenCount = 0; + const tokenProvider: TokenProvider = async () => `token-${++tokenCount}`; + const model = createOpenAILanguageModel(undefined, "test", "https://example.test/chat/completions", "", { tokenProvider }); + model.retryPauseMs = 0; + const result = await model.complete("request"); + assert.ok(result.success); + assert.deepEqual(tokens, ["Bearer token-1", "Bearer token-2"]); + assert.deepEqual(result.metadata?.responses, [{ usage: first }, { usage: second }]); + assert.deepEqual(result.metadata?.usage, { inputTokens: 18, outputTokens: 10, totalTokens: 28 }); + assert.ok(!JSON.stringify(result.metadata).includes("token-")); +}); + +for (const api of ["chat", "responses"] as const) { + const wireUsage = (input: number, output: number, total: number) => api === "responses" ? + { input_tokens: input, output_tokens: output, total_tokens: total } : + { prompt_tokens: input, completion_tokens: output, total_tokens: total }; + const body = (text: string, usage: unknown) => ({ + usage, + ...(api === "responses" ? { output: [{ type: "message", content: [{ type: "output_text", text }] }] } : + { choices: [{ message: { content: text } }] }) + }); + const fromClient = (replies: unknown[], requests: unknown[]) => { + const chat: ChatCompletionClient = { chat: { completions: { + async create(request) { requests.push(structuredClone(request)); return replies.shift(); } + } } }; + const responses: ResponsesClient = { responses: { + async create(request) { requests.push(structuredClone(request)); return replies.shift(); } + } }; + return api === "responses" ? createOpenAILanguageModelFromClient(responses, "test", { useResponsesApi: true }) : + createOpenAILanguageModelFromClient(chat, "test"); + }; + + test(`client ${api}: validation/repair retains metadata and multimodal context`, async () => { + const requests: unknown[] = []; + const model = fromClient([body(invalid, wireUsage(13, 7, 20)), body(valid, wireUsage(5, 3, 8))], requests); + const translator = createJsonTranslator(model, validator); + const preamble: PromptSection[] = [{ role: "user", content: [ + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } } + ] }]; + const result = await translator.translate("request", preamble); + assert.ok(result.success); + assert.deepEqual(result.data, { value: 42 }); + assert.deepEqual(result.metadata?.responses, [{ usage: first }, { usage: second }]); + assert.deepEqual(result.metadata?.usage, { inputTokens: 18, outputTokens: 10, totalTokens: 28 }); + assert.equal(requests.length, 2); + assert.ok(requests.every(request => JSON.stringify(request).includes("data:image/png;base64,AAAA"))); + }); + + test(`client ${api}: filtering blocks valid-looking JSON and preserves metadata`, async () => { + const requests: unknown[] = []; + const json = body(valid, wireUsage(13, 7, 20)); + const filtered = api === "responses" ? { + ...json, incomplete_details: { reason: "content_filter" }, + output: [{ type: "message", content: [{ type: "refusal", refusal: "declined" }] }] + } : { ...json, choices: [{ finish_reason: "content_filter", message: { content: valid, refusal: "declined" } }] }; + const model = fromClient([filtered], requests); + const result = await createJsonTranslator(model, validator).translate("request"); + assert.ok(!result.success); + assert.match(result.message, /filtered or refused/); + assert.ok(!result.message.includes("declined")); + assert.deepEqual(result.metadata?.usage, first); + assert.equal(result.metadata?.responses[0].filtering?.blocked, true); + assert.equal(requests.length, 1); + }); + + test(`client ${api}: malformed output retains reported-only usage`, async () => { + const model = fromClient([{ usage: wireUsage(0, 7, 7) }], []); + const result = await model.complete("request"); + assert.ok(!result.success); + assert.match(result.message, /unexpected response format/); + assert.deepEqual(result.metadata?.usage, { inputTokens: 0, outputTokens: 7, totalTokens: 7 }); + }); +} + for (const api of ["chat", "responses", "azure"] as const) { const factory = () => api === "azure" ? createAzureOpenAILanguageModel("test", "https://example.test/chat/completions") : createOpenAILanguageModel("test", "test", `https://example.test/${api === "responses" ? "responses" : "chat/completions"}`); diff --git a/typescript/tests/model.test.mjs b/typescript/tests/model.test.mjs index 6f93ddcc..fdda7490 100644 --- a/typescript/tests/model.test.mjs +++ b/typescript/tests/model.test.mjs @@ -13,7 +13,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; // Load the compiled module from dist -import { createOpenAILanguageModel, createAzureOpenAILanguageModel, createLanguageModel } from "../dist/index.js"; +import { + createOpenAILanguageModel, createAzureOpenAILanguageModel, createLanguageModel, + createOpenAILanguageModelFromClient +} from "../dist/index.js"; test("missing optional undici returns an actionable result for both API paths", async () => { const dir = mkdtempSync(join(tmpdir(), "typechat-no-undici-")); @@ -120,6 +123,275 @@ function teardownFetch() { mockResponses = []; } +describe("token-provider authentication", () => { + after(teardownFetch); + + const factories = [ + ["Azure Chat", "chat", provider => createAzureOpenAILanguageModel( + undefined, "https://example.test/chat/completions", { tokenProvider: provider })], + ["OpenAI Chat", "chat", provider => createOpenAILanguageModel( + undefined, "test-model", "https://example.test/chat/completions", "org-test", { tokenProvider: provider })], + ["OpenAI Responses", "responses", provider => createOpenAILanguageModel( + undefined, "test-model", "https://example.test/responses", "org-test", { tokenProvider: provider })] + ]; + + for (const [name, api, factory] of factories) { + const reply = () => api === "responses" ? makeResponsesAPIResponse("reply") : makeChatCompletionsResponse("reply"); + + test(`${name}: obtains a fresh synchronous token for each completion`, async () => { + setupFetch([reply(), reply()]); + let calls = 0; + const model = factory(() => `token-${++calls}`); + assert.deepEqual(await model.complete("first"), { success: true, data: "reply" }); + assert.deepEqual(await model.complete("second"), { success: true, data: "reply" }); + assert.equal(calls, 2); + assert.deepEqual(capturedRequests.map(request => request.options.headers.Authorization), ["Bearer token-1", "Bearer token-2"]); + assert.ok(capturedRequests.every(request => !("api-key" in request.options.headers))); + if (name.startsWith("OpenAI")) { + assert.equal(capturedRequests[0].options.headers["OpenAI-Organization"], "org-test"); + } + }); + + test(`${name}: awaits the provider again for each HTTP retry`, async () => { + setupFetch([makeErrorResponse(429, "Too Many Requests"), makeErrorResponse(500, "Internal Server Error"), reply()]); + let calls = 0; + const model = factory(async () => { + await Promise.resolve(); + return `token-${++calls}`; + }); + model.retryMaxAttempts = 2; + model.retryPauseMs = 0; + assert.deepEqual(await model.complete("request"), { success: true, data: "reply" }); + assert.equal(calls, 3); + assert.deepEqual(capturedRequests.map(request => request.options.headers.Authorization), + ["Bearer token-1", "Bearer token-2", "Bearer token-3"]); + }); + + test(`${name}: acquisition errors use the existing retry/failure policy`, async () => { + setupFetch([reply()]); + let calls = 0; + const model = factory(() => { + if (++calls === 1) throw new Error("credential unavailable"); + return "fresh-token"; + }); + model.retryMaxAttempts = 1; + model.retryPauseMs = 0; + assert.deepEqual(await model.complete("request"), { success: true, data: "reply" }); + assert.equal(calls, 2); + assert.equal(capturedRequests.length, 1); + + const failed = factory(async () => { throw new Error("credential unavailable"); }); + failed.retryMaxAttempts = 0; + assert.deepEqual(await failed.complete("request"), { + success: false, message: "REST API fetch error: credential unavailable" + }); + assert.equal(capturedRequests.length, 1); + }); + + for (const token of ["", " ", undefined, null, 42, {}]) { + test(`${name}: rejects invalid token ${JSON.stringify(token)} before fetch`, async () => { + setupFetch([]); + assert.deepEqual(await factory(() => token).complete("request"), { + success: false, message: "Token provider must return a nonempty string." + }); + assert.equal(capturedRequests.length, 0); + }); + } + + test(`${name}: does not silently retry unauthorized responses`, async () => { + setupFetch([makeErrorResponse(401, "Unauthorized")]); + let calls = 0; + const model = factory(() => { calls++; return "token"; }); + assert.deepEqual(await model.complete("request"), { success: false, message: "REST API error 401: Unauthorized" }); + assert.equal(calls, 1); + }); + } + + test("rejects ambiguous, missing, and non-callable authentication", () => { + const factories = [ + (key, provider) => createAzureOpenAILanguageModel(key, "https://example.test", { tokenProvider: provider }), + (key, provider) => createOpenAILanguageModel(key, "test", "https://example.test", "", { tokenProvider: provider }) + ]; + for (const factory of factories) { + assert.throws(() => factory("key", () => "token"), /not both/); + assert.throws(() => factory(undefined, undefined), /API key or a token provider/); + assert.throws(() => factory(undefined, "not-a-function"), /must be a function/); + } + }); + + test("retains proxy, timeout, and bounded-response controls", async () => { + setupFetch([new Response(new Uint8Array(200))]); + const model = createAzureOpenAILanguageModel(undefined, "https://example.test/chat/completions", { + tokenProvider: () => "token", + proxyUrl: "http://127.0.0.1:8080" + }); + model.timeoutMs = 1234; + model.maxResponseBytes = 32; + try { + const result = await model.complete("request"); + assert.equal(result.success, false); + assert.match(result.message, /maximum allowed size of 32 bytes/); + assert.ok(capturedRequests[0].options.signal instanceof AbortSignal); + assert.ok(capturedRequests[0].options.dispatcher); + assert.equal(capturedRequests[0].options.headers.Authorization, "Bearer token"); + assert.ok(!("api-key" in capturedRequests[0].options.headers)); + } finally { + await capturedRequests[0]?.options.dispatcher?.close(); + } + }); + + test("keeps literal identity environment values on the legacy key path", async () => { + setupFetch([makeChatCompletionsResponse("reply")]); + const model = createLanguageModel({ + AZURE_OPENAI_API_KEY: "identity", AZURE_OPENAI_ENDPOINT: "https://example.test/chat/completions" + }); + await model.complete("request"); + assert.equal(capturedRequests[0].options.headers["api-key"], "identity"); + assert.equal(capturedRequests[0].options.headers.Authorization, "Bearer identity"); + }); + + test("preserves explicitly supplied empty-string API keys", async () => { + setupFetch([makeChatCompletionsResponse("reply")]); + await createAzureOpenAILanguageModel("", "https://example.test/chat/completions").complete("request"); + assert.equal(capturedRequests[0].options.headers["api-key"], ""); + assert.equal(capturedRequests[0].options.headers.Authorization, "Bearer "); + }); +}); + +function clientBody(api, text) { + return api === "responses" ? + { output: [{ type: "message", content: [{ type: "output_text", text }] }] } : + { choices: [{ message: { content: text } }] }; +} + +function mockClient(api, responses) { + const queue = [...responses]; + const calls = []; + let closed = false; + const resource = { + async create(request) { + assert.equal(this, resource, "client methods must retain their receiver"); + calls.push(structuredClone(request)); + assert.ok(queue.length, "unexpected client call"); + const response = queue.shift(); + if (response instanceof Error) throw response; + return response; + } + }; + const client = Object.freeze({ + ...(api === "responses" ? { responses: resource } : { chat: { completions: resource } }), + timeout: 1234, + maxRetries: 0, + close() { closed = true; } + }); + return { client, calls, isClosed: () => closed }; +} + +describe("configured-client adapters", () => { + for (const api of ["chat", "responses"]) { + const options = { useResponsesApi: api === "responses" }; + + test(`${api}: reuses the configured client without network-control or lifecycle changes`, async () => { + const { client, calls, isClosed } = mockClient(api, [clientBody(api, "first"), clientBody(api, "second")]); + const model = createOpenAILanguageModelFromClient(client, "deployment-name", options); + assert.deepEqual(await model.complete("first prompt"), { success: true, data: "first" }); + assert.deepEqual(await model.complete("second prompt"), { success: true, data: "second" }); + assert.deepEqual(Object.keys(model), ["complete"]); + assert.equal(client.timeout, 1234); + assert.equal(client.maxRetries, 0); + assert.equal(isClosed(), false); + assert.deepEqual(calls, ["first prompt", "second prompt"].map(prompt => ({ + model: "deployment-name", temperature: 0, stream: false, + ...(api === "responses" ? { input: prompt } : { messages: [{ role: "user", content: prompt }], n: 1 }) + }))); + }); + + test(`${api}: maps mixed text and images without mutating prompts`, async () => { + const { client, calls } = mockClient(api, [clientBody(api, "reply")]); + const model = createOpenAILanguageModelFromClient(client, "test-model", options); + const prompt = [ + { role: "system", content: [{ type: "text", text: "instructions" }] }, + { role: "user", content: [ + "look", { type: "text", text: "describe" }, + { type: "image_url", image_url: { url: "https://example.test/image.png", detail: "low" } }, + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } } + ] }, + { role: "assistant", content: "previous reply" } + ]; + const original = structuredClone(prompt); + assert.deepEqual(await model.complete(prompt), { success: true, data: "reply" }); + assert.deepEqual(prompt, original); + const expected = api === "responses" ? [ + { role: "system", content: [{ type: "input_text", text: "instructions" }] }, + { role: "user", content: [ + { type: "input_text", text: "look" }, { type: "input_text", text: "describe" }, + { type: "input_image", image_url: "https://example.test/image.png", detail: "low" }, + { type: "input_image", image_url: "data:image/png;base64,AAAA", detail: "auto" } + ] }, + { role: "assistant", content: "previous reply" } + ] : [ + original[0], + { role: "user", content: [{ type: "text", text: "look" }, ...original[1].content.slice(1)] }, + original[2] + ]; + assert.deepEqual(calls[0][api === "responses" ? "input" : "messages"], expected); + }); + + for (const content of [null, 42, undefined]) { + test(`${api}: invalid output content ${content} is an explicit failure`, async () => { + const { client } = mockClient(api, [clientBody(api, content)]); + const model = createOpenAILanguageModelFromClient(client, "test", options); + const result = await model.complete("request"); + assert.equal(result.success, false); + assert.match(result.message, /unexpected response format/); + }); + } + + test(`${api}: empty string output remains a successful completion`, async () => { + const { client } = mockClient(api, [clientBody(api, "")]); + const model = createOpenAILanguageModelFromClient(client, "test", options); + assert.deepEqual(await model.complete("request"), { success: true, data: "" }); + }); + + for (const thrown of [new Error("client failed"), new DOMException("cancelled", "AbortError")]) { + test(`${api}: propagates ${thrown.name} without extra retries or closing the client`, async () => { + const { client, calls, isClosed } = mockClient(api, [thrown, clientBody(api, "unused")]); + const model = createOpenAILanguageModelFromClient(client, "test", options); + await assert.rejects(model.complete("request"), error => error === thrown); + assert.equal(calls.length, 1); + assert.equal(isClosed(), false); + }); + } + + test(`${api}: rejects unsupported/malformed input rather than dropping content`, async () => { + const { client, calls } = mockClient(api, []); + const model = createOpenAILanguageModelFromClient(client, "test", options); + const prompts = [ + [{ role: "developer", content: "unsupported role" }], + [{ role: "user", content: { type: "text", text: "not an array" } }], + [{ role: "user", content: [{ type: "audio", data: "unsupported" }] }], + [{ role: "user", content: [{ type: "text", text: 42 }] }], + [{ role: "user", content: [{ type: "image_url", image_url: { url: "image", detail: "invalid" } }] }], + [{ role: "assistant", content: [{ type: "image_url", image_url: { url: "image" } }] }] + ]; + for (const prompt of prompts) { + const result = await model.complete(prompt); + assert.equal(result.success, false); + } + assert.equal(calls.length, 0); + }); + } + + test("validates the requested client capability and model name", () => { + const chat = mockClient("chat", []).client; + const responses = mockClient("responses", []).client; + assert.throws(() => createOpenAILanguageModelFromClient(chat, "test", { useResponsesApi: true }), /responses.create/); + assert.throws(() => createOpenAILanguageModelFromClient(responses, "test"), /chat.completions.create/); + assert.throws(() => createOpenAILanguageModelFromClient(chat, ""), /must not be empty/); + assert.throws(() => createOpenAILanguageModelFromClient(chat, " "), /must not be empty/); + }); +}); + // --------------------------------------------------------------------------- // Chat Completions API (backward compatibility) // --------------------------------------------------------------------------- From 91f3b2678a896c93b2b1e44ad7c5ed961b644e44 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Tue, 3 Nov 2026 10:01:41 -0800 Subject: [PATCH 2/3] Reject malformed bearer tokens before HTTP dispatch Cover whitespace, trailing newlines, control characters, non-ASCII values, and valid printable-ASCII boundaries without changing legacy API-key handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- site/src/docs/typescript/basic-usage.md | 5 +++-- typescript/src/model.ts | 4 ++-- typescript/tests/model.test.mjs | 16 ++++++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/site/src/docs/typescript/basic-usage.md b/site/src/docs/typescript/basic-usage.md index b77874f4..ef842884 100644 --- a/site/src/docs/typescript/basic-usage.md +++ b/site/src/docs/typescript/basic-usage.md @@ -103,8 +103,9 @@ const model = createAzureOpenAILanguageModel(undefined, endpoint, { without a `Bearer ` prefix. The provider owns credential acquisition, caching, renewal, and cleanup; TypeChat adds no Azure Identity dependency. It invokes the provider for each HTTP attempt, including retries, and sends bearer authorization without an `api-key` -header. API keys and providers are mutually exclusive. Empty or invalid tokens return -an error without sending a request. Provider exceptions follow the HTTP model's existing +header. API keys and providers are mutually exclusive. Non-string/empty tokens or tokens +containing whitespace, control characters, or non-ASCII characters return an error before +dispatch. Provider exceptions follow the HTTP model's existing retry/error policy. The same option works with `createOpenAILanguageModel`, including its Responses mode: diff --git a/typescript/src/model.ts b/typescript/src/model.ts index 03d07be1..e8515667 100644 --- a/typescript/src/model.ts +++ b/typescript/src/model.ts @@ -452,8 +452,8 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o try { if (tokenProvider) { const token = await tokenProvider(); - if (typeof token !== "string" || !token.trim()) { - return finish(error("Token provider must return a nonempty string.")); + if (typeof token !== "string" || !token || /[^\x21-\x7e]/.test(token)) { + return finish(error("Token provider must return a nonempty, header-safe bearer-token string.")); } options.headers = { ...headers, "content-type": "application/json", "Authorization": `Bearer ${token}` }; } diff --git a/typescript/tests/model.test.mjs b/typescript/tests/model.test.mjs index fdda7490..5bf50af7 100644 --- a/typescript/tests/model.test.mjs +++ b/typescript/tests/model.test.mjs @@ -188,11 +188,14 @@ describe("token-provider authentication", () => { assert.equal(capturedRequests.length, 1); }); - for (const token of ["", " ", undefined, null, 42, {}]) { + for (const token of [ + "", " ", undefined, null, 42, {}, "token value", " token", + "token\r\nother", "token\u0100", "token\n", "token\r", "token\u0000", "token\u007f" + ]) { test(`${name}: rejects invalid token ${JSON.stringify(token)} before fetch`, async () => { setupFetch([]); assert.deepEqual(await factory(() => token).complete("request"), { - success: false, message: "Token provider must return a nonempty string." + success: false, message: "Token provider must return a nonempty, header-safe bearer-token string." }); assert.equal(capturedRequests.length, 0); }); @@ -256,6 +259,15 @@ describe("token-provider authentication", () => { assert.equal(capturedRequests[0].options.headers["api-key"], ""); assert.equal(capturedRequests[0].options.headers.Authorization, "Bearer "); }); + + test("accepts printable ASCII bearer-token boundary characters", async () => { + setupFetch([makeChatCompletionsResponse("reply")]); + const model = createAzureOpenAILanguageModel(undefined, "https://example.test/chat/completions", { + tokenProvider: () => "!~" + }); + assert.deepEqual(await model.complete("request"), { success: true, data: "reply" }); + assert.equal(capturedRequests[0].options.headers.Authorization, "Bearer !~"); + }); }); function clientBody(api, text) { From 5324d143fc420b1fafff1e514b6769bf17ef723b Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Tue, 3 Nov 2026 12:08:33 -0800 Subject: [PATCH 3/3] Document nonblocking TypeScript token providers Apply the Python review's shared event-loop guidance to TypeScript without changing provider invocation or the existing malformed-token guard. Add pending-Promise regressions for all HTTP factory paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- site/src/docs/typescript/basic-usage.md | 7 +++++++ typescript/src/model.ts | 6 +++++- typescript/tests/model.test.mjs | 23 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/site/src/docs/typescript/basic-usage.md b/site/src/docs/typescript/basic-usage.md index ef842884..1da0adb3 100644 --- a/site/src/docs/typescript/basic-usage.md +++ b/site/src/docs/typescript/basic-usage.md @@ -108,6 +108,13 @@ containing whitespace, control characters, or non-ASCII characters return an err dispatch. Provider exceptions follow the HTTP model's existing retry/error policy. +Providers execute on the caller's JavaScript event loop. Synchronous providers should +return cached tokens quickly; use asynchronous SDK/network APIs for acquisition or renewal. +Adding `async` or calling `Promise.resolve` around synchronous I/O does not make that work +nonblocking. If a legacy provider requires blocking I/O, offload it using application-managed +workers (for example Node.js `worker_threads`) with an appropriate thread-safety and timeout +strategy. TypeChat invokes providers directly and does not automatically move them to workers. + The same option works with `createOpenAILanguageModel`, including its Responses mode: ```ts diff --git a/typescript/src/model.ts b/typescript/src/model.ts index e8515667..c5cb8d6e 100644 --- a/typescript/src/model.ts +++ b/typescript/src/model.ts @@ -93,7 +93,11 @@ export interface TypeChatLanguageModel { complete(prompt: string | PromptSection[]): Promise>; } -/** Supplies a bearer token; credential acquisition, caching, and renewal remain caller-owned. */ +/** + * Supplies a bearer token; credential acquisition, caching, and renewal remain caller-owned. + * Invoked on the caller's event-loop thread. Keep synchronous providers nonblocking; use asynchronous + * I/O or application-managed workers for blocking acquisition rather than just wrapping it in a Promise. + */ export type TokenProvider = () => string | Promise; /** diff --git a/typescript/tests/model.test.mjs b/typescript/tests/model.test.mjs index 5bf50af7..e7607d2b 100644 --- a/typescript/tests/model.test.mjs +++ b/typescript/tests/model.test.mjs @@ -152,6 +152,29 @@ describe("token-provider authentication", () => { } }); + test(`${name}: pending credential promises keep the event loop responsive and delay fetch`, async () => { + setupFetch([reply()]); + let releaseToken; + let invoked = false; + const token = new Promise(resolve => { releaseToken = resolve; }); + const model = factory(() => { + invoked = true; + return token; + }); + const completion = model.complete("request"); + let result; + try { + await new Promise(resolve => setImmediate(resolve)); + assert.equal(invoked, true); + assert.equal(capturedRequests.length, 0); + } finally { + releaseToken("ready-token"); + result = await completion; + } + assert.deepEqual(result, { success: true, data: "reply" }); + assert.equal(capturedRequests.length, 1); + }); + test(`${name}: awaits the provider again for each HTTP retry`, async () => { setupFetch([makeErrorResponse(429, "Too Many Requests"), makeErrorResponse(500, "Internal Server Error"), reply()]); let calls = 0;