From 00e3c155b1e8757c7f3efcdf9aa22a3ff7954069 Mon Sep 17 00:00:00 2001 From: knowhy Date: Thu, 29 Oct 2026 07:39:53 +0000 Subject: [PATCH 1/2] feat: add LLMTR as a model provider --- .env.example | 7 + README.md | 4 +- apps/server/src/agent.ts | 3 +- apps/server/src/engine/tanstack-agent.ts | 26 +++- tests/llmtr-provider.test.ts | 159 +++++++++++++++++++++++ tests/model-provider.test.ts | 9 +- 6 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 tests/llmtr-provider.test.ts diff --git a/.env.example b/.env.example index 36e6580c9..dad065a56 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,13 @@ WEB_SEARCH_ENABLED=true # Providers whose /responses tool loop is incomplete (e.g. DeepSeek) can opt # into the Chat Completions wire format instead: # OPENAI_CHAT_COMPLETIONS=true +# LLMTR (https://llmtr.com) uses chat completions. The key stays on the server. +# MODEL=llmtr/qwen/qwen3.8-27b-free +# LLMTR_API_KEY= +# Hosted ids already start with llmtr/. MODEL=llmtr/qwen3-8-27b sends that id. +# Chat completions only. Responses-only and embedding ids are not called. +# Optional base URL override (default https://llmtr.com/v1): +# LLMTR_BASE_URL= # Live Google workspace: WORKSPACE_MODE=live, AGENT_BACKEND=model. # OPENMUSE_ACCESS_KEY= # random secret, at least 24 characters diff --git a/README.md b/README.md index c1c6982ca..2fd198e9c 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ The deploy form asks for three values you provide. Render generates the other tw |---|---|---| | `CPK_INTELLIGENCE_API_KEY` | You. Run `npx copilotkit@latest login`, then `npx copilotkit@latest project select`. Keep it on the server. | Chat cannot open a thread. | | `CPK_TELEMETRY_ID` | You. Keep the identity generated by the same project selection. | Runtime usage cannot be attributed to the Intelligence signup account. | -| `OPENAI_API_KEY` | You. Used by the default `openai/gpt-5`. Change `MODEL` and supply the matching provider key for Anthropic or Google. | The model call fails. | +| `OPENAI_API_KEY` | You. Used by the default `openai/gpt-5`. Change `MODEL` and supply the matching provider key for Anthropic, Google, or LLMTR. | The model call fails. | | `OPENMUSE_ACCESS_KEY` | Render | You cannot sign in. | | `TOKEN_ENCRYPTION_KEY` | Render | The API refuses to start in live mode. | @@ -141,7 +141,7 @@ The Docker computer and Google mail or calendar need the setup in the sections b Copy the commented settings in [.env.example](.env.example) into your private `.env`: -1. Set `AGENT_BACKEND=model`, `MODEL=provider/model-id`, and the matching provider key. CopilotKit supports the configured OpenAI, Anthropic or Google provider. Fictional data can still be used with a real model. Provider keys stay on the server. +1. Set `AGENT_BACKEND=model`, `MODEL=provider/model-id`, and the matching provider key. CopilotKit supports the configured OpenAI, Anthropic, Google, or LLMTR provider. Fictional data can still be used with a real model. Provider keys stay on the server. LLMTR uses `MODEL=llmtr/` and `LLMTR_API_KEY` (see [.env.example](.env.example)). 2. Create or select a CopilotKit Intelligence project with `npx copilotkit@latest login` and `npx copilotkit@latest project select`. Keep the generated `CPK_INTELLIGENCE_API_KEY` on the server. 3. For personal mail/calendar, set `WORKSPACE_MODE=live`, the generated `CPK_INTELLIGENCE_API_KEY`, a random `OPENMUSE_ACCESS_KEY` of at least 24 characters, and `TOKEN_ENCRYPTION_KEY` containing 32 random bytes encoded as base64. Restart the API. 4. Configure a Google OAuth web client with Gmail and Calendar APIs enabled. Set `GOOGLE_CLIENT_ID` and `GOOGLE_CLIENT_SECRET`; register `${PUBLIC_API_URL}/api/google/callback` as its redirect URI. Configure consent/test-user access in your Google project. diff --git a/apps/server/src/agent.ts b/apps/server/src/agent.ts index 4a5a75865..6201c540e 100644 --- a/apps/server/src/agent.ts +++ b/apps/server/src/agent.ts @@ -21,7 +21,8 @@ export function agentConfigured(config: Config) { config.model && (process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || - process.env.GOOGLE_API_KEY), + process.env.GOOGLE_API_KEY || + process.env.LLMTR_API_KEY), )) ); } diff --git a/apps/server/src/engine/tanstack-agent.ts b/apps/server/src/engine/tanstack-agent.ts index 9ebf2bf85..4bd9ae473 100644 --- a/apps/server/src/engine/tanstack-agent.ts +++ b/apps/server/src/engine/tanstack-agent.ts @@ -16,10 +16,22 @@ import { import { type AnthropicChatModel, anthropicText } from "@tanstack/ai-anthropic"; import { type GeminiTextModel, geminiText } from "@tanstack/ai-gemini"; import { type OpenAIChatModel, openaiChatCompletions, openaiText } from "@tanstack/ai-openai"; +import { openaiCompatibleText } from "@tanstack/ai-openai/compatible"; import { map, mergeMap, type Observable } from "rxjs"; import { z } from "zod"; import { MODEL_MAX_RETRIES } from "../config.ts"; +const defaultLlmtrBaseUrl = "https://llmtr.com/v1"; + +/** + * LLMTR catalog ids are `vendor/model`. Hosted ids are `llmtr/`, so a + * single segment after the provider prefix is that hosted id, not a bare name. + */ +export function llmtrModelId(model: string): string { + const id = model.trim(); + return id.includes("/") ? id : `llmtr/${id}`; +} + // Same "provider/model" strings, env vars and base URL formats as the AI SDK resolver in // @copilotkit/runtime. Each provider SDK retries transient failures up to MODEL_MAX_RETRIES times. function adapter(spec: string) { @@ -59,6 +71,18 @@ function adapter(spec: string) { retryOptions: { attempts: MODEL_MAX_RETRIES + 1 }, }, }); + case "llmtr": { + // Chat completions is the catalog wire format. The Responses adapter sends + // `reasoning` as an object; LLMTR accepts that parameter only as a boolean. + const apiKey = process.env.LLMTR_API_KEY?.trim(); + if (!apiKey) throw new Error("The llmtr provider requires LLMTR_API_KEY."); + return openaiCompatibleText(llmtrModelId(id), { + name: "llmtr", + baseURL: process.env.LLMTR_BASE_URL?.trim() || defaultLlmtrBaseUrl, + apiKey, + maxRetries: MODEL_MAX_RETRIES, + }); + } default: throw unknownProvider(provider, spec); } @@ -74,7 +98,7 @@ export function unknownProvider( ? ` For a model on your OPENAI_BASE_URL gateway, use "openai/${spec.trim()}".` : ""; return new Error( - `Unknown provider "${provider}" in "${spec}". Supported: openai, anthropic, google (gemini).${hint}`, + `Unknown provider "${provider}" in "${spec}". Supported: openai, anthropic, google (gemini), llmtr.${hint}`, ); } diff --git a/tests/llmtr-provider.test.ts b/tests/llmtr-provider.test.ts new file mode 100644 index 000000000..fd4c07f60 --- /dev/null +++ b/tests/llmtr-provider.test.ts @@ -0,0 +1,159 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { test } from "node:test"; +import { EventType, type RunAgentInput } from "@ag-ui/core"; +import type { BuiltInAgent } from "@copilotkit/runtime/v2"; +import { agentConfigured } from "../apps/server/src/agent.ts"; +import type { Config } from "../apps/server/src/config.ts"; +import { tanstackAgent } from "../apps/server/src/engine/tanstack-agent.ts"; + +const run = (agent: BuiltInAgent) => { + const input: RunAgentInput = { + threadId: "llmtr-fixture", + runId: "llmtr-fixture-run", + messages: [{ id: "m1", role: "user", content: "Reply with pong." }], + state: {}, + tools: [], + context: [], + forwardedProps: {}, + }; + return new Promise<{ error?: string; finished: boolean; text: string }>((resolve) => { + let error: string | undefined; + let finished = false; + let text = ""; + agent.run(input).subscribe({ + next: (event) => { + if ( + (event.type === EventType.TEXT_MESSAGE_CHUNK || + event.type === EventType.TEXT_MESSAGE_CONTENT) && + "delta" in event && + typeof event.delta === "string" + ) + text += event.delta; + if (event.type === EventType.RUN_ERROR && "message" in event) error = String(event.message); + if (event.type === EventType.RUN_FINISHED) finished = true; + }, + error: (cause) => { + if (error === undefined) error = String(cause); + resolve({ error, finished: false, text }); + }, + complete: () => resolve({ error, finished, text }), + }); + }); +}; + +test("LLMTR_API_KEY marks the model agent configured", () => { + const previous = { + openai: process.env.OPENAI_API_KEY, + anthropic: process.env.ANTHROPIC_API_KEY, + google: process.env.GOOGLE_API_KEY, + llmtr: process.env.LLMTR_API_KEY, + }; + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + delete process.env.GOOGLE_API_KEY; + delete process.env.LLMTR_API_KEY; + try { + const config = { + agentBackend: "model", + model: "llmtr/qwen/qwen3.8-27b-free", + } as Config; + assert.equal(agentConfigured(config), false); + process.env.LLMTR_API_KEY = "llmtr-test"; + assert.equal(agentConfigured(config), true); + } finally { + for (const [name, value] of [ + ["OPENAI_API_KEY", previous.openai], + ["ANTHROPIC_API_KEY", previous.anthropic], + ["GOOGLE_API_KEY", previous.google], + ["LLMTR_API_KEY", previous.llmtr], + ] as const) + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +test("the llmtr provider posts chat completions and keeps hosted model ids", async (t) => { + const requests: { path: string; authorization: string; body: string }[] = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + requests.push({ + path: request.url ?? "", + authorization: request.headers.authorization ?? "", + body, + }); + response.writeHead(200, { "Content-Type": "text/event-stream" }); + const chunk = (delta: object, finish: string | null) => + response.write( + `data: ${JSON.stringify({ + id: "chatcmpl-llmtr", + object: "chat.completion.chunk", + created: 1000, + model: "fixture", + choices: [{ index: 0, delta, finish_reason: finish }], + })}\n\n`, + ); + chunk({ role: "assistant" }, null); + chunk({ content: "pong" }, null); + chunk({}, "stop"); + response.end("data: [DONE]\n\n"); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const previousBase = process.env.LLMTR_BASE_URL; + const previousKey = process.env.LLMTR_API_KEY; + process.env.LLMTR_BASE_URL = `http://127.0.0.1:${address.port}/v1`; + process.env.LLMTR_API_KEY = "llmtr-test-key"; + t.after(async () => { + if (previousBase === undefined) delete process.env.LLMTR_BASE_URL; + else process.env.LLMTR_BASE_URL = previousBase; + if (previousKey === undefined) delete process.env.LLMTR_API_KEY; + else process.env.LLMTR_API_KEY = previousKey; + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + + for (const spec of ["llmtr/qwen3-8-27b", "llmtr/qwen/qwen3.8-27b-free"] as const) { + const outcome = await run( + tanstackAgent({ model: spec, maxSteps: 2, tools: [], prompt: "Reply with pong." }), + ); + assert.equal(outcome.error, undefined, outcome.error); + assert.equal(outcome.finished, true); + assert.match(outcome.text, /pong/); + } + + assert.equal(requests.length, 2); + for (const [index, model] of ["llmtr/qwen3-8-27b", "qwen/qwen3.8-27b-free"].entries()) { + const request = requests[index]; + assert.ok(request); + assert.equal(request.path, "/v1/chat/completions"); + assert.equal(request.authorization, "Bearer llmtr-test-key"); + const body = JSON.parse(request.body) as { model?: string; reasoning?: unknown }; + assert.equal(body.model, model); + assert.equal("reasoning" in body, false); + } +}); + +test("the llmtr provider fails before a request when the key is missing", async () => { + const previous = process.env.LLMTR_API_KEY; + delete process.env.LLMTR_API_KEY; + try { + const outcome = await run( + tanstackAgent({ + model: "llmtr/qwen/qwen3.8-27b-free", + maxSteps: 1, + tools: [], + prompt: "Reply with pong.", + }), + ); + assert.match(outcome.error ?? "", /LLMTR_API_KEY/); + assert.equal(outcome.finished, false); + } finally { + if (previous === undefined) delete process.env.LLMTR_API_KEY; + else process.env.LLMTR_API_KEY = previous; + } +}); diff --git a/tests/model-provider.test.ts b/tests/model-provider.test.ts index 24ec3e3eb..33e597e64 100644 --- a/tests/model-provider.test.ts +++ b/tests/model-provider.test.ts @@ -1,15 +1,22 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { unknownProvider } from "../apps/server/src/engine/tanstack-agent.ts"; +import { llmtrModelId, unknownProvider } from "../apps/server/src/engine/tanstack-agent.ts"; const spec = "deepseek/deepseek-v4.1-flash"; +test("an llmtr model id keeps a hosted llmtr/ prefix and a routed vendor/model id", () => { + assert.equal(llmtrModelId("qwen3-8-27b"), "llmtr/qwen3-8-27b"); + assert.equal(llmtrModelId("llmtr/qwen3-8-27b"), "llmtr/qwen3-8-27b"); + assert.equal(llmtrModelId("qwen/qwen3.8-27b-free"), "qwen/qwen3.8-27b-free"); +}); + test("an unknown provider suggests the openai/ prefix only when a gateway is set", () => { const gateway = unknownProvider("deepseek", spec, "https://gateway.test/v1"); assert.match( gateway.message, /^Unknown provider "deepseek" in "deepseek\/deepseek-v4.1-flash"\./, ); + assert.match(gateway.message, /google \(gemini\), llmtr\./); assert.match(gateway.message, /use "openai\/deepseek\/deepseek-v4.1-flash"\.$/); for (const baseUrl of ["", " "]) From 206c104e5517cce25b3395f12ba31ddf8ec3dc70 Mon Sep 17 00:00:00 2001 From: knowhy Date: Thu, 29 Oct 2026 07:40:41 +0000 Subject: [PATCH 2/2] docs: list LLMTR with the other model providers --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fd198e9c..0c9de1885 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ The Docker computer and Google mail or calendar need the setup in the sections b Copy the commented settings in [.env.example](.env.example) into your private `.env`: -1. Set `AGENT_BACKEND=model`, `MODEL=provider/model-id`, and the matching provider key. CopilotKit supports the configured OpenAI, Anthropic, Google, or LLMTR provider. Fictional data can still be used with a real model. Provider keys stay on the server. LLMTR uses `MODEL=llmtr/` and `LLMTR_API_KEY` (see [.env.example](.env.example)). +1. Set `AGENT_BACKEND=model`, `MODEL=provider/model-id`, and the matching provider key. Supported providers are OpenAI, Anthropic, Google, and LLMTR. Fictional data can still be used with a real model. Provider keys stay on the server. LLMTR uses `MODEL=llmtr/` and `LLMTR_API_KEY` (see [.env.example](.env.example)). 2. Create or select a CopilotKit Intelligence project with `npx copilotkit@latest login` and `npx copilotkit@latest project select`. Keep the generated `CPK_INTELLIGENCE_API_KEY` on the server. 3. For personal mail/calendar, set `WORKSPACE_MODE=live`, the generated `CPK_INTELLIGENCE_API_KEY`, a random `OPENMUSE_ACCESS_KEY` of at least 24 characters, and `TOKEN_ENCRYPTION_KEY` containing 32 random bytes encoded as base64. Restart the API. 4. Configure a Google OAuth web client with Gmail and Calendar APIs enabled. Set `GOOGLE_CLIENT_ID` and `GOOGLE_CLIENT_SECRET`; register `${PUBLIC_API_URL}/api/google/callback` as its redirect URI. Configure consent/test-user access in your Google project.