From 5eec538bcd3f2b658073b8da5d5fdb24d69a9696 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 14:52:35 -0500 Subject: [PATCH 1/2] feat: switch installer to Claude Opus 5.5 and bump @workos/skills to 0.8.0 --- bun.lock | 4 ++-- package.json | 2 +- src/cli.config.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bun.lock b/bun.lock index e9b938d5..9ab1603d 100644 --- a/bun.lock +++ b/bun.lock @@ -14,7 +14,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.7.3", + "@workos/skills": "0.8.0", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", @@ -411,7 +411,7 @@ "@workos/openapi-spec": ["@workos/openapi-spec@0.123.0", "", {}, "sha512-ZsHblNwgpc53tLfNoZU3mRA+L/vO2xadWus6Y68XVfcp9cz9ufNax8qAeWxgspmc0zazsnwy2xb0+ZB0dyrEOw=="], - "@workos/skills": ["@workos/skills@0.7.3", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-xg+TqR52bBj6nNfhzzUuLhBwaeaFs/913H5z7jsq4LF/mxm6TzQkfecFh7z15wrbbf1unpFXSY3V0HGDyRdbWA=="], + "@workos/skills": ["@workos/skills@0.8.0", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-SNn1824L/q3wbVWLJQnxrsNwmAnYEREEwbgAtWwibjWTr0JH2uCbbpKIXdEbo2zbdBmxGvBnaCfR7jJHjTRn2A=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], diff --git a/package.json b/package.json index 2d12e3c2..0d876714 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.7.3", + "@workos/skills": "0.8.0", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", diff --git a/src/cli.config.ts b/src/cli.config.ts index 03e17cb3..4d2c38a1 100644 --- a/src/cli.config.ts +++ b/src/cli.config.ts @@ -4,7 +4,7 @@ import pkg from '../package.json' with { type: 'json' }; export const version = pkg.version; export const config = { - model: 'claude-opus-4-5-20251101', + model: 'claude-opus-5-5', doctorModel: 'claude-haiku-4-5-20251001', // Production defaults - override via env vars for local dev. From 6ce22df4c08af487ec4bfece77a54938c17578d6 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 14:52:36 -0500 Subject: [PATCH 2/2] test(evals): gate eval tools like production and accept authkitProxy The eval harness passed every tool in allowedTools, which skips installerCanUseTool. Eval agents could run commands the shipped installer denies, including curl and read-only workos CLI calls against the developer's signed-in account. allowedTools is now empty, as in production. authkit-nextjs 4.4 makes authkitMiddleware a deprecated alias of authkitProxy; the Next.js grader accepts both. --- tests/evals/__tests__/agent-executor.spec.ts | 4 ++-- tests/evals/agent-executor.ts | 4 +++- tests/evals/graders/nextjs.grader.ts | 9 +++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index 1171ceb9..fc3b8b66 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -109,8 +109,8 @@ describe('AgentExecutor', () => { const [agentRunConfig] = mockRunAgent.mock.calls[0]; expect(agentRunConfig.workingDirectory).toBe(testDir); expect(agentRunConfig.model).toBe('test-model'); - expect(agentRunConfig.allowedTools).toContain('Skill'); - expect(agentRunConfig.allowedTools).toContain('Write'); + // Must match production so every tool call goes through installerCanUseTool + expect(agentRunConfig.allowedTools).toEqual([]); expect(agentRunConfig.mcpServers).toBeUndefined(); // Direct mode — no gateway URL expect(agentRunConfig.sdkEnv.ANTHROPIC_API_KEY).toBe('sk-ant-test'); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index 9da9c2b1..d43a2806 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -138,7 +138,9 @@ export class AgentExecutor { const agentRunConfig: AgentRunConfig = { workingDirectory: this.workDir, model: process.env.EVAL_MODEL ?? getConfig().model, - allowedTools: ['Skill', 'Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep', 'WebFetch'], + // Empty like production: listed tools skip installerCanUseTool, which would let + // eval agents run commands (e.g. `workos`, `curl`) the shipped installer denies. + allowedTools: [], sdkEnv, claudeExecutablePath: await ensureClaudeCodeExecutable(), }; diff --git a/tests/evals/graders/nextjs.grader.ts b/tests/evals/graders/nextjs.grader.ts index f4c3fd74..9436b343 100644 --- a/tests/evals/graders/nextjs.grader.ts +++ b/tests/evals/graders/nextjs.grader.ts @@ -113,8 +113,9 @@ export class NextjsGrader implements Grader { const sdkImportChecks = await this.fileGrader.checkFileContains(middlewareFile, ['@workos-inc/authkit-nextjs']); checks.push(...sdkImportChecks); - // Check for authkit integration: authkitMiddleware OR (authkit + handleAuthkitProxy, or its deprecated alias handleAuthkitHeaders) - const middlewareChecks = await this.fileGrader.checkFileContains(middlewareFile, ['authkitMiddleware']); + // Check for authkit integration: authkitProxy (or its deprecated alias authkitMiddleware) + // OR (authkit + handleAuthkitProxy, or its deprecated alias handleAuthkitHeaders) + const middlewareChecks = await this.fileGrader.checkFileContains(middlewareFile, [/authkit(Proxy|Middleware)\b/]); const composableChecks = await this.fileGrader.checkFileContains(middlewareFile, [ 'authkit(', /handleAuthkit(Proxy|Headers)\b/, @@ -127,10 +128,10 @@ export class NextjsGrader implements Grader { name: 'AuthKit middleware integration', passed: usesAuthkitMiddleware || usesComposable, message: usesAuthkitMiddleware - ? 'Uses authkitMiddleware' + ? 'Uses authkitProxy/authkitMiddleware' : usesComposable ? 'Uses authkit() composable with handleAuthkitProxy' - : 'Missing authkitMiddleware or authkit() composable integration', + : 'Missing authkitProxy/authkitMiddleware or authkit() composable integration', }; checks.push(authkitCheck);