diff --git a/bun.lock b/bun.lock index 9ab1603d..5e9e274e 100644 --- a/bun.lock +++ b/bun.lock @@ -14,7 +14,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.8.0", + "@workos/skills": "0.8.1", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", @@ -411,7 +411,7 @@ "@workos/openapi-spec": ["@workos/openapi-spec@0.123.0", "", {}, "sha512-ZsHblNwgpc53tLfNoZU3mRA+L/vO2xadWus6Y68XVfcp9cz9ufNax8qAeWxgspmc0zazsnwy2xb0+ZB0dyrEOw=="], - "@workos/skills": ["@workos/skills@0.8.0", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-SNn1824L/q3wbVWLJQnxrsNwmAnYEREEwbgAtWwibjWTr0JH2uCbbpKIXdEbo2zbdBmxGvBnaCfR7jJHjTRn2A=="], + "@workos/skills": ["@workos/skills@0.8.1", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-+b7WnFoHrtpMpzS60BApe7WQTzXl0oVf8WpWdpGQ88fMIa4ye9MGT/hxvC/Js2jm4es9HkCwHCFkVXUZO/YJPA=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], diff --git a/package.json b/package.json index 0d876714..569eacf0 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.8.0", + "@workos/skills": "0.8.1", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", diff --git a/src/lib/agent-interface.spec.ts b/src/lib/agent-interface.spec.ts index 0c39f813..bca3f605 100644 --- a/src/lib/agent-interface.spec.ts +++ b/src/lib/agent-interface.spec.ts @@ -169,6 +169,31 @@ describe('runAgent retry loop', () => { }) as typeof emitter.emit; }); + it('isolates the agent from user settings and states installer rules in the system prompt', async () => { + mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); + + await runAgent(makeAgentConfig(), 'Test prompt', makeOptions(), undefined, emitter); + + const { options } = mockQuery.mock.calls[0][0] as { options: Record }; + // Omitted, the SDK loads ~/.claude settings: user plugins, MCP servers, hooks, and + // permission rules that would apply before installerCanUseTool. + expect(options.settingSources).toEqual([]); + expect(options.systemPrompt).toEqual({ + type: 'preset', + preset: 'claude_code', + append: expect.stringContaining('WorkOS installer environment'), + }); + }); + + it('passes agent messages to options.onAgentMessage', async () => { + mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); + const onAgentMessage = vi.fn(); + + await runAgent(makeAgentConfig(), 'Test prompt', makeOptions({ onAgentMessage }), undefined, emitter); + + expect(onAgentMessage).toHaveBeenCalled(); + }); + it('returns retryCount=0 when no retryConfig provided', async () => { mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); @@ -567,4 +592,50 @@ describe('installerCanUseTool', () => { }); expect(result.behavior).toBe('deny'); }); + describe('with a working directory', () => { + const cwd = '/proj'; + const verdict = (command: string) => installerCanUseTool('Bash', { command }, cwd).behavior; + + it('allows an allowlisted command after cd into the working directory only', () => { + expect(verdict('cd /proj && bun run build')).toBe('allow'); + expect(verdict('cd /proj && bun run build 2>&1 | tail -20')).toBe('allow'); + expect(verdict('cd /other && bun run build')).toBe('deny'); + expect(verdict('cd /proj && bun run build && curl http://x/')).toBe('deny'); + }); + + it('allows read-only git and denies git that writes or runs programs', () => { + expect(verdict('git status --short')).toBe('allow'); + expect(verdict('git -C /proj status')).toBe('allow'); + expect(verdict('git diff --stat | head -50')).toBe('allow'); + expect(verdict('git -C /etc log')).toBe('deny'); + expect(verdict('git diff --output=/tmp/x')).toBe('deny'); + expect(verdict('git -c core.pager=sh log')).toBe('deny'); + expect(verdict('git commit -m x')).toBe('deny'); + expect(verdict('git push')).toBe('deny'); + }); + + it('allows syntax checks per tool without allowing code execution', () => { + expect(verdict('python3 -m py_compile server.py')).toBe('allow'); + expect(verdict('python3 -m venv .venv')).toBe('allow'); + expect(verdict('ruby -c app.rb')).toBe('allow'); + expect(verdict('php -l index.php')).toBe('allow'); + expect(verdict('python3 -c "print(1)"')).toBe('deny'); + expect(verdict('python3 -m http.server')).toBe('deny'); + expect(verdict('ruby -e "puts 1"')).toBe('deny'); + expect(verdict('php -r "echo 1"')).toBe('deny'); + }); + + it('allows dependency changes across ecosystems', () => { + expect(verdict('bun remove @auth0/auth0-react')).toBe('allow'); + expect(verdict('go get github.com/workos/workos-go/v6')).toBe('allow'); + expect(verdict('go mod tidy')).toBe('allow'); + expect(verdict('composer require workos/workos-php')).toBe('allow'); + }); + + it('still denies network, the workos CLI, and plain inspection commands', () => { + expect(verdict('curl -sL https://example.com')).toBe('deny'); + expect(verdict('npx workos whoami')).toBe('deny'); + expect(verdict('ls -la')).toBe('deny'); + }); + }); }); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index 22efda04..aca36106 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -3,7 +3,7 @@ * Uses Claude Agent SDK directly with WorkOS MCP server */ -import { dirname } from 'path'; +import { dirname, resolve } from 'path'; import { getSkillsDir as getSkillsPackageDir } from './skills-assets.js'; import { ensureClaudeCodeExecutable } from './agent-sdk-assets.js'; import { debug, logInfo, logWarn, logError, initLogFile, getLogFilePath } from '../utils/debug.js'; @@ -102,6 +102,23 @@ export type AgentRunConfig = { claudeExecutablePath: string; }; +/** + * Appended to Claude Code's system prompt for every installer run. It states up front + * what installerCanUseTool enforces, so the agent doesn't discover the rules one + * denied command at a time. Keep it in sync with the allowlist below. + */ +export const INSTALLER_SYSTEM_PROMPT_APPEND = `# WorkOS installer environment + +You are running inside the WorkOS CLI installer, not an interactive session, and your working directory is the user's project. + +- Bash runs only: package-manager install/add/remove commands; the project's build, typecheck, lint, format, and test scripts; language checks such as \`go build\`, \`go vet\`, \`mix compile\`, \`python3 -m py_compile\`, \`ruby -c\`, \`php -l\`, \`dotnet build\`, and \`./gradlew build\`; and read-only git (\`status\`, \`diff\`, \`log\`, \`show\`, \`ls-files\`). +- Run one command per Bash call from the working directory. Don't use \`cd\`, \`&&\`, \`;\`, \`$(...)\`, or pipes other than \`| head\` or \`| tail\`. +- Inspect files with Read, Grep, and Glob, not Bash. +- Bash has no network access: no curl, \`npm view\`, or \`git clone\`. Use WebFetch to read documentation. The \`workos\` CLI is not available to you. +- Don't start dev servers or open a browser to test sign-in. After you finish, the installer typechecks and builds the project and sends you any errors to fix. +- Don't change WorkOS dashboard settings (redirect URIs, Sign-out URI, Initiate login URI, CORS). The installer configures supported settings after your code passes validation. List anything you could not verify as unverified. +- Leave changes unstaged: don't stage, commit, or create branches.`; + /** * Package managers that can be used to run commands. * Includes JS and non-JS ecosystem package managers for multi-SDK support. @@ -153,10 +170,16 @@ const PACKAGE_MANAGERS = [ * Note: Linting tools are in LINTING_TOOLS and checked separately. */ const SAFE_SCRIPTS = [ - // Package installation + // Package installation and removal 'install', 'add', 'ci', + 'remove', + 'uninstall', + 'get', // go get + 'require', // composer require + 'mod tidy', + 'mod download', // Build 'build', // Type checking (various naming conventions) @@ -204,6 +227,46 @@ const SAFE_SCRIPTS = [ */ const DANGEROUS_OPERATORS = /[;`$()\n\r]/; +/** + * Per-tool arguments that only check syntax or create a local environment. + * Kept per tool because the same flag can execute code elsewhere + * (`ruby -c` checks syntax; `python -c` runs code). + */ +const TOOL_SAFE_ARGS: Record = { + python: ['-m py_compile', '-m compileall', '-m venv', '-m pip install'], + python3: ['-m py_compile', '-m compileall', '-m venv', '-m pip install'], + uv: ['venv', 'pip install', 'sync'], + ruby: ['-c '], + php: ['-l '], +}; + +/** Read-only git subcommands the agent may use to see what changed. */ +const READ_ONLY_GIT_SUBCOMMANDS = ['status', 'diff', 'log', 'show', 'ls-files', 'rev-parse']; +/** git arguments that write files or run external programs. */ +const UNSAFE_GIT_ARG = /^(-c|-o|--output.*|--ext-diff|--textconv|--exec.*|--upload-pack.*|--open-files-in-pager.*)$/; + +function isReadOnlyGit(command: string, cwd?: string): boolean { + const parts = command.split(/\s+/); + if (parts[0] !== 'git') return false; + let i = 1; + if (parts[i] === '--no-pager') i++; + if (parts[i] === '-C') { + if (!cwd || !parts[i + 1] || resolve(cwd, parts[i + 1]) !== resolve(cwd)) return false; + i += 2; + } + if (!READ_ONLY_GIT_SUBCOMMANDS.includes(parts[i])) return false; + return !parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)); +} + +/** Strip `cd && ` so the rest is judged on its own. */ +function stripCdIntoWorkingDirectory(command: string, cwd?: string): string { + if (!cwd) return command; + const match = command.match(/^cd\s+(?:"([^"]+)"|'([^']+)'|(\S+))\s*&&\s*(\S.*)$/s); + if (!match) return command; + const dir = match[1] ?? match[2] ?? match[3]; + return resolve(cwd, dir) === resolve(cwd) ? match[4] : command; +} + /** * Check if command is an allowed package manager command. * Matches: [run|exec] [args...] @@ -226,7 +289,8 @@ function matchesAllowedPrefix(command: string): boolean { // Check if script starts with any safe script name or linting tool return ( SAFE_SCRIPTS.some((safe) => scriptPart.startsWith(safe)) || - LINTING_TOOLS.some((tool) => scriptPart.startsWith(tool)) + LINTING_TOOLS.some((tool) => scriptPart.startsWith(tool)) || + (TOOL_SAFE_ARGS[parts[0]] ?? []).some((arg) => parts.slice(1).join(' ').startsWith(arg)) ); } @@ -237,13 +301,20 @@ function matchesAllowedPrefix(command: string): boolean { * - Piping to tail/head for output limiting is allowed * - Stderr redirection (2>&1) is allowed */ -export function installerCanUseTool(toolName: string, input: Record): PermissionResult { +export function installerCanUseTool( + toolName: string, + input: Record, + workingDirectory?: string, +): PermissionResult { // Allow all non-Bash tools if (toolName !== 'Bash') { return { behavior: 'allow', updatedInput: input }; } - const command = (typeof input.command === 'string' ? input.command : '').trim(); + const command = stripCdIntoWorkingDirectory( + (typeof input.command === 'string' ? input.command : '').trim(), + workingDirectory, + ); // Block definitely dangerous operators: ; ` $ ( ) if (DANGEROUS_OPERATORS.test(command)) { @@ -283,7 +354,7 @@ export function installerCanUseTool(toolName: string, input: Record { logInfo('canUseTool called:', { toolName, input }); - const result = installerCanUseTool(toolName, input); + const result = installerCanUseTool(toolName, input, agentConfig.workingDirectory); logInfo('canUseTool result:', result); return Promise.resolve(result); }, tools: { type: 'preset', preset: 'claude_code' }, + // Without this the SDK sends an empty system prompt, so the agent never gets + // Claude Code's tool guidance (Read/Grep over cat/grep) or our installer rules. + systemPrompt: { type: 'preset', preset: 'claude_code', append: INSTALLER_SYSTEM_PROMPT_APPEND }, + // Load no ~/.claude or project settings. Omitted, the SDK loads them all: the + // user's plugins, MCP servers, hooks, and permission rules, which apply before + // installerCanUseTool and would bypass the Bash gate. + settingSources: [], allowedTools: agentConfig.allowedTools, plugins: [{ type: 'local', path: pluginPath }], pathToClaudeCodeExecutable: agentConfig.claudeExecutablePath, @@ -704,6 +782,7 @@ export async function runAgent( } try { onMessage?.(message); + options.onAgentMessage?.(message); } catch { /* non-critical */ } diff --git a/src/lib/agent-runner.spec.ts b/src/lib/agent-runner.spec.ts index b7ace415..2f02174d 100644 --- a/src/lib/agent-runner.spec.ts +++ b/src/lib/agent-runner.spec.ts @@ -29,7 +29,7 @@ import { runAgentInstaller } from './agent-runner.js'; import { validateInstallation, quickCheckValidateAndFormat } from './validation/index.js'; import { autoConfigureWorkOSEnvironment } from './workos-management.js'; import { writeEnvLocal } from './env-writer.js'; -import { getOrAskForWorkOSCredentials } from '../utils/ui-utils.js'; +import { ensurePackageIsInstalled, getOrAskForWorkOSCredentials, getPackageDotJson } from '../utils/ui-utils.js'; const options: InstallerOptions = { debug: false, @@ -244,3 +244,38 @@ describe('installer prompt', () => { expect(runAgent).not.toHaveBeenCalled(); }); }); + +describe('project manifest checks', () => { + it.each([ + ['kotlin', 'kotlin', 'workos-kotlin'], + ['php', 'php', 'workos-php'], + ['php-laravel', 'php', 'workos-php-laravel'], + ] as const)('does not read package.json for %s projects', async (integration, language, skillName) => { + await runAgentInstaller({ ...config, metadata: { ...config.metadata, integration, language, skillName } }, options); + + expect(getPackageDotJson).not.toHaveBeenCalled(); + expect(ensurePackageIsInstalled).not.toHaveBeenCalled(); + expect(runAgent).toHaveBeenCalledTimes(1); + }); + + it('does not require a framework package for vanilla JS', async () => { + await runAgentInstaller( + { + ...config, + metadata: { ...config.metadata, integration: 'vanilla-js', skillName: 'workos-authkit-vanilla-js' }, + detection: { packageName: 'workos', packageDisplayName: 'Vanilla JavaScript', getVersion: () => undefined }, + }, + options, + ); + + expect(getPackageDotJson).toHaveBeenCalled(); + expect(ensurePackageIsInstalled).not.toHaveBeenCalled(); + expect(runAgent).toHaveBeenCalledTimes(1); + }); + + it('still requires the framework package for other JS integrations', async () => { + await runAgentInstaller(config, options); + + expect(ensurePackageIsInstalled).toHaveBeenCalledWith(expect.anything(), 'next', 'Next.js'); + }); +}); diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index 7a8ddd26..07d4cb06 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -41,8 +41,14 @@ export async function runAgentInstaller(config: FrameworkConfig, options: Instal // Git check is now handled by the state machine - no need to check here // Framework detection and version - const packageJson = await getPackageDotJson(options); - await ensurePackageIsInstalled(packageJson, config.detection.packageName, config.detection.packageDisplayName); + // Only JS integrations have a package.json; reading it elsewhere aborts the install + // (Kotlin, PHP, Laravel). Vanilla JS is the fallback for any JS project, so it has + // no framework package to require. + const isJavaScriptProject = config.metadata.language === 'javascript'; + const packageJson = isJavaScriptProject ? await getPackageDotJson(options) : {}; + if (isJavaScriptProject && config.metadata.integration !== 'vanilla-js') { + await ensurePackageIsInstalled(packageJson, config.detection.packageName, config.detection.packageDisplayName); + } const frameworkVersion = config.detection.getVersion(packageJson); @@ -345,21 +351,27 @@ ${baseContent ? `## General Guidelines\n\n${baseContent}\n\n` : ''}## Integratio ${refContent} ${ - setupContent - ? `## Required Application Setup and Verification + baseContent && !setupContent + ? `The references above link to workos-authkit-setup.md. It covers dashboard configuration and browser checks that this installer handles itself, so you don't need to read it. + +` + : '' +}${ + setupContent + ? `## Required Application Setup and Verification ${setupContent} ## Installer execution boundary The setup reference above is already included in this prompt. Do not read a relative workos-authkit-setup.md from the app directory. -The agent's shell permissions do not allow WorkOS management commands. Do not run workos, install another CLI, use curl or SDK scripts to bypass that boundary, or attempt dashboard authentication. Implement and validate the app code only. The installer handles supported dashboard configuration outside the agent after code validation; unavailable configuration must remain explicitly unverified. +Don't install another CLI or use SDK scripts to change WorkOS settings or authenticate to the dashboard. Implement and validate the app code only. Create a dedicated /sign-in GET route in the App Router using getSignInUrl() from @workos-inc/authkit-nextjs and redirect(await getSignInUrl()) from next/navigation. Keep the OAuth callback using handleAuth() separate. The Initiate login URI is the app origin plus /sign-in, NEVER the callback URI. Read existing files before editing; do not replace an unrelated existing sign-in flow. If a page already serves /sign-in (including inside route groups), stop: this installer cannot place a route handler alongside that page. Never delete the page or create a conflicting route. Keep /sign-in public and follow the SDK README for PKCE cookie handling. Do not claim the full integration or browser flows are verified. Report code implementation separately from application configuration and browser testing. ` - : '' -} + : '' + } ${signInSection}Report your progress using [STATUS] prefixes. diff --git a/src/utils/types.ts b/src/utils/types.ts index 32b67733..f6addcad 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -119,6 +119,12 @@ export type InstallerOptions = { */ maxRetries?: number; + /** + * Observe every message from the agent session. Used by evals to record tool + * calls and latency while running the real integration flow. + */ + onAgentMessage?: (message: import('@anthropic-ai/claude-agent-sdk').SDKMessage) => void; + /** * Scaffold a new Next.js app when run in an empty directory. * Auto-enabled in headless mode; opt-in via --scaffold in interactive mode. diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index fc3b8b66..f940b5b6 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -4,8 +4,9 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; // Hoist mocks so they're available in vi.mock factories -const { mockRunAgent, mockConfig, mockCredentials } = vi.hoisted(() => ({ +const { mockRunAgent, mockIntegrationRun, mockConfig, mockCredentials } = vi.hoisted(() => ({ mockRunAgent: vi.fn(), + mockIntegrationRun: vi.fn(), mockConfig: { model: 'test-model', workos: { clientId: 'client_test', authkitDomain: 'test.workos.com', llmGatewayUrl: 'http://localhost:8000' }, @@ -34,6 +35,13 @@ vi.mock('../../../src/lib/agent-interface.js', () => ({ runAgent: mockRunAgent, })); +// The eval runs the real integration flow; the integration itself is mocked here. +vi.mock('../../../src/lib/registry.js', () => ({ + getRegistry: vi.fn(async () => ({ + get: (name: string) => (name === 'unknown' ? undefined : { run: mockIntegrationRun }), + })), +})); + vi.mock('../../../src/lib/agent-sdk-assets.js', () => ({ ensureClaudeCodeExecutable: vi.fn(async () => '/synthetic/claude'), })); @@ -92,193 +100,133 @@ describe('AgentExecutor', () => { // Create package.json so env writing works writeFileSync(join(testDir, 'package.json'), JSON.stringify({ name: 'test' })); mockRunAgent.mockReset(); + mockIntegrationRun.mockReset(); + mockConfig.model = 'test-model'; + delete process.env.EVAL_MODEL; }); afterEach(() => { rmSync(testDir, { recursive: true, force: true }); }); - it('calls production runAgent with correct AgentRunConfig', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('runs the real integration with installer options like production', async () => { + mockIntegrationRun.mockResolvedValue('done'); const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - expect(mockRunAgent).toHaveBeenCalledTimes(1); - - const [agentRunConfig] = mockRunAgent.mock.calls[0]; - expect(agentRunConfig.workingDirectory).toBe(testDir); - expect(agentRunConfig.model).toBe('test-model'); - // Must match production so every tool call goes through installerCanUseTool - expect(agentRunConfig.allowedTools).toEqual([]); - expect(agentRunConfig.mcpServers).toBeUndefined(); - // Direct mode — no gateway URL - expect(agentRunConfig.sdkEnv.ANTHROPIC_API_KEY).toBe('sk-ant-test'); - expect(agentRunConfig.sdkEnv.ANTHROPIC_BASE_URL).toBeUndefined(); - }); + const result = await executor.run(); - it('uses isolated Ruby environment and explicit callback without inheriting host secrets', async () => { - mockRunAgent.mockResolvedValue({}); - const executor = new AgentExecutor(testDir, 'ruby', { - environment: { HOME: '/synthetic/home', PATH: '/synthetic/bin' }, - redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + expect(result.success).toBe(true); + expect(mockIntegrationRun).toHaveBeenCalledTimes(1); + const [options] = mockIntegrationRun.mock.calls[0]; + expect(options).toMatchObject({ + installDir: testDir, + ci: true, + skipAuth: true, + direct: true, + apiKey: 'sk_test_key', + clientId: 'client_test_id', + noValidate: false, + maxRetries: 2, }); - await executor.run({ enabled: false, maxRetries: 0 }); - const [config, prompt] = mockRunAgent.mock.calls[0]; - expect(config.sdkEnv.HOME).toBe('/synthetic/home'); - expect(config.sdkEnv.PATH).toBe('/synthetic/bin'); - expect(config.sdkEnv).not.toHaveProperty('WORKOS_API_KEY'); - expect(prompt).toContain('configured in .env:'); - expect(prompt).not.toContain('configured in .env.local'); - expect(prompt).toContain('WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback'); - expect(readFileSync(join(testDir, '.env'), 'utf8')).toContain( - 'WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback', - ); + expect(typeof options.onAgentMessage).toBe('function'); + expect(options.emitter).toBeDefined(); + expect(mockRunAgent).not.toHaveBeenCalled(); }); - it('passes RetryConfig when correction is enabled', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('disables validation retries when correction is disabled', async () => { + mockIntegrationRun.mockResolvedValue('done'); - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run({ enabled: true, maxRetries: 3 }); + await new AgentExecutor(testDir, 'nextjs').run({ enabled: false, maxRetries: 0 }); - const retryConfig = mockRunAgent.mock.calls[0][5]; // 6th arg - expect(retryConfig).toBeDefined(); - expect(retryConfig.maxRetries).toBe(3); - expect(typeof retryConfig.validateAndFormat).toBe('function'); + expect(mockIntegrationRun.mock.calls[0][0]).toMatchObject({ noValidate: true, maxRetries: 0 }); }); - it('passes no RetryConfig when correction is disabled', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run({ enabled: false, maxRetries: 0 }); - - const retryConfig = mockRunAgent.mock.calls[0][5]; - expect(retryConfig).toBeUndefined(); - }); - - it('passes InstallerOptions with skipAuth=true', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - const installerOptions = mockRunAgent.mock.calls[0][2]; // 3rd arg - expect(installerOptions.skipAuth).toBe(true); - expect(installerOptions.installDir).toBe(testDir); - }); - - it('passes onMessage callback as 7th argument', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - const onMessage = mockRunAgent.mock.calls[0][6]; // 7th arg - expect(typeof onMessage).toBe('function'); - }); - - it('maps retryCount=0 to correctionAttempts=0, selfCorrected=false', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); - - expect(result.success).toBe(true); - expect(result.correctionAttempts).toBe(0); - expect(result.selfCorrected).toBe(false); - }); + it('counts agent:retry events as correction attempts', async () => { + mockIntegrationRun.mockImplementation(async (options) => { + options.emitter.emit('agent:retry', { attempt: 1, maxRetries: 2 }); + options.emitter.emit('agent:retry', { attempt: 2, maxRetries: 2 }); + return 'done'; + }); - it('maps retryCount>0 to selfCorrected=true on success', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 2 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); expect(result.success).toBe(true); expect(result.correctionAttempts).toBe(2); expect(result.selfCorrected).toBe(true); }); - it('maps runAgent error result to failed AgentResult', async () => { - mockRunAgent.mockResolvedValue({ - error: 'EXECUTION_ERROR', - errorMessage: 'SDK crashed', - retryCount: 1, - }); + it('maps a thrown integration error to a failed result', async () => { + mockIntegrationRun.mockRejectedValue(new Error('Installation validation failed')); - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); expect(result.success).toBe(false); - expect(result.error).toBe('SDK crashed'); - expect(result.correctionAttempts).toBe(1); + expect(result.error).toBe('Installation validation failed'); expect(result.selfCorrected).toBe(false); }); - it('handles runAgent throwing an exception', async () => { - mockRunAgent.mockRejectedValue(new Error('Connection refused')); + it('records tool calls and text from agent messages', async () => { + mockIntegrationRun.mockImplementation(async (options) => { + options.onAgentMessage({ + type: 'assistant', + message: { + content: [ + { type: 'text', text: 'Installing AuthKit...' }, + { type: 'tool_use', name: 'Read', input: { file_path: 'package.json' } }, + ], + }, + }); + return 'done'; + }); - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); - expect(result.success).toBe(false); - expect(result.error).toBe('Connection refused'); - expect(result.correctionAttempts).toBe(0); + expect(result.output).toContain('Installing AuthKit...'); + expect(result.toolCalls).toEqual([expect.objectContaining({ tool: 'Read' })]); }); - it('writes env vars before calling runAgent', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('applies EVAL_MODEL to the installer config', async () => { + process.env.EVAL_MODEL = 'claude-opus-5-5'; + mockIntegrationRun.mockResolvedValue('done'); - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); + await new AgentExecutor(testDir, 'nextjs').run(); - expect(writeEnvLocal).toHaveBeenCalledWith(testDir, { - WORKOS_API_KEY: 'sk_test_key', - WORKOS_CLIENT_ID: 'client_test_id', - }); + expect(mockConfig.model).toBe('claude-opus-5-5'); }); - it('onMessage callback collects text output from assistant messages', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - // Get the onMessage callback and simulate a message - const onMessage = mockRunAgent.mock.calls[0][6]; - onMessage({ - type: 'assistant', - message: { - content: [{ type: 'text', text: 'Installing AuthKit...' }], - }, + it('writes env vars before running the integration', async () => { + mockIntegrationRun.mockImplementation(async () => { + expect(writeEnvLocal).toHaveBeenCalledWith(testDir, { + WORKOS_API_KEY: 'sk_test_key', + WORKOS_CLIENT_ID: 'client_test_id', + }); + return 'done'; }); - // Run again to verify output is collected (can't check internal state, - // but we can verify it doesn't throw) - expect(onMessage).toBeDefined(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); + expect(result.success).toBe(true); }); - it('builds prompt with correct skill name for framework', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'react-router'); - await executor.run(); - - const prompt = mockRunAgent.mock.calls[0][1]; // 2nd arg - expect(prompt).toContain('workos-authkit-react-router'); - expect(prompt).toContain('react-router'); + it('rejects a framework with no installer integration', async () => { + await expect(new AgentExecutor(testDir, 'unknown').run()).rejects.toThrow('No installer integration'); }); - it('defaults to correction enabled with maxRetries=2', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); // no retryConfig arg — uses default - - const retryConfig = mockRunAgent.mock.calls[0][5]; - expect(retryConfig).toBeDefined(); - expect(retryConfig.maxRetries).toBe(2); + it('uses isolated Ruby environment and explicit callback without inheriting host secrets', async () => { + mockRunAgent.mockResolvedValue({}); + const executor = new AgentExecutor(testDir, 'ruby', { + environment: { HOME: '/synthetic/home', PATH: '/synthetic/bin' }, + redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + }); + await executor.run({ enabled: false, maxRetries: 0 }); + const [config, prompt] = mockRunAgent.mock.calls[0]; + expect(config.sdkEnv.HOME).toBe('/synthetic/home'); + expect(config.sdkEnv.PATH).toBe('/synthetic/bin'); + expect(config.sdkEnv).not.toHaveProperty('WORKOS_API_KEY'); + expect(prompt).toContain('configured in .env:'); + expect(prompt).not.toContain('configured in .env.local'); + expect(prompt).toContain('WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback'); + expect(readFileSync(join(testDir, '.env'), 'utf8')).toContain( + 'WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback', + ); }); }); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index d43a2806..b2ccf236 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -9,6 +9,8 @@ import { quickCheckValidateAndFormat } from '../../src/lib/validation/quick-chec import { runAgent, type AgentRunConfig, type RetryConfig } from '../../src/lib/agent-interface.js'; import { ensureClaudeCodeExecutable } from '../../src/lib/agent-sdk-assets.js'; import type { InstallerOptions } from '../../src/utils/types.js'; +import { getRegistry } from '../../src/lib/registry.js'; +import { createInstallerEventEmitter } from '../../src/lib/events.js'; import type { ToolCall, LatencyMetrics } from './types.js'; import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; @@ -123,6 +125,11 @@ export class AgentExecutor { writeEnvFile(this.workDir, envVars); } + if (!this.options.environment) { + return this.runProductionIntegration(config, toolCalls, collectedOutput, label); + } + + // Isolated real-app fixtures (Fizzy) keep their own allowlisted environment and prompt. const skillName = SKILL_NAMES[this.framework]; const prompt = this.buildPrompt(skillName); @@ -205,6 +212,63 @@ export class AgentExecutor { } } + /** + * Run the framework's real installer integration, so the eval measures the same + * prompt, tool gate, and validation/retry loop that users get. + */ + private async runProductionIntegration( + config: AgentRetryConfig, + toolCalls: ToolCall[], + collectedOutput: string[], + label: string, + ): Promise { + const integration = (await getRegistry()).get(this.framework); + if (!integration) throw new Error(`No installer integration for framework: ${this.framework}`); + + if (process.env.EVAL_MODEL) getConfig().model = process.env.EVAL_MODEL; + + let correctionAttempts = 0; + const emitter = createInstallerEventEmitter(); + emitter.on('agent:retry', () => correctionAttempts++); + + const options: InstallerOptions = { + debug: this.options.verbose ?? false, + forceInstall: false, + installDir: this.workDir, + local: false, + ci: true, + skipAuth: true, + direct: true, + // Passing credentials marks configuration as caller-handled: no dashboard calls. + apiKey: this.credentials.workosApiKey, + clientId: this.credentials.workosClientId, + ...(this.options.redirectUri ? { redirectUri: this.options.redirectUri } : {}), + noValidate: !config.enabled, + maxRetries: config.maxRetries, + emitter, + onAgentMessage: (message) => this.trackMessage(message, toolCalls, collectedOutput, label), + }; + + const base = () => ({ + output: collectedOutput.join('\n'), + toolCalls, + latencyMetrics: this.latencyTracker.finish(), + correctionAttempts, + }); + + try { + await integration.run(options); + return { ...base(), success: true, selfCorrected: correctionAttempts > 0 }; + } catch (error) { + return { + ...base(), + success: false, + error: error instanceof Error ? error.message : String(error), + selfCorrected: false, + }; + } + } + private buildPrompt(skillName: string): string { return `You are integrating WorkOS AuthKit into this application. diff --git a/tests/evals/index.ts b/tests/evals/index.ts index 56f6d84a..22cd3ebd 100644 --- a/tests/evals/index.ts +++ b/tests/evals/index.ts @@ -8,6 +8,8 @@ import { diffRuns, printDiff } from './commands/diff.js'; import { listHistory, pruneHistory } from './commands/history.js'; async function main() { + // Eval runs call the real installer integrations; keep them out of installer telemetry. + process.env.WORKOS_TELEMETRY ??= 'false'; const options = parseArgs(process.argv.slice(2)); if (options.help) { diff --git a/tests/fixtures/nextjs/conflicting-middleware/package.json b/tests/fixtures/nextjs/conflicting-middleware/package.json index 40cd3b2e..4639d37e 100644 --- a/tests/fixtures/nextjs/conflicting-middleware/package.json +++ b/tests/fixtures/nextjs/conflicting-middleware/package.json @@ -8,14 +8,14 @@ "start": "next start" }, "dependencies": { - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/example-auth0/package.json b/tests/fixtures/nextjs/example-auth0/package.json index 5d2912fc..5feff399 100644 --- a/tests/fixtures/nextjs/example-auth0/package.json +++ b/tests/fixtures/nextjs/example-auth0/package.json @@ -9,14 +9,14 @@ }, "dependencies": { "@auth0/nextjs-auth0": "^3.5.0", - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/example/package.json b/tests/fixtures/nextjs/example/package.json index e5de786d..6f7a7a9b 100644 --- a/tests/fixtures/nextjs/example/package.json +++ b/tests/fixtures/nextjs/example/package.json @@ -8,14 +8,14 @@ "start": "next start" }, "dependencies": { - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/partial-install/package.json b/tests/fixtures/nextjs/partial-install/package.json index e0332a1c..a913cc59 100644 --- a/tests/fixtures/nextjs/partial-install/package.json +++ b/tests/fixtures/nextjs/partial-install/package.json @@ -8,15 +8,15 @@ "start": "next start" }, "dependencies": { - "@workos-inc/authkit-nextjs": "^0.15.0", - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "@workos-inc/authkit-nextjs": "^4.0.0", + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } }