Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .agents/skills/run-codex/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@ npm run codex-test -- -p "Your prompt here"

| Option | Short | Description | Default |
|--------|-------|-------------|---------|
| `--prompt` | `-p` | Prompt to send to Codex (required) | - |
| `--prompt` | `-p` | Prompt to send to Codex (unless listing models) | - |
| `--cwd` | `-c` | Working directory for the session | Current dir |
| `--output` | `-o` | Output format: `all`, `codex`, `acp`, `summary` | `all` |
| `--json` | - | Output events as JSON | false |
| `--list-models` | - | Read live model IDs and efforts without a turn | false |
| `--model` | `-m` | Exact advertised `id[effort]` | Codex default |
| `--mcp-servers` | - | JSON file containing ACP MCP server entries | none |
| `--help` | `-h` | Show help message | - |

### Examples
Expand Down Expand Up @@ -72,3 +75,5 @@ The script outputs events in real-time as they arrive:
- Event type frequency

4. **PromptResponse**: Full response object including `_meta.quota.token_count`

The harness honors `CODEX_PATH` and `CODEX_CONFIG`. Run `--list-models` before choosing a model; never guess model IDs. Client permission requests are cancelled and fail the probe. JSON mode emits JSON lines; use `npm run --silent` to suppress npm banners. The harness is included in `npm run typecheck`.
275 changes: 86 additions & 189 deletions .agents/skills/run-codex/scripts/run-codex-test.ts
Original file line number Diff line number Diff line change
@@ -1,209 +1,106 @@
#!/usr/bin/env tsx
/**
* Run real Codex with a prompt to test and verify code during development.
*
* Usage:
* npm run codex-test -- --prompt "Your prompt here"
* npm run codex-test -- -p "Hello" -c /path/to/project
* npm run codex-test -- -p "Hello" -o codex --json
*/

import path from "node:path";
/** Real app-server/ACP probe. Never approves a permission request. */
import fs from "node:fs";
import {CodexAcpClient} from "../../../../src/CodexAcpClient";
import {type CodexConnectionEvent, CodexAppServerClient} from "../../../../src/CodexAppServerClient";
import {parseArgs} from "node:util";
import * as acp from "@agentclientprotocol/sdk";
import {CodexAcpClient, type JsonObject} from "../../../../src/CodexAcpClient";
import {CodexAppServerClient} from "../../../../src/CodexAppServerClient";
import {startCodexConnection} from "../../../../src/CodexJsonRpcConnection";
import {CodexAcpServer} from "../../../../src/CodexAcpServer";
import type {AcpClientConnection} from "../../../../src/ACPSessionConnection";
import {RequestError} from "@agentclientprotocol/sdk/unstable";

// Parse command line arguments
function parseArgs(): { prompt: string; cwd: string; output: string; json: boolean } {
const args = process.argv.slice(2);
let prompt = "";
let cwd = process.cwd();
let output = "all";
let json = false;

for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === "--prompt" || arg === "-p") {
prompt = args[++i] || "";
} else if (arg === "--cwd" || arg === "-c") {
cwd = args[++i] || process.cwd();
} else if (arg === "--output" || arg === "-o") {
output = args[++i] || "all";
} else if (arg === "--json") {
json = true;
} else if (arg === "--help" || arg === "-h") {
console.log(`
Usage: npm run codex-test -- [options]

Options:
-p, --prompt <text> Prompt to send to Codex (required)
-c, --cwd <path> Working directory for the session (default: current dir)
-o, --output <type> Output type: all, codex, acp, summary (default: all)
--json Output events as JSON
-h, --help Show this help message

Examples:
npm run codex-test -- -p "What files are here?"
npm run codex-test -- -p "Read README" -c /path/to/project
npm run codex-test -- -p "Hello" -o codex --json
`);
process.exit(0);
}
}

if (!prompt) {
console.error("Error: --prompt is required");
process.exit(1);
}

return { prompt, cwd, output, json };
const {values} = parseArgs({options: {
prompt: {type: "string", short: "p"},
cwd: {type: "string", short: "c", default: process.cwd()},
output: {type: "string", short: "o", default: "all"},
json: {type: "boolean", default: false},
model: {type: "string", short: "m"},
"list-models": {type: "boolean", default: false},
"mcp-servers": {type: "string"},
help: {type: "boolean", short: "h", default: false},
}});

if (values.help) {
console.log(`Usage: npm run codex-test -- [options]
-p, --prompt <text> Prompt (required unless listing models)
-c, --cwd <path> Session working directory
-o, --output <type> all, codex, acp, summary (default: all)
-m, --model <id[effort]> Exact ACP model ID from --list-models
--list-models Read the live model catalog without starting a turn
--mcp-servers <file> JSON array of ACP MCP server entries
--json Emit JSON lines only
-h, --help Show help
Uses the bundled Codex unless CODEX_PATH is set; forwards CODEX_CONFIG.
Permission requests are cancelled and make the run fail.`);
} else {
await main().catch(error => {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
});
}

type MethodCallEvent = { method: string; args: unknown[] };

function createMockAcpConnection(
events: MethodCallEvent[],
onNotification: (method: string, params: unknown) => void,
): AcpClientConnection {
return {
notify: async (method: string, params?: unknown) => {
events.push({method, args: [params]});
onNotification(method, params);
async function main(): Promise<void> {
if (!values.prompt && !values["list-models"]) throw new Error("--prompt or --list-models is required");
if (!["all", "codex", "acp", "summary"].includes(values.output)) throw new Error("Invalid --output");
const config: unknown = JSON.parse(process.env["CODEX_CONFIG"] ?? "{}");
if (!config || typeof config !== "object" || Array.isArray(config)) throw new Error("CODEX_CONFIG must be an object");
const mcpServers: acp.McpServer[] = values["mcp-servers"]
? JSON.parse(fs.readFileSync(values["mcp-servers"], "utf8")) : [];
if (!Array.isArray(mcpServers)) throw new Error("--mcp-servers must contain an array");
const emit = (type: string, data: unknown) => console.log(values.json
? JSON.stringify({type, data}) : `[${type}] ${JSON.stringify(data, null, 2)}`);
const connection = startCodexConnection(process.env["CODEX_PATH"]);
const client = new CodexAppServerClient(connection.connection);
const adapter = new CodexAcpClient(client, config as JsonObject);
let codexEvents = 0;
let acpEvents = 0;
let permissionRequests = 0;
const acpConnection: AcpClientConnection = {
notify: async (method: string, params: unknown) => {
acpEvents++;
if (values.output === "all" || values.output === "acp") emit("acp", {method, params});
},
request: async (method: string) => {
throw RequestError.methodNotFound(method);
if (method === acp.methods.client.session.requestPermission) {
permissionRequests++;
emit("permission-cancelled", {method});
return {outcome: {outcome: "cancelled"}} as never;
}
throw new Error(`Harness does not implement client request ${method}`);
},
};
}

async function main() {
const { prompt, cwd, output, json } = parseArgs();

// Find Codex binary
const pathToCodex = path.resolve(process.cwd(), "node_modules", ".bin", process.platform === "win32" ? "codex.cmd" : "codex");
if (!fs.existsSync(pathToCodex)) {
console.error(`Error: Codex binary not found at ${pathToCodex}. Did you run 'npm install'?`);
process.exit(1);
}

console.log("=".repeat(60));
console.log("Codex Test Runner");
console.log("=".repeat(60));
console.log(`Prompt: ${prompt}`);
console.log(`CWD: ${cwd}`);
console.log(`Output: ${output}`);
console.log("=".repeat(60));
console.log("");

// Start Codex connection
const codexConnection = startCodexConnection(pathToCodex);
const codexAppServerClient = new CodexAppServerClient(codexConnection.connection);

// Collect events
const codexEvents: CodexConnectionEvent[] = [];
const acpEvents: MethodCallEvent[] = [];

codexAppServerClient.onClientTransportEvent((event) => {
codexEvents.push(event);
if (output === "all" || output === "codex") {
if (json) {
console.log(JSON.stringify(event));
} else {
console.log(`[CODEX] ${event.eventType}:`, JSON.stringify(event, null, 2));
}
}
});

const acpConnection = createMockAcpConnection(acpEvents, (method, params) => {
if (output === "all" || output === "acp") {
if (json) {
console.log(JSON.stringify({method, params}));
} else {
console.log(`[ACP] ${method}:`, JSON.stringify(params, null, 2));
}
}
const agent = new CodexAcpServer(acpConnection, adapter, undefined, () => connection.process.exitCode);
client.onClientTransportEvent(event => {
codexEvents++;
if (values.output === "all" || values.output === "codex") emit("codex", event);
});
const codexAcpClient = new CodexAcpClient(codexAppServerClient);
const codexAcpAgent = new CodexAcpServer(acpConnection, codexAcpClient, undefined, () => codexConnection.process.exitCode);

try {
// Initialize
console.log("\n--- Initializing ---\n");
await codexAcpAgent.initialize({ protocolVersion: 1 });

// Check auth
const authRequirement = await codexAcpClient.readAuthRequirement();
if (authRequirement.required) {
console.error("Error: Authentication required. Please login to Codex first.");
process.exitCode = 1;
await agent.initialize({protocolVersion: acp.PROTOCOL_VERSION});
const models = await adapter.fetchAvailableModels();
const available = models.filter(model => !model.hidden).flatMap(model =>
model.supportedReasoningEfforts.map(({reasoningEffort}) => ({
modelId: `${model.id}[${reasoningEffort}]`,
model: model.model,
isDefault: model.isDefault && reasoningEffort === model.defaultReasoningEffort,
})));
if (values["list-models"]) {
emit("models", available);
return;
}

// Create session
console.log("\n--- Creating Session ---\n");
const sessionResponse = await codexAcpAgent.newSession({ cwd, mcpServers: [] });
console.log(`Session ID: ${sessionResponse.sessionId}`);
console.log(`Model: ${sessionResponse.models?.currentModelId}`);

// Send prompt
console.log("\n--- Sending Prompt ---\n");
const startTime = Date.now();
const promptResponse = await codexAcpAgent.prompt({
sessionId: sessionResponse.sessionId,
prompt: [{ type: "text", text: prompt }]
});
const duration = Date.now() - startTime;

// Summary
if (output === "all" || output === "summary") {
console.log("\n" + "=".repeat(60));
console.log("Summary");
console.log("=".repeat(60));
console.log(`Stop Reason: ${promptResponse.stopReason}`);
console.log(`Duration: ${duration}ms`);
console.log(`Codex Events: ${codexEvents.length}`);
console.log(`ACP Events: ${acpEvents.length}`);

const usage = promptResponse.usage;
if (usage) {
console.log(`\nToken Usage:`);
console.log(` Input Tokens: ${usage.inputTokens}`);
console.log(` Cached Input: ${usage.cachedReadTokens ?? 0}`);
console.log(` Output Tokens: ${usage.outputTokens}`);
console.log(` Reasoning Tokens: ${usage.thoughtTokens ?? 0}`);
console.log(` Total Tokens: ${usage.totalTokens}`);
}

// Event type breakdown
const eventTypes = new Map<string, number>();
for (const event of codexEvents) {
const key = `${event.eventType}:${"method" in event ? event.method : "response"}`;
eventTypes.set(key, (eventTypes.get(key) || 0) + 1);
}
console.log(`\nEvent Types:`);
for (const [type, count] of eventTypes) {
console.log(` ${type}: ${count}`);
}
if (values.model && !available.some(model => model.modelId === values.model)) {
throw new Error(`Model ${values.model} is not advertised; use --list-models`);
}

// Full response meta
if (output === "all") {
console.log("\n--- PromptResponse ---");
console.log(JSON.stringify(promptResponse, null, 2));
}

await codexAcpAgent.closeSession({sessionId: sessionResponse.sessionId});
} catch (err) {
console.error("Error:", err);
process.exitCode = 1;
if ((await adapter.readAuthRequirement()).required) throw new Error("Authentication required. Login to Codex first.");
const session = await agent.newSession({cwd: values.cwd, mcpServers, _meta: {mcpStartupAwaitTimeoutMs: 30_000}});
if (values.model) await agent.unstable_setSessionModel({sessionId: session.sessionId, modelId: values.model});
emit("session", {sessionId: session.sessionId, model: agent.getSessionState(session.sessionId).currentModelId});
const start = Date.now();
const response = await agent.prompt({sessionId: session.sessionId, prompt: [{type: "text", text: values.prompt!}]});
emit("summary", {response, durationMs: Date.now() - start, codexEvents, acpEvents, permissionRequests});
await agent.closeSession({sessionId: session.sessionId});
if (permissionRequests) throw new Error("Permission requests were cancelled; validation did not pass");
} finally {
codexConnection.connection.end();
codexConnection.process.kill();
connection.connection.end();
connection.process.kill();
}
}

main();
6 changes: 6 additions & 0 deletions .agents/skills/run-codex/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"extends": "../../../tsconfig.json",
"compilerOptions": {"rootDir": "../../..", "noEmit": true},
"include": ["scripts/**/*.ts"],
"exclude": []
}
34 changes: 34 additions & 0 deletions docs/pricing-column-regression-2026-10-24.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Pricing-column regression, 2026-10-24

The published OpenAI Markdown pricing table changed from
`input, cached input, cache writes, output` to
`input, output, cached input, 30m write, 12h write` within each context band.
The positional parser therefore read Astra cached input as $50/M instead of $1/M,
and output as $12.50/M instead of $50/M in standard short-context mode.

The reported $402.42 was reproduced exactly from this chat's recorded usage:
8,136,314 input tokens, including 8,014,208 cached input tokens, and 39,285 output
tokens. Fresh input was 122,106; cache writes were zero. The old column mapping
produced $402.4225225; the correct mapping produces an API-rate estimate of
$11.199518 for that same point in time. This is an estimate, not a billed invoice.

Source: https://developers.openai.com/api/docs/pricing.md, fetched 2026-10-24.
The regression fixture retains the relevant official table headers and Astra rows.

The parser now maps normalized, uniquely named headers, tolerates reordered/new
columns and repeated pricing sections, and rejects missing/ambiguous columns,
empty rates, and contradictory duplicates. Both prior and current table layouts
are covered. The change is shared by ACP v1 and v2 cost calculations.

Verification: 1,886 tests passed, 36 skipped; typecheck and build passed. A live
fetch through the corrected provider confirmed current standard/fast rates.

The current development chat's running adapter instance was narrowly repaired
under the Node debugger after verifying the exact session ID, standard Astra
model, zero cache writes and the known-bad rate mapping. Its accumulated amount
was recomputed from its existing cumulative counters; no messages or token usage
were rewritten. At repair, $518.5491025 became $14.171778 before the next usage
increment. Subsequent raw ACP usage records carry the corrected totals. The
breakpoint was removed and the debugger listener shut down. The built adapter
contains the durable parser fix; other already-running adapter instances are not
hot-patched and need a normal restart to load it.
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@
"test:integration": "vitest run --no-file-parallelism --hookTimeout=30000 src/__tests__/integration",
"test:e2e": "npm run build && RUN_E2E_TESTS=true vitest run --no-file-parallelism --retry=2 src/__tests__/CodexACPAgent/e2e",
"test:watch": "vitest",
"typecheck": "tsc --noEmit && tsc --noEmit -p examples/tsconfig.json",
"codex-test": "tsx .agents/skills/run-codex/scripts/run-codex-test.ts"
"typecheck": "tsc --noEmit && tsc --noEmit -p examples/tsconfig.json && tsc --noEmit -p .agents/skills/run-codex/tsconfig.json",
"codex-test": "node --import tsx .agents/skills/run-codex/scripts/run-codex-test.ts"
},
"homepage": "https://github.com/agentclientprotocol/codex-acp#readme",
"bugs": {
Expand Down
16 changes: 16 additions & 0 deletions readme-dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,22 @@ Command replies, review results, and terminal/retrying errors retain their exist
failure channels. Clients advertising session compaction support continue to receive the dedicated
compaction lifecycle instead of the legacy completion advisory.


### Real Codex model harness

`npm run codex-test -- --list-models --output summary --json` reads the live,
paginated app-server catalog without starting a model turn. Use an exact advertised
`id[effort]` with `--model` when running a prompt. The harness uses the bundled
Codex version unless `CODEX_PATH` is set and forwards `CODEX_CONFIG` to sessions.
`--mcp-servers <file>` accepts a JSON array of ACP MCP transport entries. All client
permission requests are cancelled and cause the probe to fail; it never grants
shell, filesystem, or MCP approval. `--json` emits JSON lines (use `npm run --silent`
to suppress npm's banner).

On 2026-10-23 the bundled 0.164.0 app-server advertised `gpt-6.1-sol[medium]` as
the default and `gpt-6-astra[low]` as an available alternative. The live-test fixture
uses these verified IDs; runtime model availability continues to come from Codex.

### Unsupported skills

The adapter disables Codex skills whose output ACP clients cannot show, and leaves them out of the skill commands and
Expand Down
Loading