Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions apps/sim/executor/handlers/pi/cloud/cli-runtime.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import { execFile } from 'node:child_process'
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { promisify } from 'node:util'
import { describe, expect, it } from 'vitest'
import { PI_EVENT_FILTER_SOURCE } from '@/executor/handlers/pi/cloud/event-filter-source'
import { buildPiScript } from '@/executor/handlers/pi/cloud/shared'
import { PI_PACKAGE_VERSION } from '@/scripts/pi-sandbox-packages'

const exec = promisify(execFile)

async function invokeSandbox(version: string) {
const root = await mkdtemp(join(tmpdir(), 'sim-pi-cli-'))
const workspace = join(root, 'workspace')
const repo = join(workspace, 'repo')
const bin = join(root, 'bin')
await mkdir(repo, { recursive: true })
await mkdir(bin)
await writeFile(join(workspace, 'pi-prompt.txt'), 'fixture prompt')
await writeFile(join(workspace, 'sim-pi-event-filter.mjs'), PI_EVENT_FILTER_SOURCE)
await writeFile(
join(bin, 'pi'),
`#!/bin/bash
if [ "$1" = "--version" ]; then
printf "%s\\n" "${version}"
exit 0
fi
printf "%s" "$PI_CODING_AGENT_DIR" > "${root}/invoked"
printf "%s\\n" '{"type":"message_update","assistantMessageEvent":{"type":"text_delta","delta":"ok"}}'
exit 0
Comment thread
BillLeoutsakosvl346 marked this conversation as resolved.
`,
{ mode: 0o700 }
)
const script = buildPiScript().replaceAll('/workspace', workspace)
try {
const result = await exec('/bin/bash', ['-c', script], {
env: {
...process.env,
PATH: `${bin}:${process.env.PATH}`,
PI_PROVIDER: 'fixture',
PI_MODEL: 'fixture',
PI_THINKING: 'off',
},
}).then(
({ stdout, stderr }) => ({ code: 0, stdout, stderr }),
(error: { code: number; stdout: string; stderr: string }) => error
)
const invoked = await readFile(join(root, 'invoked'), 'utf8').catch(() => null)
const settings = invoked
? await readFile(join(invoked, 'settings.json'), 'utf8').catch(() => null)
: null
const storedFiles = invoked ? await readdir(invoked).catch(() => []) : []
return { ...result, invoked, settings, storedFiles, repo }
} finally {
await rm(root, { recursive: true, force: true })
}
}

describe('sandbox Pi runtime boundary', () => {
it.each(['0.80.10', 'unexpected-version'])(
'rejects %s before starting the agent',
async (version) => {
const result = await invokeSandbox(version)
expect(result.code).not.toBe(0)
expect(result.invoked).toBeNull()
expect(result.stderr).toMatch(/rebuild|update/i)
expect(result.stderr).toContain(PI_PACKAGE_VERSION)
}
)

it('uses private settings outside the repository with warming off and no saved credentials', async () => {
const result = await invokeSandbox(PI_PACKAGE_VERSION)
expect(result.code).toBe(0)
expect(result.invoked).toBeTruthy()
expect(result.invoked?.startsWith(result.repo)).toBe(false)
expect(JSON.parse(result.settings ?? '{}')).toEqual({ cacheWarming: 'off' })
expect(result.storedFiles).toEqual(['settings.json'])
expect(JSON.parse(result.stdout.trim())).toEqual({
type: 'message_update',
assistantMessageEvent: { type: 'text_delta', delta: 'ok' },
})
})
})
7 changes: 3 additions & 4 deletions apps/sim/executor/handlers/pi/cloud/event-filter-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@
* The stdout filter the sandbox modes pipe the Pi CLI through, written at runtime like the search
* extension and the review tools script next to it.
*
* Pi's `--mode json` writes `JSON.stringify(event)` for every session event with no filtering, and
* `message_update` repeats the whole assistant message alongside each delta — so raw stdout grows
* with the square of the response length, and `tool_execution_end`, `turn_end`, and `agent_end`
* each add a full tool result, turn transcript, or run transcript on top of that.
* Pi 1.0 emits delta-only `message_update` events, but `tool_execution_end`, `turn_end`, and
* `agent_end` still include tool results and transcripts. The filter preserves Sim's event
* contract without retaining those cumulative payloads.
*
* The reduction has to happen in the sandbox because by the time Sim could drop the bytes they are
* already retained: E2B's SDK accumulates every callback-delivered chunk internally, so its adapter
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/executor/handlers/pi/cloud/review/backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const mockAgentSession = {
const sealedResourceLoader = { kind: 'sealed' }

const mockSdk = {
SettingsManager: { inMemory: vi.fn(() => ({})) },
SettingsManager: { inMemory: () => ({}) },
SessionManager: { inMemory: vi.fn(() => ({})) },
createAgentSession: mockCreateAgentSession,
defineTool: vi.fn((tool) => tool),
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/executor/handlers/pi/cloud/review/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ export const runCloudReviewPi: PiBackendRun<PiCloudReviewRunParams> = async (par
)
}

const settingsManager = sdk.SettingsManager.inMemory()
const settingsManager = sdk.SettingsManager.inMemory({ cacheWarming: 'off' })
const resourceLoader = createSealedPiResourceLoader(
sdk,
buildReviewSystemPrompt(Boolean(searchTool))
Expand Down
17 changes: 16 additions & 1 deletion apps/sim/executor/handlers/pi/cloud/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
import { resolvePiSandboxLifetimeMs } from '@/lib/execution/remote-sandbox/pi-lifetime'
import { PI_EVENT_FILTER_PATH } from '@/executor/handlers/pi/cloud/event-filter-source'
import { scrubPiSecrets } from '@/executor/handlers/pi/core/redaction'
import { PI_PACKAGE_VERSION } from '@/scripts/pi-sandbox-packages'

export const REPO_DIR = '/workspace/repo'
export const PROMPT_PATH = '/workspace/pi-prompt.txt'
Expand Down Expand Up @@ -141,6 +142,10 @@ export const PUSH_SCRIPT = `cd ${REPO_DIR}
* there first — skipping that write does not fall back to the raw stream, it fails the run on the
* missing module.
*
* Rejects stale images before starting the agent. Each invocation resets Sim's private agent
* directory outside the clone, so neither stored credentials nor cache-warming settings can leak
* between rounds. Model credentials remain environment-only.
*
* Selects `/bin/bash` explicitly because `pipefail` is not portable to `/bin/sh`, and without it
* the pipeline reports the filter's exit code rather than Pi's, so an upstream crash would read as
* a clean run. Both dedicated Pi images are Debian-based and provide Bash, so provider
Expand All @@ -165,7 +170,17 @@ export function buildPiScript(
? ' --no-extensions'
: ''
const extensionArgs = extensionPath ? ` -e ${extensionPath}` : ''
return `/bin/bash -o pipefail -c 'cd ${REPO_DIR}
return `/bin/bash -o pipefail -c 'set -e
if ! PI_INSTALLED_VERSION="$(pi --version 2>/dev/null)" || [ "$PI_INSTALLED_VERSION" != "${PI_PACKAGE_VERSION}" ]; then
printf "%s\\n" "Pi sandbox runtime must be ${PI_PACKAGE_VERSION}. Rebuild or update the configured Pi sandbox image before running this workflow." >&2
exit 1
fi
export PI_CODING_AGENT_DIR=/workspace/sim-pi-agent
(umask 077
rm -rf "$PI_CODING_AGENT_DIR"
mkdir -p "$PI_CODING_AGENT_DIR"
printf "%s\\n" "{\\"cacheWarming\\":\\"off\\"}" > "$PI_CODING_AGENT_DIR/settings.json")
cd ${REPO_DIR}
pi -p --mode json --provider "$PI_PROVIDER" --model "$PI_MODEL" --thinking "$PI_THINKING"${repositoryArgs}${extensionArgs} < ${PROMPT_PATH} | node ${PI_EVENT_FILTER_PATH}'`
}

Expand Down
9 changes: 5 additions & 4 deletions apps/sim/executor/handlers/pi/core/pi-sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ function isToolArguments(value: unknown): value is Record<string, unknown> {
* credentials from thrown and reported tool errors. Successful tool output is ordinary model
* content and stays verbatim; Sim-secret projection is owned by the tool adapter's provenance.
*
* A spec's `isError` is rethrown rather than reported in the result: Pi derives a call's error state
* solely from whether `execute` threw, so a resolved failure would reach the model as a successful
* tool call whose text happens to describe a failure. Throwing also matches Pi's own `bash`, which
* throws on a non-zero exit with the output appended, so the failure text survives either way.
* A spec's `isError` is rethrown with scrubbed text, preserving Sim's existing failure semantics.
* Pi 1.0 also accepts an explicit error result, but throwing keeps the same diagnostic boundary
* for reported failures and exceptions.
*
* Shared by both host-side backends: Local Dev converts its SSH, Sim, and search tools here, and
* Review Code converts its search tool here alongside the review tools it builds directly.
Expand Down Expand Up @@ -90,7 +89,9 @@ export function createSealedPiResourceLoader(sdk: PiSdk, systemPrompt: string):
getThemes: () => ({ themes: [], diagnostics: [] }),
getAgentsFiles: () => ({ agentsFiles: [] }),
getSystemPrompt: () => systemPrompt,
getSystemPromptSource: () => undefined,
getAppendSystemPrompt: () => [],
getAppendSystemPromptSources: () => [],
extendResources: () => {},
reload: async () => {},
}
Expand Down
1 change: 1 addition & 0 deletions apps/sim/executor/handlers/pi/local/backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const mockAgentSession = {
}
const mockSdk = {
defineTool: vi.fn((tool) => tool),
SettingsManager: { inMemory: () => ({}) },
SessionManager: { inMemory: vi.fn(() => ({})) },
createAgentSession: mockCreateAgentSession,
}
Expand Down
1 change: 1 addition & 0 deletions apps/sim/executor/handlers/pi/local/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ async function runLocalAgent(
noTools: 'builtin',
customTools,
modelRuntime,
settingsManager: sdk.SettingsManager.inMemory({ cacheWarming: 'off' }),
sessionManager: sdk.SessionManager.inMemory(isolatedDir),
})

Expand Down
8 changes: 6 additions & 2 deletions apps/sim/executor/handlers/pi/pi-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,13 +213,17 @@ describe('PiBlockHandler', () => {
expect(mockRunLocal).not.toHaveBeenCalled()
})

it('rejects an unavailable model before resolving credentials', async () => {
it('rejects an unavailable model with replacement guidance before credentials or remote setup', async () => {
mockResolvePiModelId.mockReturnValue(undefined)

await expect(handler.execute(ctx(), block, localInputs())).rejects.toThrow(
/not available.*installed Pi catalog/
/not available.*Choose a supported Pi model/
)
expect(mockResolveKey).not.toHaveBeenCalled()
expect(mockRunLocal).not.toHaveBeenCalled()
expect(mockRunCloud).not.toHaveBeenCalled()
expect(mockRunCloudPlan).not.toHaveBeenCalled()
expect(mockRunCloudReview).not.toHaveBeenCalled()
})

it('adds successful Function tool cost once to a non-streaming Local Dev result', async () => {
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/executor/handlers/pi/pi-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ export class PiBlockHandler implements BlockHandler {
const piModel = resolvePiModelId(providerId, model)
if (!piModel) {
throw new Error(
`Pi model "${model}" is not available for provider "${providerId}" in the installed Pi catalog`
`Pi model "${model}" is not available for provider "${providerId}" in the installed Pi catalog. Choose a supported Pi model in the block settings.`
)
}

Expand Down
4 changes: 2 additions & 2 deletions apps/sim/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@
"@cerebras/cerebras_cloud_sdk": "^1.23.0",
"@daytona/sdk": "0.207.0",
"@e2b/code-interpreter": "2.7.1",
"@earendil-works/pi-ai": "0.80.10",
"@earendil-works/pi-coding-agent": "0.80.10",
"@earendil-works/pi-ai": "1.0.0",
"@earendil-works/pi-coding-agent": "1.0.0",
"@floating-ui/dom": "1.7.6",
"@google-cloud/storage": "7.21.0",
"@google/genai": "2.13.0",
Expand Down
Loading
Loading