diff --git a/.github/workflows/mobile-eas-production.yml b/.github/workflows/mobile-eas-production.yml index b6581e0e8ea49..4dd434a956e25 100644 --- a/.github/workflows/mobile-eas-production.yml +++ b/.github/workflows/mobile-eas-production.yml @@ -275,6 +275,7 @@ jobs: --environment production \ --platform ${{ inputs.platform }} \ --message "$OTA_MESSAGE" \ + --emit-metadata \ --non-interactive # Reconcile each version separately because the major is in the fingerprint. @@ -313,8 +314,11 @@ jobs: --environment production \ --platform "$platform" \ --message "$message" \ + --emit-metadata \ --non-interactive; then echo ":white_check_mark: $platform/$profile: OTA published to production (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY" + # The next target's export replaces dist; keep this one for the map upload. + rm -rf "dist-$platform-$profile" && mv dist "dist-$platform-$profile" else failed=1 echo ":x: $platform/$profile: OTA publish failed (fingerprint \`$hash\`)" >> "$GITHUB_STEP_SUMMARY" @@ -354,3 +358,18 @@ jobs: fi done exit "$failed" + + # Lets the relay symbolicate crash stacks from the updates published + # above. A step of its own so the R2 credentials reach only this script, + # not `eas update` and the dependencies it runs. Never fails the job. + - name: Upload OTA source maps + if: ${{ !cancelled() && steps.expo-token.outputs.present == 'true' }} + working-directory: apps/mobile + env: + MOBILE_SOURCEMAPS_R2: ${{ secrets.MOBILE_SOURCEMAPS_R2 }} + run: | + for dir in dist dist-*; do + if [ -f "$dir/eas-update-metadata.json" ]; then + node scripts/upload-sourcemaps.mts ota "$dir" + fi + done diff --git a/apps/desktop/scripts/verify-preload-bundle.mjs b/apps/desktop/scripts/verify-preload-bundle.mjs index f6976220d07fe..b1335aea1fde8 100644 --- a/apps/desktop/scripts/verify-preload-bundle.mjs +++ b/apps/desktop/scripts/verify-preload-bundle.mjs @@ -108,6 +108,7 @@ const executeBundle = (source, sandboxModules) => { { process: sandboxProcess, require: requireSandboxModule, + window: new EventTarget(), }, { filename: "desktop-preload.cjs", diff --git a/apps/desktop/scripts/verify-preload-bundle.test.mjs b/apps/desktop/scripts/verify-preload-bundle.test.mjs index 8af6ae00c3f92..5aa27368f68db 100644 --- a/apps/desktop/scripts/verify-preload-bundle.test.mjs +++ b/apps/desktop/scripts/verify-preload-bundle.test.mjs @@ -15,6 +15,16 @@ const validPreload = ` `; describe("desktop preload bundle verifier", () => { + it("accepts preloads that register window lifecycle listeners", () => { + assert.doesNotThrow(() => + verifyPreloadBundle(` + ${validPreload} + window.addEventListener("DOMContentLoaded", () => {}, { once: true }); + window.addEventListener("resize", () => {}); + `), + ); + }); + it("rejects required API names that only appear in strings", () => { assert.throws( () => diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 92e587f171c7b..980da9c83c33a 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -17,6 +17,12 @@ const runtimeVersionPolicy = const personalTeamBundleIdentifier = repoEnv.T3CODE_IOS_PERSONAL_TEAM_BUNDLE_ID?.trim(); const IOS_BUNDLE_IDENTIFIER_PATTERN = /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/; +// Compiled into the native project. Without a relay, Observe's native side could dispatch to +// Expo's own ingest before observe.ts turns it off; `.invalid` never resolves (RFC 6761). +const observeEndpointUrl = repoEnv.T3CODE_RELAY_URL?.trim() + ? new URL("/v1/mobile/observe", repoEnv.T3CODE_RELAY_URL.trim()).toString() + : "https://observe.invalid"; + const fromRepoRoot = (relativePath: string) => `../../${relativePath}`; const androidAdaptiveForeground = "./assets/android-icon-foreground.png"; @@ -450,6 +456,7 @@ const config: ExpoConfig = { }, eas: { projectId: "d763fcb8-d37c-41ea-a773-b54a0ab4a454", + observe: { endpointUrl: observeEndpointUrl }, }, }, owner: "pingdotgg", diff --git a/apps/mobile/eas.json b/apps/mobile/eas.json index f694fc8d6ef26..f773c2c5bbc3a 100644 --- a/apps/mobile/eas.json +++ b/apps/mobile/eas.json @@ -66,7 +66,8 @@ "corepack": true, "env": { "APP_VARIANT": "production", - "NODE_OPTIONS": "--max-old-space-size=4096" + "NODE_OPTIONS": "--max-old-space-size=4096", + "SOURCEMAP_FILE": "/tmp/t3code-ios-main.jsbundle.map" }, "channel": "production", "environment": "production", diff --git a/apps/mobile/index.ts b/apps/mobile/index.ts index 6aac4869d9717..169d5cc4048ba 100644 --- a/apps/mobile/index.ts +++ b/apps/mobile/index.ts @@ -1,5 +1,7 @@ // Must stay first: the render counter wraps the DevTools hook before any React commit. import "./src/features/development/render-counter-entry"; +// Next, so errors thrown while the rest of the app loads are recorded too. +import "./src/features/observability/observe"; import { registerRootComponent } from "expo"; import "react-native-gesture-handler"; import { LogBox } from "react-native"; diff --git a/apps/mobile/package.json b/apps/mobile/package.json index f5e817165b957..42d81c8adada0 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -46,6 +46,7 @@ "measure-renders": "node scripts/measure-renders.mts", "generate": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/generate-uniwind-themes.mts", "generate:highlights": "node scripts/generate-highlights.mts", + "eas-build-on-success": "node scripts/upload-sourcemaps.mts build", "test": "vp test run --project unit --project react", "typecheck": "tsc --noEmit" }, @@ -103,6 +104,7 @@ "expo-linking": "~58.0.13", "expo-network": "~58.0.3", "expo-notifications": "~58.1.5", + "expo-observe": "~58.0.15", "expo-paste-input": "^0.1.15", "expo-quick-actions": "^6.0.2", "expo-secure-store": "~58.0.2", @@ -145,6 +147,7 @@ "@types/mdast": "^4.0.4", "@types/react": "catalog:", "@types/react-dom": "catalog:", + "aws4fetch": "1.0.20", "babel-preset-expo": "~58.0.17", "jsdom": "^30.0.1", "tailwindcss": "^4.0.0", diff --git a/apps/mobile/scripts/sourcemap-upload.mts b/apps/mobile/scripts/sourcemap-upload.mts new file mode 100644 index 0000000000000..a962f98a37805 --- /dev/null +++ b/apps/mobile/scripts/sourcemap-upload.mts @@ -0,0 +1,165 @@ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import { mobileSourceMapKey, type MobilePlatform } from "@t3tools/shared/mobileSourceMaps"; +import { AwsClient } from "aws4fetch"; + +export interface SourceMapUpload { + readonly key: string; + readonly path: string; +} + +/** The `MOBILE_SOURCEMAPS_R2` secret: the relay stack's `mobileSourceMapsUploadCredentials`. */ +interface UploadCredentials { + readonly endpoint: string; + readonly bucketName: string; + readonly accessKeyId: string; + readonly secretAccessKey: string; +} + +const isPlatform = (value: unknown): value is MobilePlatform => + value === "ios" || value === "android"; + +const readJson = (path: string): unknown => JSON.parse(NodeFS.readFileSync(path, "utf8")); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export function parseCredentials(value: string | undefined): UploadCredentials | undefined { + if (!value?.trim()) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + // JSON.parse's message quotes the input, which here is the secret. + throw new Error("MOBILE_SOURCEMAPS_R2 is not valid JSON."); + } + if (!isRecord(parsed)) throw new Error("MOBILE_SOURCEMAPS_R2 is not a JSON object."); + const { endpoint, bucketName, accessKeyId, secretAccessKey } = parsed; + if ( + typeof endpoint !== "string" || + typeof bucketName !== "string" || + typeof accessKeyId !== "string" || + typeof secretAccessKey !== "string" + ) { + throw new Error( + "MOBILE_SOURCEMAPS_R2 needs endpoint, bucketName, accessKeyId and secretAccessKey.", + ); + } + return { endpoint, bucketName, accessKeyId, secretAccessKey }; +} + +/** One map per update `eas update --emit-metadata` published, next to the bundle `metadata.json` names. */ +export function otaSourceMapUploads(distDir: string): ReadonlyArray { + const metadata = readJson(NodePath.join(distDir, "metadata.json")); + const easMetadata = readJson(NodePath.join(distDir, "eas-update-metadata.json")); + const fileMetadata = isRecord(metadata) ? metadata.fileMetadata : undefined; + const updates = isRecord(easMetadata) ? easMetadata.updates : undefined; + if (!isRecord(fileMetadata) || !Array.isArray(updates)) { + throw new Error(`${distDir} is missing Expo export or EAS update metadata.`); + } + return updates.flatMap((update): Array => { + if (!isRecord(update) || !isPlatform(update.platform) || typeof update.id !== "string") { + return []; + } + const platformFiles = fileMetadata[update.platform]; + const bundle = isRecord(platformFiles) ? platformFiles.bundle : undefined; + if (typeof bundle !== "string") return []; + return [ + { + key: mobileSourceMapKey(update.platform, "ota", update.id), + path: NodePath.join(distDir, `${bundle}.map`), + }, + ]; + }); +} + +/** Android's Gradle plugin writes the map under the release variant; iOS to `SOURCEMAP_FILE` (eas.json). */ +export function buildSourceMapUpload( + env: Readonly>, + projectDir: string, +): SourceMapUpload { + const platform = env.EAS_BUILD_PLATFORM; + const buildId = env.EAS_BUILD_ID; + if (!isPlatform(platform) || !buildId) { + throw new Error("EAS_BUILD_PLATFORM and EAS_BUILD_ID are only set on EAS Build."); + } + if (platform === "ios") { + if (!env.SOURCEMAP_FILE) throw new Error("SOURCEMAP_FILE is not set for this iOS build."); + return { key: mobileSourceMapKey("ios", "build", buildId), path: env.SOURCEMAP_FILE }; + } + const variantsDir = NodePath.join(projectDir, "android/app/build/generated/sourcemaps/react"); + const variant = NodeFS.existsSync(variantsDir) + ? NodeFS.readdirSync(variantsDir).find((name) => /release$/iu.test(name)) + : undefined; + if (!variant) throw new Error(`No release source map under ${variantsDir}.`); + return { + key: mobileSourceMapKey("android", "build", buildId), + path: NodePath.join(variantsDir, variant, "index.android.bundle.map"), + }; +} + +/** + * EAS writes the app's debug symbols next to the `.ipa`, so native frames in iOS crash + * reports can be read with `atos`. Undefined when the build left none. + */ +export function buildDsymUpload( + env: Readonly>, + projectDir: string, +): SourceMapUpload | undefined { + if (env.EAS_BUILD_PLATFORM !== "ios" || !env.EAS_BUILD_ID) return undefined; + const buildDir = NodePath.join(projectDir, "ios/build"); + const dsym = NodeFS.existsSync(buildDir) + ? NodeFS.readdirSync(buildDir).find((name) => name.endsWith(".app.dSYM.zip")) + : undefined; + if (!dsym) return undefined; + return { key: `ios/dsym/${env.EAS_BUILD_ID}.zip`, path: NodePath.join(buildDir, dsym) }; +} + +/** Only what the relay reads: source text stays out of the bucket, and the rest is dead weight. */ +function stripToMappings(text: string): string { + const map: unknown = JSON.parse(text); + if (!isRecord(map)) throw new Error("Source map is not a JSON object."); + const { version, sources, sourceRoot, mappings } = map; + return JSON.stringify({ version, sources, sourceRoot, mappings }); +} + +export function uploadSourceMap( + credentials: UploadCredentials, + upload: SourceMapUpload, +): Promise { + return putObject( + credentials, + upload.key, + stripToMappings(NodeFS.readFileSync(upload.path, "utf8")), + "application/json", + ); +} + +export function uploadDsym(credentials: UploadCredentials, upload: SourceMapUpload) { + const body = new Uint8Array(NodeFS.readFileSync(upload.path)); + return putObject(credentials, upload.key, body, "application/zip"); +} + +async function putObject( + credentials: UploadCredentials, + key: string, + body: string | Uint8Array, + contentType: string, +): Promise { + const client = new AwsClient({ + accessKeyId: credentials.accessKeyId, + secretAccessKey: credentials.secretAccessKey, + service: "s3", + region: "auto", + }); + const url = `${credentials.endpoint.replace(/\/$/u, "")}/${credentials.bucketName}/${key}`; + const response = await client.fetch(url, { + method: "PUT", + body, + headers: { "content-type": contentType }, + }); + if (!response.ok) { + throw new Error(`R2 answered ${response.status}: ${(await response.text()).slice(0, 500)}`); + } + return body.length; +} diff --git a/apps/mobile/scripts/sourcemap-upload.test.ts b/apps/mobile/scripts/sourcemap-upload.test.ts new file mode 100644 index 0000000000000..ee8ae57d690ae --- /dev/null +++ b/apps/mobile/scripts/sourcemap-upload.test.ts @@ -0,0 +1,136 @@ +import * as NodeFS from "node:fs"; +import * as NodeHttp from "node:http"; +import type * as NodeNet from "node:net"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { + buildDsymUpload, + buildSourceMapUpload, + otaSourceMapUploads, + uploadSourceMap, +} from "./sourcemap-upload.mts"; + +const directories: Array = []; +const tempDir = () => { + const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "sourcemap-upload-")); + directories.push(dir); + return dir; +}; +const writeFile = (path: string, contents: string) => { + NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true }); + NodeFS.writeFileSync(path, contents); +}; + +afterEach(() => { + for (const dir of directories.splice(0)) NodeFS.rmSync(dir, { recursive: true, force: true }); +}); + +describe("uploadSourceMap", () => { + it("PUTs each published update's map, only what the relay reads, under its update id", async () => { + const dist = tempDir(); + writeFile( + NodePath.join(dist, "metadata.json"), + JSON.stringify({ + fileMetadata: { + android: { bundle: "_expo/static/js/android/index-abc.hbc" }, + ios: { bundle: "_expo/static/js/ios/index-def.hbc" }, + }, + }), + ); + writeFile( + NodePath.join(dist, "eas-update-metadata.json"), + JSON.stringify({ + updates: [ + { id: "update-android", platform: "android" }, + { id: "update-ios", platform: "ios" }, + { id: "update-web", platform: "web" }, + ], + }), + ); + for (const platform of ["android", "ios"]) { + writeFile( + NodePath.join( + dist, + `_expo/static/js/${platform}/index-${platform === "ios" ? "def" : "abc"}.hbc.map`, + ), + JSON.stringify({ + version: 3, + sources: [`${platform}.ts`], + sourcesContent: ["secret"], + names: ["render"], + x_facebook_sources: [[null]], + mappings: "", + }), + ); + } + + const requests: Array<{ method?: string; url?: string; auth?: string; body: string }> = []; + const server = NodeHttp.createServer((request, response) => { + let body = ""; + request.on("data", (chunk) => (body += chunk)); + request.on("end", () => { + requests.push({ + method: request.method, + url: request.url, + auth: request.headers.authorization, + body, + }); + response.end(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address() as NodeNet.AddressInfo; + const credentials = { + endpoint: `http://127.0.0.1:${address.port}`, + bucketName: "maps", + accessKeyId: "key-id", + secretAccessKey: "secret-key", + }; + for (const upload of otaSourceMapUploads(dist)) { + await uploadSourceMap(credentials, upload); + } + expect(requests).toEqual( + ["android", "ios"].map((platform) => ({ + method: "PUT", + url: `/maps/${platform}/ota/update-${platform}.map`, + auth: expect.stringMatching(/^AWS4-HMAC-SHA256 Credential=key-id\/\d{8}\/auto\/s3\//), + body: JSON.stringify({ version: 3, sources: [`${platform}.ts`], mappings: "" }), + })), + ); + } finally { + server.close(); + } + }); +}); + +describe("buildSourceMapUpload", () => { + it("names the EAS build's embedded bundle map under its build id", () => { + const project = tempDir(); + const androidMap = NodePath.join( + project, + "android/app/build/generated/sourcemaps/react/release/index.android.bundle.map", + ); + writeFile(androidMap, "{}"); + expect( + buildSourceMapUpload({ EAS_BUILD_PLATFORM: "android", EAS_BUILD_ID: "build-1" }, project), + ).toEqual({ key: "android/build/build-1.map", path: androidMap }); + expect( + buildSourceMapUpload( + { EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-2", SOURCEMAP_FILE: "/tmp/main.map" }, + project, + ), + ).toEqual({ key: "ios/build/build-2.map", path: "/tmp/main.map" }); + }); + + it("names an iOS build's dSYMs under its build id", () => { + const project = tempDir(); + const dsym = NodePath.join(project, "ios/build/T3Code.app.dSYM.zip"); + writeFile(dsym, "zip"); + expect( + buildDsymUpload({ EAS_BUILD_PLATFORM: "ios", EAS_BUILD_ID: "build-3" }, project), + ).toEqual({ key: "ios/dsym/build-3.zip", path: dsym }); + }); +}); diff --git a/apps/mobile/scripts/upload-sourcemaps.mts b/apps/mobile/scripts/upload-sourcemaps.mts new file mode 100644 index 0000000000000..1656035906e44 --- /dev/null +++ b/apps/mobile/scripts/upload-sourcemaps.mts @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Usage: `ota ` after `eas update --emit-metadata`, or `build` from eas-build-on-success. +// Problems are warnings, never failures: an unreadable stack is not worth failing a release over. +import { + buildDsymUpload, + buildSourceMapUpload, + otaSourceMapUploads, + parseCredentials, + uploadDsym, + uploadSourceMap, + type SourceMapUpload, +} from "./sourcemap-upload.mts"; + +const warn = (message: string) => { + // GitHub Actions shows this as an annotation; elsewhere it is a log line. + console.warn(process.env.GITHUB_ACTIONS ? `::warning::${message}` : `warning: ${message}`); +}; + +const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +async function main() { + const [mode, distDir] = process.argv.slice(2); + const credentials = parseCredentials(process.env.MOBILE_SOURCEMAPS_R2); + if (!credentials) { + warn("MOBILE_SOURCEMAPS_R2 is not set; crash stacks from this bundle stay minified."); + return; + } + let uploads: ReadonlyArray; + if (mode === "ota" && distDir) { + uploads = otaSourceMapUploads(distDir); + } else if (mode === "build") { + uploads = [buildSourceMapUpload(process.env, process.cwd())]; + } else { + throw new Error("Usage: upload-sourcemaps.mts ota | build"); + } + if (uploads.length === 0) warn("Found no source maps to upload."); + for (const upload of uploads) { + try { + const bytes = await uploadSourceMap(credentials, upload); + console.log(`Uploaded ${upload.key} (${(bytes / 1e6).toFixed(1)} MB)`); + } catch (error) { + warn(`Could not upload ${upload.key}: ${errorMessage(error)}`); + } + } + if (mode !== "build" || process.env.EAS_BUILD_PLATFORM !== "ios") return; + const dsym = buildDsymUpload(process.env, process.cwd()); + if (!dsym) { + warn("Found no dSYM under ios/build; native iOS frames from this build stay unreadable."); + return; + } + try { + const bytes = await uploadDsym(credentials, dsym); + console.log(`Uploaded ${dsym.key} (${(bytes / 1e6).toFixed(1)} MB)`); + } catch (error) { + warn(`Could not upload ${dsym.key}: ${errorMessage(error)}`); + } +} + +await main().catch((error: unknown) => { + warn(`Source map upload skipped: ${errorMessage(error)}`); +}); diff --git a/apps/mobile/src/components/RenderErrorBoundary.tsx b/apps/mobile/src/components/RenderErrorBoundary.tsx index 16f9439a2cd38..f4a6035b09704 100644 --- a/apps/mobile/src/components/RenderErrorBoundary.tsx +++ b/apps/mobile/src/components/RenderErrorBoundary.tsx @@ -1,3 +1,4 @@ +import { Observe } from "expo-observe"; import { Component, type ReactNode } from "react"; import { ScrollView, View } from "react-native"; @@ -60,7 +61,9 @@ export class RenderErrorBoundary extends Component< return { failed: true, error }; } - override componentDidCatch(_error: unknown, info: { componentStack?: string }) { + override componentDidCatch(error: unknown, info: { componentStack?: string }) { + // A caught render error never reaches Observe's global handler. + Observe.reportError(error); this.setState({ componentStack: info.componentStack }); } diff --git a/apps/mobile/src/features/connection/LinkedEnvironmentsSection.tsx b/apps/mobile/src/features/connection/LinkedEnvironmentsSection.tsx index 66dd7dfd8f485..9153c9fa6a1c8 100644 --- a/apps/mobile/src/features/connection/LinkedEnvironmentsSection.tsx +++ b/apps/mobile/src/features/connection/LinkedEnvironmentsSection.tsx @@ -1,3 +1,4 @@ +import { offersLinkedEnvironments } from "@t3tools/client-runtime/link-request"; import { peerLinkHealth } from "@t3tools/client-runtime/state/peerLinks"; import { useFocusEffect } from "@react-navigation/native"; import { AuthAccessReadScope, type EnvironmentId } from "@t3tools/contracts"; @@ -18,7 +19,7 @@ export function LinkedEnvironmentsSection(props: { }) { const config = useEnvironmentServerConfig(props.environmentId); const canRead = useEnvironmentScope(props.environmentId, AuthAccessReadScope); - const supported = canRead && config?.environment.capabilities.peerLinks === true; + const supported = canRead && offersLinkedEnvironments(config); const links = useEnvironmentQuery( supported ? peerLinkEnvironment.list({ environmentId: props.environmentId, input: {} }) : null, ); diff --git a/apps/mobile/src/features/conversation-summaries/ConversationSummariesRouteScreen.test.tsx b/apps/mobile/src/features/conversation-summaries/ConversationSummariesRouteScreen.test.tsx index c6f8d2513f6ed..179bf3d21647b 100644 --- a/apps/mobile/src/features/conversation-summaries/ConversationSummariesRouteScreen.test.tsx +++ b/apps/mobile/src/features/conversation-summaries/ConversationSummariesRouteScreen.test.tsx @@ -1,4 +1,4 @@ -import { act, type ReactNode } from "react"; +import { act, useSyncExternalStore, type ReactNode } from "react"; import { createRoot } from "react-dom/client"; import { expect, it, vi } from "vite-plus/test"; import { @@ -15,6 +15,7 @@ import { } from "@t3tools/contracts"; const state = vi.hoisted(() => ({ + listeners: new Set<() => void>(), supported: true, focused: true, list: null as ConversationSummaryList | null, @@ -29,7 +30,16 @@ vi.mock("@react-navigation/native", () => ({ useIsFocused: () => state.focused, useNavigation: () => ({ goBack: () => {} }), })); -vi.mock("@effect/atom-react", () => ({ useAtomValue: (value: unknown) => value })); +vi.mock("@effect/atom-react", () => ({ + useAtomValue: () => + useSyncExternalStore( + (listener) => { + state.listeners.add(listener); + return () => state.listeners.delete(listener); + }, + () => state.supported, + ), +})); vi.mock("../../components/ui/AppBar", () => ({ AppBar: () =>
Conversation summaries
, })); @@ -91,6 +101,7 @@ vi.mock("../../state/query", () => ({ import { ConversationSummariesRouteScreen } from "./ConversationSummariesRouteScreen"; function setup() { + state.listeners.clear(); state.supported = true; state.focused = false; state.lists = []; @@ -99,15 +110,16 @@ function setup() { const container = document.createElement("div"); const root = createRoot(container); const rerender = () => - act(() => + act(() => { + for (const listener of state.listeners) listener(); root.render( , - ), - ); + ); + }); rerender(); return { container, diff --git a/apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx b/apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx index 418727293463a..8c0cbff3352ab 100644 --- a/apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx +++ b/apps/mobile/src/features/diagnostics/SettingsDiagnosticsRouteScreen.tsx @@ -3,8 +3,9 @@ import { ScreenScrollView as ScrollView } from "../../components/ui/ScreenScroll import * as Application from "expo-application"; import Constants from "expo-constants"; import * as Updates from "expo-updates"; -import { useAtomValue } from "@effect/atom-react"; +import { useAtomSet, useAtomValue } from "@effect/atom-react"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import { AsyncResult, Atom } from "effect/reactivity"; import { useState } from "react"; import { View } from "react-native"; @@ -13,9 +14,12 @@ import { useSafeAreaInsets } from "react-native-safe-area-context"; import { SymbolView } from "../../components/ui/AppSymbol"; import { AppText as Text } from "../../components/ui/AppText"; import { tryCopyTextWithHaptic } from "../../lib/copyTextWithHaptic"; +import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { crashReportingAvailable, setCrashReportsEnabled } from "../observability/observe"; import { SettingsActionRow } from "../settings/components/SettingsActionRow"; import { SettingsScreen } from "../settings/components/SettingsScreen"; import { SettingsSection } from "../settings/components/SettingsSection"; +import { SettingsSwitchRow } from "../settings/components/SettingsSwitchRow"; import { formatStartupCrashReport, parseStartupCrashRecords, @@ -86,6 +90,8 @@ export function SettingsDiagnosticsRouteScreen() { className="flex-1" contentContainerClassName="gap-6 px-5 pt-4 pb-[18px]" > + {crashReportingAvailable ? : null} + {state.status === "loading" ? ( @@ -131,6 +137,54 @@ export function SettingsDiagnosticsRouteScreen() { ); } +function CrashReportsSetting() { + const preferences = useAtomValue(mobilePreferencesAtom); + const savePreferences = useAtomSet(updateMobilePreferencesAtom, { mode: "promiseExit" }); + // One change at a time, so a slow save never turns sending back on after a later opt-out. + const [saving, setSaving] = useState(false); + const ready = AsyncResult.isSuccess(preferences) && !preferences.waiting; + const enabled = AsyncResult.isSuccess(preferences) + ? preferences.value.crashReportsEnabled !== false + : true; + + const change = (value: boolean) => { + setSaving(true); + // Sending stops before the save and starts only after it. + if (!value) void setCrashReportsEnabled(false); + savePreferences({ crashReportsEnabled: value }) + .then((exit) => { + if (Exit.isFailure(exit)) { + console.warn("[diagnostics] could not save the crash report setting", exit.cause); + return value ? undefined : setCrashReportsEnabled(true); + } + return value ? setCrashReportsEnabled(true) : undefined; + }) + .catch((error: unknown) => { + console.warn("[diagnostics] could not apply the crash report setting", error); + }) + .finally(() => setSaving(false)); + }; + + return ( + + + + + + When the app crashes or hits an error, it sends the error message, the stack trace, the app + version, and the device model and OS version to the T3 Code team. Each launch also sends + startup timings. Reports carry a random ID for this install. + + + ); +} + function EmptyState(props: { readonly icon: "exclamationmark.triangle" | "checkmark.circle"; readonly title: string; diff --git a/apps/mobile/src/features/loom/ProjectLoomRouteScreen.test.tsx b/apps/mobile/src/features/loom/ProjectLoomRouteScreen.test.tsx index fd86420e18129..04220ecd50adc 100644 --- a/apps/mobile/src/features/loom/ProjectLoomRouteScreen.test.tsx +++ b/apps/mobile/src/features/loom/ProjectLoomRouteScreen.test.tsx @@ -1,4 +1,4 @@ -import { act, type ReactNode } from "react"; +import { act, useSyncExternalStore, type ReactNode } from "react"; import { createRoot } from "react-dom/client"; import { expect, it, vi } from "vite-plus/test"; import { @@ -14,6 +14,7 @@ import { } from "@t3tools/contracts"; const state = vi.hoisted(() => ({ + listeners: new Set<() => void>(), window: null as LoomWindow | null, supported: true, queries: [] as LoomWindowInput[], @@ -119,7 +120,21 @@ vi.mock("../workspace/ListFilterMenu", () => ({ ), })); -vi.mock("@effect/atom-react", () => ({ useAtomValue: (value: unknown) => value })); +vi.mock("@effect/atom-react", () => ({ + useAtomValue: () => ({ + environment: { + capabilities: { + projectLoom: useSyncExternalStore( + (listener) => { + state.listeners.add(listener); + return () => state.listeners.delete(listener); + }, + () => state.supported, + ), + }, + }, + }), +})); vi.mock("../../state/server", () => ({ serverEnvironment: { configValueAtom: () => ({ environment: { capabilities: { projectLoom: state.supported } } }), @@ -176,6 +191,7 @@ const run = (id: string, thread: string, day: number, hour: number): LoomRun => completedAt: at(day, hour + 1), }); function setup() { + state.listeners.clear(); state.supported = true; state.actions.length = 0; state.queries.length = 0; @@ -235,13 +251,14 @@ function setup() { const container = document.createElement("div"); const root = createRoot(container); const render = () => - act(() => + act(() => { + for (const listener of state.listeners) listener(); root.render( , - ), - ); + ); + }); render(); const press = async (label: string) => { const button = [...container.querySelectorAll("button")].find( diff --git a/apps/mobile/src/features/observability/observe.ts b/apps/mobile/src/features/observability/observe.ts new file mode 100644 index 0000000000000..fbbb5e22b8339 --- /dev/null +++ b/apps/mobile/src/features/observability/observe.ts @@ -0,0 +1,63 @@ +import Constants from "expo-constants"; +import { Observe } from "expo-observe"; + +// Importing expo-observe installs its global JS error handler; reports leave only through the relay. +const relayUrl: unknown = Constants.expoConfig?.extra?.relay?.url; +const appVariant: unknown = Constants.expoConfig?.extra?.appVariant; + +export const crashReportingAvailable = typeof relayUrl === "string" && relayUrl.trim() !== ""; + +interface PromiseRejectionTracker { + readonly enablePromiseRejectionTracker?: (options: { + readonly allRejections: boolean; + readonly onUnhandled: (id: number, rejection: unknown) => void; + }) => void; +} + +// React Native tracks unhandled rejections only in development, for LogBox; Observe has no hook. +if (!__DEV__) { + const hermes: PromiseRejectionTracker | undefined = Reflect.get(globalThis, "HermesInternal"); + hermes?.enablePromiseRejectionTracker?.({ + allRejections: true, + onUnhandled: (_id, rejection) => Observe.reportError(rejection), + }); +} + +let latestConfiguration = 0; + +// While sending is off, a dispatch pass drops what it finds, so one runs before turning it back on. +// Best effort: Observe skips the pass while it is backing off from a failed send. +export async function setCrashReportsEnabled(enabled: boolean) { + const configuration = ++latestConfiguration; + const dispatchingEnabled = crashReportingAvailable && enabled; + if (dispatchingEnabled) { + await Observe.dispatchEvents(); + if (configuration !== latestConfiguration) return; + } + Observe.configure({ + dispatchingEnabled, + ...(typeof appVariant === "string" ? { environment: appVariant } : {}), + }); +} + +if (!crashReportingAvailable) { + void setCrashReportsEnabled(false); +} else { + // Observe keeps its last configuration across launches; configuring it off while the setting + // loads would drop the last session's native crash. Imported lazily so the persistence graph + // never runs ahead of the imports `index.ts` orders first. + const changedSinceLoad = () => latestConfiguration !== 0; + void import("../../persistence/imperative") + .then(({ loadPreferences }) => loadPreferences()) + .then( + (preferences) => + changedSinceLoad() + ? undefined + : setCrashReportsEnabled(preferences.crashReportsEnabled !== false), + // Observe keeps the last choice it was given, which stands in for the unreadable one. + (error: unknown) => console.warn("[observe] could not read the crash report setting", error), + ) + .catch((error: unknown) => { + console.warn("[observe] could not apply the crash report setting", error); + }); +} diff --git a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx index b3094f648b5c7..5463542d24cff 100644 --- a/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx +++ b/apps/mobile/src/features/pull-requests/PullRequestRouteScreen.tsx @@ -341,6 +341,8 @@ export function PullRequestRouteScreen({ route }: StaticScreenProps({ tab: "overview", - offset: -insets.top, + offset: -barInsets.top, }); const bodyY = useRef(0); const [headerHeight, setHeaderHeight] = useState(0); @@ -386,14 +388,15 @@ export function PullRequestRouteScreen({ route }: StaticScreenProps { scrollViews.current.overview?.scrollToOffset({ - offset: bodyY.current + filesY.current - insets.top, + offset: bodyY.current + filesY.current - barInsets.top, animated: true, }); pendingFilesJump.current = false; @@ -675,7 +678,9 @@ export function PullRequestRouteScreen({ route }: StaticScreenProps ({ ), })); vi.mock("../../state/server", () => ({ serverEnvironment: { updateSettings: {} } })); +vi.mock("../../state/environments", () => ({ useEnvironments: () => ({ environments: [] }) })); vi.mock("../../state/use-atom-command", () => ({ useAtomCommand: () => mocks.updateSettings, })); diff --git a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx index f982820a71610..5149ffa54c6fc 100644 --- a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx +++ b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx @@ -11,7 +11,10 @@ import { useSafeAreaInsets } from "react-native-safe-area-context"; import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts"; import { supportsSharedSettingsSync } from "@t3tools/client-runtime/state/shared-settings"; import { AppText as Text } from "../../components/ui/AppText"; +import { useEnvironments } from "../../state/environments"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; import { SettingsSection } from "./components/SettingsSection"; import { SettingsSwitchRow } from "./components/SettingsSwitchRow"; import { SettingsScreen } from "./components/SettingsScreen"; @@ -31,6 +34,7 @@ export function SettingsLabsRouteScreen() { contentContainerClassName="gap-6 px-5 pt-4" contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }} > + @@ -214,6 +218,47 @@ export function NewThreadSettingsSection() { ); } +/** Per machine, since each machine's own setting decides whether it links and answers links. */ +function BetaSettingsSection() { + const { environments } = useEnvironments(); + const supported = environments.filter( + (environment) => + environment.connection.phase === "connected" && + environment.serverConfig?.environment.capabilities.peerLinks === true, + ); + const writable = useEnvironmentsWithScope(supported, AuthSettingsWriteScope); + const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { + label: "linked environments beta update", + reportFailure: true, + }); + if (supported.length === 0) return null; + return ( + + + {supported.map((environment) => ( + + void updateSettings({ + environmentId: environment.environmentId, + input: { patch: { linkedEnvironmentsEnabled: value } }, + }) + } + /> + ))} + + + Let agents on a machine work on your other machines, and theirs on it. Turn it on for each + machine. While it is off, nothing new starts over a link; tasks already running finish. + + + ); +} + function LegacySettingsSection() { const savePreferences = useAtomSet(updateMobilePreferencesAtom); const preferences = useAtomValue(mobilePreferencesAtom); diff --git a/apps/mobile/src/features/settings/settings-search.ts b/apps/mobile/src/features/settings/settings-search.ts index a177011e29c84..55f2afa74a2af 100644 --- a/apps/mobile/src/features/settings/settings-search.ts +++ b/apps/mobile/src/features/settings/settings-search.ts @@ -87,7 +87,7 @@ const SETTINGS_SEARCH_TERMS = { "submodules", "start from remote", ], - SettingsLabs: ["legacy", "plan mode", "beta"], + SettingsLabs: ["legacy", "plan mode", "beta", "linked environments", "link machine"], SettingsEnvironmentMaintenance: ["updates", "provider updates", "restart", "manage environments"], SettingsAbout: [ "version", diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 61bd16ea0831b..2449454b9ce9d 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -104,6 +104,8 @@ import { ComposerCommandPopover } from "./ComposerCommandPopover"; import { ComposerPopoverAnchor } from "./ComposerPopoverAnchor"; import { ComposerQueuedEditAttachments } from "./ComposerQueuedEdit"; import { useComposerCommandMenu } from "./use-composer-command-menu"; +import type { ComposerEnvironmentCandidate } from "@t3tools/client-runtime/composerThreadItems"; +import { offersLinkedEnvironments } from "@t3tools/client-runtime/link-request"; import { ComposerDictationCancelAction, ComposerDictationDraftContent, @@ -122,6 +124,7 @@ import { export const COMPOSER_COLLAPSED_CHROME = 60; +const NO_MACHINES: ReadonlyArray = []; const COLLAPSED_EDITOR_DEFAULT_INSET = 6; // iOS puts line-height slack above glyphs. DM Sans's lowercase midline is // descender (0.31em) + half x-height (0.504em) above the line bottom. @@ -544,12 +547,15 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps props.selectedThread.projectId, "composer", ); + const mentionableEnvironments = useMentionableEnvironments(); const composerMenu = useComposerCommandMenu({ draftMessage: draftMessage, ownerKey: composerDraftKey, environmentId: props.environmentId, threadShells: useThreadShells(), - environments: useMentionableEnvironments(), + environments: offersLinkedEnvironments(props.serverConfig) + ? mentionableEnvironments + : NO_MACHINES, currentThreadId: props.selectedThread.id, projectCwd: props.projectCwd, pullRequestProjectId: props.serverConfig?.environment.capabilities.pullRequests diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 2a80b7d39d28a..27c99334d5788 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -648,7 +648,7 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) { accessibilityLabel: `Open ${childInput.hasPendingApprovals ? "approval" : "question"} from ${childInput.title}`, onPress: () => { Keyboard.dismiss(); - openLinkedThread({ environmentId: props.environmentId, threadId: childInput.id }); + openLinkedThread({ environmentId: childInput.environmentId, threadId: childInput.id }); }, }; } diff --git a/apps/mobile/src/lib/composerChipPresentation.test.ts b/apps/mobile/src/lib/composerChipPresentation.test.ts index 9e1ce087bd47c..5264817d0847e 100644 --- a/apps/mobile/src/lib/composerChipPresentation.test.ts +++ b/apps/mobile/src/lib/composerChipPresentation.test.ts @@ -112,9 +112,6 @@ describe("contextChipAccent", () => { describe("contextChipPresentation", () => { it("draws a mentioned machine as a machine, not the generic file chip", () => { - expect(contextChipPresentation("environment")).toEqual({ - accent: "#3f9a4f", - icon: "server", - }); + expect(contextChipPresentation("environment")).toEqual({ accent: "#3f9a4f", icon: "server" }); }); }); diff --git a/apps/mobile/src/lib/filePreview.test.ts b/apps/mobile/src/lib/filePreview.test.ts index a1f06df49837f..b3a0a69a3868e 100644 --- a/apps/mobile/src/lib/filePreview.test.ts +++ b/apps/mobile/src/lib/filePreview.test.ts @@ -273,6 +273,7 @@ vi.mock("../features/threads/thread-context-divider", () => ({})); vi.mock("../features/threads/thread-handoff-row", () => ({})); vi.mock("../features/threads/ThreadGuestShareBar", () => ({ useThreadGuestShare: () => null })); vi.mock("../features/threads/SecretRequestCard", () => ({})); +vi.mock("../features/threads/LinkRequestCard", () => ({ LinkRequestCard: () => null })); vi.mock("../features/threads/worktree-setup-card", () => ({})); vi.mock("../features/threads/CopyTextButton", () => ({ CopyTextButton: () => null })); vi.mock("../features/threads/thread-work-log", () => ({ THREAD_DISCLOSURE_TRANSITION_MS: 100 })); diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index 3a79a73ee3c36..b38e65c0f2520 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -103,6 +103,8 @@ export interface Preferences { readonly codeFontSize?: number | null; readonly codeWordBreak?: boolean; readonly mathRenderingEnabled?: boolean; + /** Unset means on: crash reports are sent unless the user turns them off. */ + readonly crashReportsEnabled?: boolean; readonly connectOnboardingOptOutAccounts?: ReadonlyArray; readonly collapsedProjectGroups?: readonly string[]; readonly dismissedDatabaseStorageNotices?: readonly string[]; @@ -183,6 +185,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { codeFontSize?: number | null; codeWordBreak?: boolean; mathRenderingEnabled?: boolean; + crashReportsEnabled?: boolean; connectOnboardingOptOutAccounts?: ReadonlyArray; collapsedProjectGroups?: readonly string[]; dismissedDatabaseStorageNotices?: readonly string[]; @@ -272,6 +275,9 @@ function sanitizePreferences(parsed: Preferences): Preferences { if (typeof parsed.codeWordBreak === "boolean") preferences.codeWordBreak = parsed.codeWordBreak; if (typeof parsed.mathRenderingEnabled === "boolean") preferences.mathRenderingEnabled = parsed.mathRenderingEnabled; + if (typeof parsed.crashReportsEnabled === "boolean") { + preferences.crashReportsEnabled = parsed.crashReportsEnabled; + } if (Array.isArray(parsed.connectOnboardingOptOutAccounts)) { preferences.connectOnboardingOptOutAccounts = parsed.connectOnboardingOptOutAccounts.filter( (account): account is string => typeof account === "string", diff --git a/apps/mobile/src/state/peerLinks.ts b/apps/mobile/src/state/peerLinks.ts index 9ab0d664c68bb..c0b36dfadec02 100644 --- a/apps/mobile/src/state/peerLinks.ts +++ b/apps/mobile/src/state/peerLinks.ts @@ -28,6 +28,7 @@ export function useLinkableMachines(environmentId: EnvironmentId, enabled: boole const discoverIfUnasked = useEffectEvent(() => { if (discovery.environments.size === 0 && !discovery.refreshing) void refreshDiscovery(); }); + // oxlint-disable-next-line t3code/react-effect-budget -- discovery follows the picker's visibility, which a card's mount or a dialog's open sets useEffect(() => { if (enabled) discoverIfUnasked(); }, [enabled]); @@ -39,7 +40,7 @@ export function useLinkableMachines(environmentId: EnvironmentId, enabled: boole linkableMachines({ environments, threadEnvironmentId: environmentId, - linkedIds: new Set(links.data?.links.map((link) => link.environmentId)), + links: links.data?.links ?? [], connect: discovery.environments, }), [discovery.environments, environmentId, environments, links.data], diff --git a/apps/server/src/auth/McpOAuth.test.ts b/apps/server/src/auth/McpOAuth.test.ts index b2eda56357d87..89bdb90f9ed40 100644 --- a/apps/server/src/auth/McpOAuth.test.ts +++ b/apps/server/src/auth/McpOAuth.test.ts @@ -21,6 +21,7 @@ import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as Sqlite from "../persistence/Sqlite.ts"; import * as EnvironmentAuth from "./EnvironmentAuth.ts"; import * as McpOAuth from "./McpOAuth.ts"; @@ -46,7 +47,12 @@ const makeLayerRoutes = (capture: (auth: EnvironmentAuth.EnvironmentAuth["Servic ), HttpApiBuilder.layer(AuthTestApi).pipe( Layer.provide(AuthHttp.layer), - Layer.provide(McpOAuthHttp.layer.pipe(Layer.provide(McpOAuth.layer))), + Layer.provide( + McpOAuthHttp.layer.pipe( + Layer.provide(McpOAuth.layer), + Layer.provide(ServerSettings.layerTest()), + ), + ), Layer.provide(AuthHttp.layerAuthenticatedAuth), ), ).pipe( diff --git a/apps/server/src/auth/McpOAuth.ts b/apps/server/src/auth/McpOAuth.ts index dcca4bcf35de1..0db7c0ded396b 100644 --- a/apps/server/src/auth/McpOAuth.ts +++ b/apps/server/src/auth/McpOAuth.ts @@ -29,6 +29,7 @@ import * as Schema from "effect/Schema"; import { HttpServerRequest } from "effect/http"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import * as EnvironmentAuth from "./EnvironmentAuth.ts"; @@ -244,6 +245,11 @@ const make = Effect.gen(function* () { const environmentAuth = yield* EnvironmentAuth.EnvironmentAuth; const secretStore = yield* ServerSecretStore.ServerSecretStore; const crypto = yield* Crypto.Crypto; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const linksEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.linkedEnvironmentsEnabled), + Effect.orElseSucceed(() => false), + ); const signingKey = yield* secretStore .getOrCreateRandom(SIGNING_SECRET_NAME, 32) .pipe(Effect.orDie); @@ -320,11 +326,15 @@ const make = Effect.gen(function* () { "Redirect URIs must be https, or http on localhost, 127.0.0.1 or [::1].", }); } - // Requested grant types, scopes and token endpoint auth methods are ignored rather - // than rejected: RFC 7591 lets the server answer with what it supports. Claude Code - // asks for refresh_token, and hosted agents may ask for a client secret; every client - // is registered as public and proves itself with PKCE instead. + // RFC 7591 allows ignoring unsupported registration options; all clients use public PKCE. const peerLink = metadata.software_id === MCP_PEER_LINK_SOFTWARE_ID; + if (peerLink && !(yield* linksEnabled)) { + return yield* new AuthMcpRegistrationError({ + error: "invalid_client_metadata", + error_description: + "Linked environments is off on that machine. Turn it on in its Settings → Labs.", + }); + } return { clientId: signClientId(name, redirectUris, peerLink), name, diff --git a/apps/server/src/cli/environment.ts b/apps/server/src/cli/environment.ts index d831bc44e828e..c8d6a2e0112ee 100644 --- a/apps/server/src/cli/environment.ts +++ b/apps/server/src/cli/environment.ts @@ -15,6 +15,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as PeerLinks from "../peer/PeerLinks.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import { authLocationFlags, type CliAuthLocationFlags, resolveCliAuthConfig } from "./config.ts"; @@ -41,6 +42,11 @@ const runWithPeerLinks = Effect.fn("environment.runWithPeerLinks")(function* issuedAt: 1, }; const layerServer = McpHttpServer.layerMcpTransport.pipe( + Layer.provide(ServerSettings.layerTest()), Layer.provide( Layer.mock(McpSessionRegistry.McpSessionRegistry)({ resolve: (presented) => @@ -782,7 +786,9 @@ it.effect("admits provider and OAuth client credentials and points only clients ), }), ).pipe( - Layer.provideMerge(McpHttpServer.layerMcpTransport), + Layer.provideMerge( + McpHttpServer.layerMcpTransport.pipe(Layer.provide(ServerSettings.layerTest())), + ), Layer.provide( Layer.mock(McpSessionRegistry.McpSessionRegistry)({ resolve: (token) => @@ -900,6 +906,7 @@ it.effect( }), ).pipe( Layer.provideMerge(McpHttpServer.layerMcpTransport), + Layer.provide(ServerSettings.layerTest({ linkedEnvironmentsEnabled: true })), Layer.provide(McpToolAccessTestkit.liveThreadsLayer), Layer.provide( Layer.mock(McpSessionRegistry.McpSessionRegistry)({ @@ -1000,6 +1007,130 @@ it.effect( ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("hides tools for links and refuses another environment while the beta is off", () => + Effect.scoped( + Effect.gen(function* () { + const token = "linksToken"; + const scope: McpInvocationContext.McpThreadInvocationScope = { + environmentId, + requestNamespace: token, + thread: { + threadId: ThreadId.make("thread-links"), + providerSessionId: token, + providerInstanceId: ProviderInstanceId.make("codex"), + }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + const LinksToolkit = Toolkit.make( + Tool.make("links", { + description: "Lists links.", + success: Schema.Struct({ links: Schema.Number }), + failure: OrchestratorMcpFailure, + failureMode: "return", + }).annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff), + Tool.make("where", { + description: "Says where it ran.", + parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget }), + success: Schema.Struct({ ranIn: Schema.String }), + failure: OrchestratorMcpFailure, + failureMode: "return", + }), + ); + const settingsContext = yield* Layer.build(ServerSettings.layerTest()); + const settings = Context.get(settingsContext, ServerSettings.ServerSettingsService); + const serverLayer = McpHttpServer.toolkitRegistration( + LinksToolkit, + McpToolAccess.toLayer(LinksToolkit, { + links: McpToolAccess.reads(() => Effect.succeed({ links: 0 })), + where: McpToolAccess.reads(() => Effect.succeed({ ranIn: "here" })), + }), + ).pipe( + Layer.provideMerge( + McpHttpServer.layerMcpTransport.pipe( + Layer.provide(Layer.succeedContext(settingsContext)), + ), + ), + Layer.provide(McpToolAccessTestkit.liveThreadsLayer), + Layer.provide( + Layer.mock(McpSessionRegistry.McpSessionRegistry)({ + resolve: (presented) => Effect.succeed(presented === token ? scope : undefined), + }), + ), + ); + yield* HttpRouter.serve(serverLayer, { disableListenLog: true, disableLogger: true }).pipe( + Layer.build, + ); + const httpClient = yield* HttpClient.HttpClient; + const headers = { + accept: "application/json, text/event-stream", + authorization: `Bearer ${token}`, + }; + const initialize = yield* httpClient.post("/mcp", { + headers, + body: HttpBody.text( + `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"probe","version":"1.0.0"}}}`, + "application/json", + ), + }); + const rpc = (method: string, params: object) => + httpClient + .post("/mcp", { + headers: { + ...headers, + "mcp-session-id": initialize.headers["mcp-session-id"]!, + "mcp-protocol-version": "2025-06-18", + }, + body: HttpBody.text( + JSON.stringify({ jsonrpc: "2.0", id: 2, method, params }), + "application/json", + ), + }) + .pipe(Effect.flatMap((response) => response.json)); + const listed = () => + rpc("tools/list", {}).pipe( + Effect.map((body) => + (body as { result: { tools: ReadonlyArray<{ name: string }> } }).result.tools.map( + (tool) => tool.name, + ), + ), + ); + const ranIn = (where: EnvironmentId) => + rpc("tools/call", { name: "where", arguments: { environmentId: where } }).pipe( + Effect.map( + (body) => + ( + body as { + result: { + isError?: boolean; + structuredContent?: unknown; + content: ReadonlyArray<{ text: string }>; + }; + } + ).result, + ), + ); + const elsewhere = EnvironmentId.make("environment-elsewhere"); + + expect(yield* listed()).toEqual(["where"]); + expect(yield* ranIn(elsewhere)).toMatchObject({ + isError: true, + content: [{ text: expect.stringContaining("Linked environments is off on this machine") }], + }); + expect((yield* ranIn(environmentId)).structuredContent).toEqual({ ranIn: "here" }); + + yield* settings.updateSettings({ linkedEnvironmentsEnabled: true }); + expect(yield* listed()).toEqual(["links", "where"]); + // On, the call goes on to the forwarding, which this server lacks. + expect(yield* ranIn(elsewhere)).toMatchObject({ + isError: true, + content: [{ text: expect.stringContaining("cannot reach linked environments") }], + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("lists pull request watch tools only to threads that are not delegated children", () => Effect.scoped( Effect.gen(function* () { @@ -1024,7 +1155,9 @@ it.effect("lists pull request watch tools only to threads that are not delegated ["childToken", scopeFor("childToken", true)], ]); const serverLayer = McpHttpServer.layerPullRequestsToolkit.pipe( - Layer.provideMerge(McpHttpServer.layerMcpTransport), + Layer.provideMerge( + McpHttpServer.layerMcpTransport.pipe(Layer.provide(ServerSettings.layerTest())), + ), Layer.provide( Layer.mock(McpSessionRegistry.McpSessionRegistry)({ resolve: (token) => Effect.succeed(scopes.get(token)), @@ -1086,3 +1219,149 @@ it.effect("lists pull request watch tools only to threads that are not delegated }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + +it.effect( + "narrows an OAuth client's modes to the T3-Mode-Limit it sends, and never widens them", + () => + Effect.scoped( + Effect.gen(function* () { + const clientToken = "client-payload.client-signature"; + const clientScope: McpInvocationContext.McpInvocationScope = { + environmentId, + requestNamespace: "client:linked", + thread: undefined, + client: { sessionId: "linked", label: "T3 Code · laptop", access: "auto" }, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + // Starts a thread at the modes the gate hands it, so the outcome is those modes. + const StartsToolkit = Toolkit.make( + Tool.make("starts", { + description: "Reports the modes a started thread would run with.", + parameters: Schema.Struct({ + runtimeMode: Schema.optional(RuntimeMode), + interactionMode: Schema.optional(ProviderInteractionMode), + }), + success: Schema.Struct({ modes: Schema.String }), + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies: [ + McpInvocationContext.McpInvocationContext, + ThreadManagementService.ThreadManagementService, + ], + }), + ); + const serverLayer = McpHttpServer.toolkitRegistration( + StartsToolkit, + McpToolAccess.toLayer(StartsToolkit, { + starts: McpToolAccess.startsThreads( + (input) => input, + (_, modes) => + Effect.succeed({ modes: `${modes.runtimeMode}/${modes.interactionMode}` }), + "stamped", + ), + }), + ).pipe( + Layer.provideMerge( + McpHttpServer.layerMcpTransport.pipe(Layer.provide(ServerSettings.layerTest())), + ), + Layer.provide(McpToolAccessTestkit.liveThreadsLayer), + Layer.provide( + Layer.mock(McpSessionRegistry.McpSessionRegistry)({ + // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. + resolve: () => Effect.succeed(undefined), + }), + ), + Layer.provide( + Layer.succeed(McpHttpServer.McpClientAuthenticator, { + authenticate: (request) => + Effect.succeed( + request.headers.authorization === `Bearer ${clientToken}` + ? clientScope + : undefined, + ), + }), + ), + ); + yield* HttpRouter.serve(serverLayer, { disableListenLog: true, disableLogger: true }).pipe( + Layer.build, + ); + const httpClient = yield* HttpClient.HttpClient; + const decodeOutcome = Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + result: Schema.Struct({ + content: Schema.Array(Schema.Struct({ text: Schema.String })), + }), + }), + ), + ); + const decodeText = Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Union([ + Schema.Struct({ modes: Schema.String }), + Schema.Struct({ code: Schema.String }), + ]), + ), + ); + /** Starts a thread as the client, with `limit` as T3-Mode-Limit; the HTTP status or the outcome. */ + const start = (limit: string | undefined, args: Record = {}) => + Effect.gen(function* () { + const headers = { + accept: "application/json, text/event-stream", + authorization: `Bearer ${clientToken}`, + ...(limit === undefined ? {} : { "t3-mode-limit": limit }), + }; + const initialize = yield* httpClient.post("/mcp", { + headers, + body: HttpBody.text( + `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"probe","version":"1.0.0"}}}`, + "application/json", + ), + }); + if (initialize.status !== 200) return initialize.status; + const response = yield* httpClient.post("/mcp", { + headers: { + ...headers, + "mcp-session-id": initialize.headers["mcp-session-id"]!, + "mcp-protocol-version": "2025-06-18", + }, + body: HttpBody.text( + JSON.stringify({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { name: "starts", arguments: args }, + }), + "application/json", + ), + }); + const body = yield* decodeOutcome(yield* response.text); + const outcome = yield* decodeText( + body.result.content.map((part) => part.text).join(""), + ); + return "code" in outcome ? outcome.code : outcome.modes; + }); + + // Approved for auto: without the header the client keeps its ceiling. + expect(yield* start(undefined)).toBe("auto/default"); + // The header narrows both modes, and asking above it is refused. + expect(yield* start("approval-required/plan")).toBe("approval-required/plan"); + expect(yield* start("approval-required/plan", { runtimeMode: "auto" })).toBe( + "runtime_mode_escalation_denied", + ); + expect(yield* start("auto-accept-edits/plan", { interactionMode: "default" })).toBe( + "interaction_mode_escalation_denied", + ); + // A broader header changes nothing: the approved ceiling still holds. + expect(yield* start("full-access/default")).toBe("auto/default"); + expect(yield* start("full-access/default", { runtimeMode: "full-access" })).toBe( + "runtime_mode_escalation_denied", + ); + // A malformed header is refused before anything runs. + expect(yield* start("auto")).toBe(400); + expect(yield* start("auto/default/extra")).toBe(400); + expect(yield* start("root/default")).toBe(400); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index e141674c2f478..ae5013120bdbe 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -32,6 +32,7 @@ import { import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as DeviceService from "../device/DeviceService.ts"; import * as HtmlRender from "../htmlRender/HtmlRender.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; @@ -189,9 +190,23 @@ const withModeLimit = ( : { ...invocation, client: { ...invocation.client, narrowedTo: limit.value } }; }; +// The Linked environments beta is off here: links wait until it is on again. +const linkedEnvironmentsOff = HttpServerResponse.jsonUnsafe( + { + error: "linked_environments_off", + message: "Linked environments is off on this machine. Turn it on in its Settings → Labs.", + }, + { status: 403, headers: { "cache-control": "no-store" } }, +); + const makeMcpAuthMiddleware = Effect.gen(function* () { const registry = yield* McpSessionRegistry.McpSessionRegistry; const clients = yield* Effect.serviceOption(McpClientAuthenticator); + const serverSettings = yield* ServerSettings.ServerSettingsService; + const linksEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.linkedEnvironmentsEnabled), + Effect.orElseSucceed(() => false), + ); return Effect.fn("McpHttpServer.authenticateRequest")(function* (httpEffect) { const request = yield* HttpServerRequest.HttpServerRequest; const authorization = request.headers.authorization; @@ -216,8 +231,13 @@ const makeMcpAuthMiddleware = Effect.gen(function* () { } const limited = withModeLimit(invocation, request); if (limited === undefined) return invalidModeLimit; + const linkedEnvironments = yield* linksEnabled; + if (limited.client?.linked === true && !linkedEnvironments) return linkedEnvironmentsOff; return yield* httpEffect.pipe( - Effect.provideService(McpInvocationContext.McpInvocationContext, limited), + Effect.provideService(McpInvocationContext.McpInvocationContext, { + ...limited, + linkedEnvironments, + }), Effect.map(normalizeMcpHttpResponse), ); }) satisfies McpAuthMiddleware; diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 2134ca54146f3..cc9ba417db8dd 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -102,6 +102,8 @@ export interface McpInvocationScope { readonly requestNamespace: string; readonly thread: McpThreadCaller | undefined; readonly client: McpClientCaller | undefined; + /** False while the Linked environments beta is off here; `/mcp` reads it per request. */ + readonly linkedEnvironments?: boolean; } export class McpInvocationContext extends Context.Service< @@ -119,6 +121,13 @@ export const hiddenFromSubagents = (): boolean => { return scope?.thread?.subagent !== true; }; +/** `McpSchema.EnabledWhen` for tools that only serve links, hidden while the beta is off. */ +export const hiddenWhileLinksOff = (): boolean => { + const fiber = Fiber.getCurrent(); + const scope = fiber && Context.getOrUndefined(fiber.context, McpInvocationContext); + return scope?.linkedEnvironments !== false; +}; + export type McpCapabilityError = C extends "preview" ? PreviewAutomationUnavailableError : McpCapabilityUnavailableError; diff --git a/apps/server/src/mcp/McpToolAccess.ts b/apps/server/src/mcp/McpToolAccess.ts index f7a98a2ee2732..b82d9bb5ae59c 100644 --- a/apps/server/src/mcp/McpToolAccess.ts +++ b/apps/server/src/mcp/McpToolAccess.ts @@ -162,7 +162,7 @@ export const writes = ( () => orchestratingCaller, ); -/** Requires handler capability checks, not thread control; linked callers are refused. */ +/** Requires handler capability checks, not thread control; linked work is refused the same way. */ export const writesWithCapability = (handle: (params: P) => Effect.Effect) => declare( (params: P) => @@ -191,9 +191,7 @@ export const writesThreads = ( yield* assertSameLink(caller, target); } } - // The target's user can raise its modes after the check above; the - // orchestrator checks again under the thread's lock and records its - // refusal here, since handlers wrap dispatch errors their own way. + // Modes can rise after this check; the orchestrator rechecks under lock and records refusal before handlers wrap errors. const refused = yield* Ref.make(undefined); return yield* handle(params).pipe( Effect.provideService(DispatchModeLimit, { ...caller.limits, refused }), @@ -266,14 +264,20 @@ export const either = ( first: Declaration<(params: P) => Effect.Effect>, second: Declaration<(params: P) => Effect.Effect>, ) => { - const firstRemote = remoteCheckOf(first) as RemoteCheck

| undefined; - const secondRemote = remoteCheckOf(second) as RemoteCheck

| undefined; + const remotes = [remoteCheckOf(first), remoteCheckOf(second)] as ReadonlyArray< + RemoteCheck

| undefined + >; return declare( (params: P): Effect.Effect => useFirst(params) ? checkedHandler(first)(params) : checkedHandler(second)(params), - firstRemote === undefined || secondRemote === undefined - ? undefined - : (params: P) => (useFirst(params) ? firstRemote(params) : secondRemote(params)), + (params) => + remotes[useFirst(params) ? 0 : 1]?.(params) ?? + Effect.fail( + new OrchestratorMcpFailure({ + code: "invalid_request", + message: "This action acts as the calling thread, which is here. Omit environmentId.", + }), + ), ); }; @@ -395,6 +399,7 @@ const routed = Effect.fn("McpToolAccess.routed")(function* )(params); if (Option.isNone(forwarding)) return yield* linksUnavailable; return yield* forwarding.value.call(scope, tool, target, { ...params, environmentId }); diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 6d815f60d2c34..31b06c82e9bfb 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -16,6 +16,8 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + ProviderThreadId, + ProviderTurnId, RunId, ThreadId, type OrchestrationV2ServerCommand, @@ -25,13 +27,17 @@ import { import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; +import * as Stream from "effect/Stream"; import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts"; +import * as EffectWorker from "../orchestration-v2/EffectWorker.ts"; import * as LegacyV1ThreadImporter from "../orchestration-v2/legacy/LegacyV1ThreadImporter.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProviderReplayHarness from "../orchestration-v2/testkit/ProviderReplayHarness.ts"; @@ -3202,3 +3208,186 @@ describe("OrchestratorMcpService thread lookup", () => { }), ); }); + +describe("OrchestratorMcpService thread stop", () => { + const driver = ProviderDriverKind.make("codex"); + const instanceId = ProviderInstanceId.make("codex"); + const modelSelection = { instanceId, model: "gpt-5.4" }; + const threadId = ThreadId.make("thread:mcp-stop"); + const client: McpInvocationScope = { + environmentId: EnvironmentId.make("environment:mcp-stop"), + requestNamespace: "client:mcp-stop", + thread: undefined, + client: { sessionId: "mcp-stop", label: "Claude Code", access: "full-access" }, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + }; + + // A provider whose turns run until they are interrupted. + const liveAdapter = (cwd: string): ProviderAdapter.ProviderAdapterV2["Service"] => ({ + instanceId, + driver, + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const events = yield* Queue.unbounded(); + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId: appThreadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:${appThreadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: `native:${appThreadId}`, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: null, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }).pipe(Effect.asVoid), + steerTurn: () => Effect.die("unused"), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.die("unused"), + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }); + + it.effect.each([ + { name: "a running turn", queued: false }, + { name: "a running turn with a queued message", queued: true }, + ])("t3_thread_interrupt stop reports the run it interrupted on $name", ({ queued }) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("mcp-thread-stop"); + const layerAdapters = ProviderAdapterRegistry.layerSingle(liveAdapter(cwd)); + const layerDatabase = SqlitePersistence.layerMemory; + const layerOrchestrator = ProviderReplayHarness.layerWithRegistry( + { name: "mcp-thread-stop" }, + layerAdapters, + { databaseLayer: layerDatabase, runEffectWorker: false }, + ); + const layerThreads = ThreadManagementService.layer.pipe( + Layer.provideMerge(layerOrchestrator), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), + ); + const layerTest = layerOrchestratorService.pipe( + Layer.provideMerge(layerThreads), + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + layerAdapters, + ServerSettings.layerTest(), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ), + ), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const send = (text: string, type: "start_immediately" | "queue_after_active") => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`send:${text}`), + threadId, + messageId: MessageId.make(`message:${text}`), + text, + attachments: [], + dispatchMode: { type }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project:mcp-stop"), + title: "Stop", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* send("work", "start_immediately"); + yield* worker.drain(); + yield* Fiber.join(running); + if (queued) yield* send("later", "queue_after_active"); + const before = yield* orchestrator.getThreadProjection(threadId); + const runningRun = before.runs.find((run) => run.status === "running")!; + assert.equal(before.runs.filter((run) => run.status === "queued").length, queued ? 1 : 0); + + const stopped = yield* service.interruptThread(client, { + threadId, + stop: true, + clientRequestId: "stop", + }); + assert.deepEqual(stopped, { + threadId, + runId: runningRun.id, + status: "interrupt_requested", + }); + }).pipe(Effect.provide(layerTest)); + }), + ), + ); +}); diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index d3aebcf6cbf15..14b86b32f9ac9 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -2975,6 +2975,11 @@ const make = Effect.gen(function* () { resultRun: { runId: result.resultRun.id, status: runOutcome(result.resultRun) }, }), ...(wakesCaller ? { next: ORCHESTRATOR_MCP_RUNNING_TASK_NEXT } : {}), + ...(result.resultRun === undefined + ? {} + : { + resultRun: { runId: result.resultRun.id, status: runOutcome(result.resultRun) }, + }), } satisfies OrchestratorMcpThreadWaitResult; }), interruptThread: Effect.fn("OrchestratorMcpService.interruptThread")(function* (scope, input) { @@ -3012,9 +3017,15 @@ const make = Effect.gen(function* () { .stopDelegatedTasks({ threadId: input.threadId, commandId, reason: input.reason }) .pipe(Effect.catch(stopFailed)); // The run this stop interrupted, from its own events, so a retry - // reports the same run even if another has started since. + // reports the same run even if another has started since. Stop also + // holds queued runs and marks runs it could not interrupt, with no + // provider turn; neither is the run it interrupted. const interrupted = stopped.storedEvents.find( - (stored) => stored.event.type === "run.updated", + ({ event }) => + (event.type === "run.updated" && event.payload.status === "interrupted") || + (event.type === "turn-item.updated" && + event.payload.type === "run_interrupt_request" && + event.payload.providerTurnId !== null), )?.event.runId; return { threadId: input.threadId, diff --git a/apps/server/src/mcp/linkOrigin.test.ts b/apps/server/src/mcp/linkOrigin.test.ts index 94feef3449152..c17f4ba579dd6 100644 --- a/apps/server/src/mcp/linkOrigin.test.ts +++ b/apps/server/src/mcp/linkOrigin.test.ts @@ -115,8 +115,9 @@ const layerLaunches = Layer.effect( const layerTools = Layer.mergeAll( McpHttpServer.layerOrchestratorToolkit, McpHttpServer.layerThreadToolkit, - McpHttpServer.layerProjectRegistration.pipe(Layer.provide(DocumentText.layer)), + McpHttpServer.layerProjectRegistration, ).pipe( + Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provideMerge(McpServer.McpServer.layer), Layer.provide(layerLaunches), Layer.provide(NodeCrypto.layer), diff --git a/apps/server/src/mcp/toolkits/environment/tools.ts b/apps/server/src/mcp/toolkits/environment/tools.ts index 1b240a0927853..4664a176a3fc8 100644 --- a/apps/server/src/mcp/toolkits/environment/tools.ts +++ b/apps/server/src/mcp/toolkits/environment/tools.ts @@ -41,7 +41,7 @@ const shared = { const EnvironmentReadTool = Tool.make("t3_environment_read", { ...shared, description: - "Read this server's identity and preferences, or a linked environment with `environmentId`. Providers and models: `orchestrator_capabilities`.", + "Read this server's identity and preferences. Providers and models: `orchestrator_capabilities`.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget }), success: Schema.Struct({ environmentId: ExecutionEnvironmentDescriptor.fields.environmentId, diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts new file mode 100644 index 0000000000000..ea0caa4ce166a --- /dev/null +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "@effect/vitest"; +import * as Exit from "effect/Exit"; +import * as Schema from "effect/Schema"; +import { Tool } from "effect/ai"; + +import { OrchestratorToolkit } from "./tools.ts"; + +// Oversized page sizes clamp on decode, so the schema has no `maximum`; Codex also drops +// bounds from its TypeScript view. The description is the only place a model sees a limit. +it("states t3_thread_read page size limits in the parameter descriptions", () => { + const schema = Tool.getJsonSchema(OrchestratorToolkit.tools.t3_thread_read) as { + readonly properties: Readonly>; + }; + const limits = { limit: "100", runLimit: "50", maxCharsPerItem: "50000" }; + for (const [name, maximum] of Object.entries(limits)) { + expect(schema.properties[name]?.description, name).toContain(maximum); + } +}); + +it("refuses a t3_thread_update whose fields do not fit its action", () => { + const decode = Schema.decodeUnknownExit( + OrchestratorToolkit.tools.t3_thread_update.parametersSchema, + ); + expect(Exit.isFailure(decode({ action: "rename" }))).toBe(true); + expect(Exit.isFailure(decode({ action: "regenerate_title", title: "New" }))).toBe(true); + expect(Exit.isSuccess(decode({ action: "rename", title: "New", environmentId: undefined }))).toBe( + true, + ); +}); diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index c2025179b18da..42226e606196b 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -38,7 +38,7 @@ import { ThreadMetadataMcpUpdateResult, TrimmedNonEmptyString, } from "@t3tools/contracts"; -import { Tool, Toolkit } from "effect/ai"; +import { McpSchema, Tool, Toolkit } from "effect/ai"; import * as Schema from "effect/Schema"; import * as Struct from "effect/Struct"; @@ -307,7 +307,7 @@ const ThreadInterruptTool = Tool.make("t3_thread_interrupt", { const EnvironmentLinksTool = Tool.make("t3_environment_links", { description: - "List the other T3 Code environments this one is linked to, such as the user's other machines, and whether each answers now. A machine the user names that is not listed is not linked yet: call t3_environment_link for it rather than looking elsewhere. Pass one's environmentId to any tool that takes environmentId, such as t3_thread_launch, t3_thread_send, t3_pending_request_respond, t3_queue_list or t3_project_list, to act there; delegate_task takes it as target.environmentId. A linked environment lets this one's agents do at most its access there, and never more than the calling agent may do here. An expired link has to be linked again from a new pairing code.", + "List linked environments (the user's other machines) and whether each answers now. Pass one's `environmentId` to a tool to act there; `delegate_task` takes `target.environmentId`. Expired: link it again with `t3_environment_link`.", success: OrchestratorMcpEnvironmentLinksResult, failure: OrchestratorMcpFailure, failureMode: "return", @@ -316,11 +316,12 @@ const EnvironmentLinksTool = Tool.make("t3_environment_links", { .annotate(Tool.Title, "List linked environments") .annotate(Tool.Readonly, true) .annotate(Tool.Destructive, false) - .annotate(Tool.Idempotent, true); + .annotate(Tool.Idempotent, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff); const EnvironmentLinkTool = Tool.make("t3_environment_link", { description: - 'Ask the user to link another T3 Code environment, such as their VPS or another machine, so agents here can work there (t3_environment_links). Call it when the user asks to link, connect, or delegate to another machine. You usually do not know where it answers, so pass no address: pass a hint from their words ("vps") and the card lets the user pick the machine. Pass environmentId when a machine the user mentioned names it; if it is already linked, this returns linked at once. Pass url only when the user gave an address. The card lets the user pick the access agents here get there and approve it; their app signs in there with their own session, or they paste a pairing code from there into the card. It returns pending at once: the user answers in the card whenever they get to it, and the outcome (linked with the environmentId to pass to other tools, declined, or failed with the reason) arrives as a message in this thread. End your turn or keep working; do not poll or call it again to check. Never ask for a pairing code in chat.', + "Ask the user to link another machine, in a card. For a machine the user names that is not linked. Pass `hint` from their words, `environmentId` when one names it (already linked: returns `linked`), `url` only when the user gave an address. Returns `pending`; the outcome arrives as a message in this thread. Do not wait, poll, or call it again to check. Never ask for a pairing code in chat.", parameters: OrchestratorMcpEnvironmentLinkInput, success: OrchestratorMcpEnvironmentLinkResult, failure: OrchestratorMcpFailure, @@ -329,11 +330,12 @@ const EnvironmentLinkTool = Tool.make("t3_environment_link", { }) .annotate(Tool.Title, "Link another environment") .annotate(Tool.Destructive, false) - .annotate(Tool.OpenWorld, true); + .annotate(Tool.OpenWorld, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff); const EnvironmentUnlinkTool = Tool.make("t3_environment_unlink", { description: - "Forget a linked environment, so agents here stop working there. The session there stays listed in that environment's Connections until it is revoked there.", + "Forget a linked environment. That machine's Connections list its session until revoked there.", parameters: OrchestratorMcpEnvironmentUnlinkInput, success: OrchestratorMcpEnvironmentUnlinkResult, failure: OrchestratorMcpFailure, @@ -342,7 +344,8 @@ const EnvironmentUnlinkTool = Tool.make("t3_environment_unlink", { }) .annotate(Tool.Title, "Forget a linked environment") .annotate(Tool.Destructive, true) - .annotate(Tool.Idempotent, true); + .annotate(Tool.Idempotent, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff); export const OrchestratorToolkit = Toolkit.make( OrchestratorCapabilitiesTool, diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index 85e41bebf7c73..b6b6d23d843ef 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -255,6 +255,7 @@ it.effect("a launch retried with the same clientRequestId replays the first one" // The same key sends the same command and thread ids, which the launch // service replays instead of starting a second thread. expect(ids[0]).toEqual(ids[1]); + expect(ids[0]?.[0]).toMatch(/^mcp:launch:[0-9a-f]{64}$/); expect(ids[2]).not.toEqual(ids[0]); // Without a key every launch is new. expect(ids[3]).not.toEqual(ids[4]); diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index f5fa81af78349..e0e54e9b9aabc 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -265,8 +265,7 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { Effect.gen(function* () { const projects = yield* Project.ProjectService; if (workspaceRoot === undefined) { - // Project creation records no model default (only an update does), so - // reject what this mode would otherwise drop silently. + // Only updates record model defaults; reject defaults that creation would silently drop. if ( input.scripts !== undefined || input.createWorkspaceRootIfMissing !== undefined || diff --git a/apps/server/src/mcp/toolkits/project/tools.ts b/apps/server/src/mcp/toolkits/project/tools.ts index 1b03dbffd9414..4dc3107c7a027 100644 --- a/apps/server/src/mcp/toolkits/project/tools.ts +++ b/apps/server/src/mcp/toolkits/project/tools.ts @@ -63,8 +63,7 @@ const ProjectListTool = Tool.make("t3_project_list", { .annotate(Tool.Destructive, false); const ProjectReadTool = Tool.make("t3_project_read", { ...shared, - description: - "Read one project: workspace and saved scripts. Pass environmentId for a linked environment.", + description: "Read one project: workspace and saved scripts.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, projectId: ProjectId, @@ -105,7 +104,7 @@ const ProjectDeleteTool = Tool.make("t3_project_delete", { const ProjectCloneTool = Tool.make("t3_project_clone", { ...shared, description: - "Clone a repository (`destinationPath` new or empty). Register after with `t3_project_create`. Pass environmentId to clone there, then register there with the same ID. Existing destinations are not adopted or removed on failure.", + "Clone a repository (`destinationPath` new or empty). Register after with `t3_project_create`, with the same `environmentId`.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, ...SourceControlCloneRepositoryInput.fields, @@ -121,7 +120,7 @@ const ProjectCloneTool = Tool.make("t3_project_clone", { const ThreadLaunchTool = Tool.make("t3_thread_launch", { ...shared, description: - "Create one new top-level thread, bound to its workspace before the agent starts. Task in `message`. Omit `projectId`, model and modes to inherit yours (outside a thread, pass `projectId` or `scratch`); modes cannot exceed yours. Without a retry key, after an error or lost reply, check `t3_thread_list` before retrying. Follow it with `t3_thread_read` or `t3_thread_wait`. Mention it with its `link`. Pass clientRequestId and reuse the same arguments on retries to return the first accepted thread. environmentId launches remotely; project and provider IDs belong there, attachments are unsupported, and remote threads should be named by title rather than linked.", + "Create one new top-level thread, bound to its workspace before the agent starts. Task in `message`. Omit `projectId`, model and modes to inherit yours (outside a thread, pass `projectId` or `scratch`); modes cannot exceed yours. After an error or lost reply, retry with the same `clientRequestId`. Follow it with `t3_thread_read` or `t3_thread_wait`. Mention it with its `link`.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, projectId: Schema.optional(ProjectId), diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index 3354033cd6483..f1e020aae893f 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -309,9 +309,7 @@ const ListThreadPullRequestsTool = Tool.make("list_thread_pull_requests", { "List PRs linked to a thread, with last known state and stack order (bottom to top). Call only when you need that list. Never as an end-of-work check.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, - threadId: Schema.optional( - ThreadId.annotate({ description: "Thread to list. Omit for this thread." }), - ), + threadId: Schema.optional(ThreadId.annotate({ description: "Thread. Default: this thread." })), }), success: ListThreadPullRequestsResult, failure: PullRequestToolError, diff --git a/apps/server/src/mcp/toolkits/thread/tools.ts b/apps/server/src/mcp/toolkits/thread/tools.ts index 5ccfb9bbe090f..3d55130faceff 100644 --- a/apps/server/src/mcp/toolkits/thread/tools.ts +++ b/apps/server/src/mcp/toolkits/thread/tools.ts @@ -85,7 +85,6 @@ const QueueTool = Tool.make("t3_queue", { description: "Manage a thread's queued messages. Omit `threadId` for this thread. list: send order, paged by `cursor` and `limit`. read: up to 16,000 chars of one. edit: replace its `text`; attachments stay. cancel. reorder: before `beforeRunId`, or to the end with null. promote_to_steer: send it now as steering to active `targetRunId`. Writes fail once the run is no longer queued.", parameters: Schema.Struct({ - environmentId: OrchestratorMcpEnvironmentTarget, action: Schema.Literals([ ...QUEUE_READ_ACTIONS, "edit", @@ -93,6 +92,7 @@ const QueueTool = Tool.make("t3_queue", { "reorder", "promote_to_steer", ]), + environmentId: OrchestratorMcpEnvironmentTarget, threadId: Schema.optional(ThreadId), queuedRunId: Schema.optional(RunId), text: Schema.optional(Schema.String.check(Schema.isMaxLength(100000))), @@ -129,8 +129,8 @@ const PendingRequestTool = Tool.make("t3_pending_request", { description: "Pending user questions in a thread. Omit `threadId` for this thread. list: their ids; approvals are not listed. read: one by `requestId`. respond: answer one with `answers`; this cannot approve a permission request.", parameters: Schema.Struct({ - environmentId: OrchestratorMcpEnvironmentTarget, action: Schema.Literals(["list", "read", "respond"]), + environmentId: OrchestratorMcpEnvironmentTarget, threadId: Schema.optional(ThreadId), requestId: Schema.optional(RuntimeRequestId), answers: Schema.optional(ProviderUserInputAnswers), @@ -167,8 +167,8 @@ const ThreadTransferTool = Tool.make("t3_thread_transfer", { description: "Move context between threads. Omit `threadId` for this thread. fork: copy it from `sourcePoint` (default latest stable run) into a new thread with the same setup; no turn runs yet. merge_back: merge it from `sourcePoint` into a related `targetThreadId` in the same project. status: its context transfers.", parameters: Schema.Struct({ - environmentId: OrchestratorMcpEnvironmentTarget, action: Schema.Literals(["fork", "merge_back", "status"]), + environmentId: OrchestratorMcpEnvironmentTarget, threadId: Schema.optional(ThreadId), sourcePoint: Schema.optional(OrchestrationV2ThreadForkSourcePoint), targetThreadId: Schema.optional(ThreadId), diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index a919726c9a56e..92eb5b1047ff8 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -909,7 +909,11 @@ describe("AcpAdapterV2", () => { flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, protocolEvents }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + }), }, selfInvocation, }); @@ -2406,6 +2410,99 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(layerTest), Effect.scoped), ); + it.live("creates a fresh ACP identity when recovering an exhausted target", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const path = yield* Path.Path; + const selfInvocation = yield* resolveSelfInvocation(); + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.unbounded(); + const instanceId = ProviderInstanceId.make("acp-test-exhausted-target"); + const adapter = yield* makeAcpAdapterV2({ + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + protocolEvents, + wrapRuntime: (runtime, ordinal) => ({ + ...runtime, + start: () => + runtime.start().pipe( + Effect.map((started) => ({ + ...started, + sessionId: `mock-recovery-session-${ordinal}`, + })), + ), + }), + }), + }, + selfInvocation, + }); + const threadId = ThreadId.make("thread-acp-exhausted-target"); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-exhausted-target"), + modelSelection, + runtimePolicy, + }); + const original = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread: original, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 1, + }), + ); + yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runDrain, + ); + const replacement = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...original, nativeThreadRef: null }, + forceNewNativeThread: true, + }); + assert.notEqual(replacement.nativeThreadRef?.nativeId, original.nativeThreadRef?.nativeId); + assert.equal(replacement.id, original.id); + assert.equal(replacement.nativeMetadata?.itemIdentityVersion, 2); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread: replacement, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + }), + ); + const events = yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ); + assert.isTrue( + events.some((event) => event.type === "turn.terminal" && event.status === "completed"), + ); + const methods = yield* pollProtocolMethods(protocolEvents); + assert.equal(methods.filter((method) => method === "session/new").length, 2); + }).pipe(Effect.provide(layerTest), Effect.scoped), + ); + it.live("preserves new-session fallback when an eager ACP session load is stale", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -5240,6 +5337,19 @@ describe("AcpAdapterV2", () => { yield* hasPendingBackgroundWork, "carryover live subagent must pin hasPendingBackgroundWork after root settle", ); + const refusedReplacement = yield* runtime + .ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...providerThread, nativeThreadRef: null }, + forceNewNativeThread: true, + }) + .pipe(Effect.flip); + assert.equal(refusedReplacement._tag, "ProviderAdapterEnsureThreadError"); + const preserved = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); + assert.equal(preserved.nativeThreadRef?.nativeId, providerThread.nativeThreadRef?.nativeId); + assert.isTrue(yield* hasPendingBackgroundWork); assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); subagentPhase = "complete"; diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index a13fcc866e36f..7dc1b4d432563 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -7811,7 +7811,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("stops background work after the turn settled", () => + it.effect("preserves settled background work until an explicit stop", () => Effect.scoped( Effect.gen(function* () { let closes = 0; @@ -7840,6 +7840,29 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(settledThread?.pendingBackgroundTasks?.[0]?.taskId, WAKE_TASK_ID); assert.isTrue(yield* harness.hasPendingBackgroundWork); + const replacementAttempt = yield* harness.runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: { + ...harness.providerThread, + nativeThreadRef: { + ...harness.providerThread.nativeThreadRef!, + nativeId: "replacement-session", + }, + }, + now, + attemptId: RunAttemptId.make("attempt-claude-background-replacement"), + text: "Continue in a fresh session.", + attachments: [], + providerTurnOrdinal: 2, + }), + ) + .pipe(Effect.result); + assert.equal(replacementAttempt._tag, "Failure"); + assert.equal(closes, 0); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + yield* harness.runtime.interruptTurn({ providerThread: settledThread ?? harness.providerThread, providerTurnId: harness.terminalEvents()[0]!.providerTurnId, @@ -12267,6 +12290,25 @@ describe("ClaudeAdapterV2 background wake turns", () => { yield* changes.until(() => terminals().length === 1); const settledTurn = terminals()[0]!; + assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(providerThread)); + assert.isTrue( + yield* runtime.hasPendingBackgroundWorkForThread!(providerThread, { + includeSubagents: true, + }), + ); + assert.isFalse( + yield* runtime.hasPendingBackgroundWorkForThread!( + { + ...providerThread, + nativeThreadRef: { + ...providerThread.nativeThreadRef!, + nativeId: "sibling-native-session", + }, + }, + { includeSubagents: true }, + ), + ); + const alternateModel = CLAUDE_TEST_RESTART_MODEL_SELECTION; const switchTurn = (attempt: string) => runtime.startTurn( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 35dc61378a3dd..1185f7af08187 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -9686,10 +9686,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return existing; } - // Selection changes kill CLI background work; refuse until it finishes or Stop closes the process. + // Replacing this thread's process kills its background work, even with a new native identity. if ( existing !== null && - existing.nativeThreadId === nativeThreadId && + (existing.nativeThreadId === nativeThreadId || + existing.lastTurn?.input.threadId === turnInput.threadId) && !existing.stopping && (yield* liveProcessRunsBackgroundWork(existing)) ) { @@ -10555,11 +10556,20 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), hasPendingBackgroundWorkForThread: Effect.fn( "ClaudeAdapterV2.hasPendingBackgroundWorkForThread", - )(function* (providerThread) { + )(function* (providerThread, options) { const nativeThreadId = providerThread.nativeThreadRef?.nativeId; if (nativeThreadId === undefined || nativeThreadId === null) { return false; } + if (options?.includeSubagents) { + const live = yield* Ref.get(queryContext); + if ( + live?.nativeThreadId === nativeThreadId && + !live.stopping && + (yield* liveProcessRunsBackgroundWork(live)) + ) + return true; + } return ( rosterForNativeThread( yield* Ref.get(pendingBackgroundTasksByNativeThread), diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 5570401bdfef0..05390d35adc37 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -2492,6 +2492,436 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each([ + "recover", + "completion-only", + "exhaust", + "stop", + "stop-continued", + "stop-starting", + "started-first", + "completed-first", + "new-run", + "steer", + "start-fails", + ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) => + Effect.scoped( + Effect.gen(function* () { + const scenario = `capacity-${outcome}`; + const nativeThreadId = `native-${scenario}`; + const nativeTurnId = (attempt: number) => `${nativeThreadId}-${attempt}`; + const prompt = "Continue the task."; + const capacityError = { + message: "Selected model is at capacity. Please try a different model.", + codexErrorInfo: "serverOverloaded", + }; + const failureEntries = (attempt: number): Array => [ + { + type: "emit_inbound", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId(attempt), + willRetry: false, + error: capacityError, + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "failed" }), + error: capacityError, + }, + }, + }, + }, + ]; + const entries = codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(0), + prompt, + }); + entries.push( + ...(outcome === "completion-only" ? failureEntries(0).slice(1) : failureEntries(0)), + ); + if (outcome === "completion-only") entries.push(...failureEntries(0).slice(1)); + const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1; + const steeredText = "Use the smaller change."; + for (let attempt = 1; attempt <= retries; attempt++) { + const start = codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(attempt), + prompt, + }).slice(-3); + if ( + outcome === "started-first" || + outcome === "stop-starting" || + outcome === "completed-first" || + outcome === "new-run" + ) { + start.splice(1, 0, ...start.splice(2, 1)); + } + for (const entry of start) { + if (entry.type === "expect_outbound") { + entries.push({ + ...entry, + frame: { + id: 3 + attempt, + method: "turn/start", + params: { + threadId: nativeThreadId, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }); + } else if (entry.type === "emit_inbound" && entry.label === "turn/start") { + if (outcome === "completed-first" || outcome === "new-run") + entries.push({ + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }), + }, + }, + }); + if (outcome === "new-run") { + entries.push( + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: nativeTurnId(2), + prompt: "Next task.", + }) + .slice(-3) + .map((next, index) => (index < 2 ? withReplayRequestId(next, 5) : next)), + ...failureEntries(2), + ); + } + entries.push( + outcome === "start-fails" + ? { + type: "emit_inbound", + frame: { + id: 3 + attempt, + error: { code: -32603, message: "Start failed" }, + }, + } + : withReplayRequestId(entry, 3 + attempt), + ); + } else if (outcome !== "start-fails") { + entries.push( + outcome === "stop-starting" && entry.type === "emit_inbound" + ? { ...entry, afterMs: 1_000 } + : entry, + ); + if (outcome === "stop-starting") { + entries.push( + { + type: "expect_outbound", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) }, + }, + }, + { type: "emit_inbound", frame: { id: 5, result: {} } }, + ); + } + } + } + if (outcome === "exhaust") entries.push(...failureEntries(attempt)); + if ( + outcome === "recover" || + outcome === "completion-only" || + outcome === "started-first" || + outcome === "stop-starting" + ) { + entries.push({ + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ + id: nativeTurnId(attempt), + status: outcome === "stop-starting" ? "interrupted" : "completed", + }), + }, + }, + }); + } + if (outcome === "stop-continued" || outcome === "steer") { + const steering = outcome === "steer"; + entries.push( + { + type: "expect_outbound", + frame: { + id: 5, + method: steering ? "turn/steer" : "turn/interrupt", + params: steering + ? { + threadId: nativeThreadId, + expectedTurnId: nativeTurnId(attempt), + input: [{ type: "text", text: steeredText }], + } + : { threadId: nativeThreadId, turnId: nativeTurnId(attempt) }, + }, + }, + { + type: "emit_inbound", + frame: { id: 5, result: steering ? { turnId: nativeTurnId(attempt) } : {} }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ + id: nativeTurnId(attempt), + status: steering ? "completed" : "interrupted", + }), + }, + }, + }, + ); + } + } + const milestones = yield* Queue.unbounded(); + const starts: Array = []; + const retryStarted = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ scenario, entries }), + (event) => + event.type === "turn.terminal" || + (event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running") + ? Queue.offer(milestones, event) + : Effect.void, + (method, params) => + Effect.gen(function* () { + if (method === "turn/start") { + starts.push(params); + if (starts.length === 2) yield* Deferred.succeed(retryStarted, undefined); + } + }), + ); + const turnInput = makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(scenario), + text: prompt, + }); + yield* harness.runtime.startTurn(turnInput); + for (let attempt = 1; attempt <= Math.max(retries, 1); attempt++) { + const milestone = yield* Queue.take(milestones); + assert.equal(milestone.type, "turn_item.updated"); + if (milestone.type !== "turn_item.updated" || milestone.turnItem.type !== "error") return; + const providerTurnId = milestone.turnItem.providerTurnId; + assert.isNotNull(providerTurnId); + if (providerTurnId === null) return; + assert.deepEqual(milestone.turnItem.retry, { + attempt, + maxAttempts: 5, + retryDelayMs: Math.min(2_000 * 2 ** (attempt - 1), 30_000), + }); + assert.isEmpty(harness.terminalEvents()); + if (outcome === "stop") { + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + } + const steering = + outcome === "steer" + ? yield* harness.runtime + .steerTurn({ + ...turnInput, + providerTurnId, + message: { ...turnInput.message, text: steeredText }, + }) + .pipe(Effect.forkScoped) + : undefined; + yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000)); + if (outcome === "stop-starting") { + yield* Deferred.await(retryStarted); + const stopping = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(stopping); + } + if (steering !== undefined) yield* Fiber.join(steering); + if (outcome === "stop-continued") { + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === nativeTurnId(1), + ), + ); + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + } + } + yield* harness.awaitTerminal(1); + if (outcome === "new-run") { + assert.equal((yield* Queue.take(milestones)).type, "turn.terminal"); + yield* harness.runtime.startTurn({ + ...turnInput, + attemptId: RunAttemptId.make("capacity-next-task"), + message: { ...turnInput.message, text: "Next task." }, + }); + const nextRetry = yield* Queue.take(milestones); + assert.equal(nextRetry.type, "turn_item.updated"); + if (nextRetry.type !== "turn_item.updated" || nextRetry.turnItem.type !== "error") return; + assert.equal(nextRetry.turnItem.retry?.attempt, 1); + return; + } + const terminal = harness.terminalEvents()[0]; + assert.equal( + terminal?.status, + outcome === "recover" || + outcome === "completion-only" || + outcome === "steer" || + outcome === "started-first" || + outcome === "completed-first" + ? "completed" + : outcome === "stop" || outcome === "stop-continued" || outcome === "stop-starting" + ? "interrupted" + : "failed", + ); + assert.lengthOf(starts, 1 + retries); + assert.lengthOf(harness.terminalEvents(), 1); + assert.lengthOf( + harness.events.filter( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "running", + ), + outcome === "start-fails" ? 1 : 1 + retries, + ); + const retryItems = harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "error" + ? [event.turnItem] + : [], + ); + assert.isTrue( + ["completed", "failed", "interrupted"].includes(retryItems.at(-1)?.status ?? ""), + ); + assert.lengthOf(new Set(retryItems.map((item) => item.id)), 1); + + if (outcome === "recover") { + const fs = yield* FileSystem.FileSystem; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-capacity-retry-" }); + const localTranscript = yield* decodeReplayTranscriptJson( + (yield* encodeReplayTranscriptJson( + makeCodexReplayTranscript({ scenario, entries }), + )).replaceAll(yield* encodeStringJson("/workspace"), yield* encodeStringJson(cwd)), + ); + const driver = yield* CodexReplay.makeReplayDriver(localTranscript); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = ThreadId.make("thread:capacity-retry"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-capacity-retry"), + threadId, + projectId: ProjectId.make("project:capacity-retry"), + title: "Capacity retry", + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const retrying = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "error" && + event.payload.retry != null && + event.payload.status === "running", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-capacity-retry"), + threadId, + messageId: MessageId.make("message:capacity-retry"), + text: prompt, + attachments: [], + createdBy: "user", + creationSource: "web", + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + yield* Fiber.join(retrying); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "running", + ); + const completed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "run.updated" && event.payload.status === "waiting", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("2 seconds"); + yield* Fiber.join(completed); + yield* worker.drain(); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(projection.runs.at(-1)?.status, "completed"); + assert.deepEqual( + projection.providerTurns.map((turn) => turn.status), + ["failed", "completed"], + ); + assert.lengthOf( + projection.messages.filter((message) => message.role === "user"), + 1, + ); + assert.isNull((yield* Ref.get(driver.state)).failure); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "codex-capacity-retry", runtimePolicyOverride: { cwd } }, + CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver }), + { runEffectWorker: false }, + ), + ), + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("resolves retryable app-server errors on resumed provider activity", () => Effect.scoped( Effect.gen(function* () { @@ -5285,6 +5715,265 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }; + it.effect.each(["completed", "failed", "interrupted", "starting", "replied-last"] as const)( + "handles a late native child after the capacity retry %s", + (parentStatus) => + Effect.scoped( + Effect.gen(function* () { + const retriedTurnId = "native-capacity-recovered-parent"; + const retrying = yield* Deferred.make(); + const entries: Array = [ + ...lateChildPrefix().spawned, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), + error: { + message: "Selected model is at capacity.", + codexErrorInfo: "serverOverloaded", + }, + }, + }, + }, + }, + { + type: "expect_outbound", + frame: { + id: 4, + method: "turn/start", + params: { + threadId: RESUME_NATIVE_THREAD, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }, + ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []), + { + type: "emit_inbound", + frame: { + id: 4, + result: { + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/started", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }), + }, + }, + }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ + id: retriedTurnId, + status: + parentStatus === "starting" || parentStatus === "replied-last" + ? "completed" + : parentStatus, + }), + error: + parentStatus === "failed" + ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" } + : null, + }, + }, + }, + }, + childAgentMessage({ + id: "child-after-capacity", + text: "CAPACITY_CHILD_DONE", + turnId: RESUME_CHILD_TURN_1, + completedAtMs: 1782622480000, + afterMs: 60_000, + }), + childTurnCompleted(RESUME_CHILD_TURN_1), + ]; + if (parentStatus === "replied-last") { + entries.push(...entries.splice(lateChildPrefix().spawned.length + 2, 1)); + } + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ scenario: "capacity-late-child", entries }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("capacity-late-child"), + text: RESUME_PROMPT, + }), + ); + yield* Deferred.await(retrying); + yield* TestClock.adjust("2 seconds"); + if (parentStatus === "interrupted") assert.isEmpty(harness.terminalEvents()); + else { + yield* harness.awaitTerminal(1); + const terminal = harness.terminalEvents()[0]; + if (terminal?.status === "failed") assert.isTrue(terminal.childTurnsContinue); + } + yield* TestClock.adjust("60 seconds"); + yield* harness.until( + () => harness.subagentUpdates().at(-1)?.subagent.status === "completed", + ); + yield* harness.awaitTerminal(1); + assert.lengthOf( + harness.continuationRequests, + parentStatus === "completed" || parentStatus === "replied-last" ? 1 : 0, + ); + if (parentStatus === "completed" || parentStatus === "replied-last") + assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("stops native children when capacity retry startup fails", () => + Effect.scoped( + Effect.gen(function* () { + const retrying = yield* Deferred.make(); + const starting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const interrupts: unknown[] = []; + let starts = 0; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "capacity-start-fails-stop-child", + entries: [ + ...lateChildPrefix().spawned, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }), + error: { message: "At capacity", codexErrorInfo: "serverOverloaded" }, + }, + }, + }, + }, + { + type: "expect_outbound", + frame: { + id: 4, + method: "turn/start", + params: { + threadId: RESUME_NATIVE_THREAD, + input: [], + cwd: "/workspace", + model: "gpt-5.4", + approvalPolicy: "never", + approvalsReviewer: "user", + sandboxPolicy: { type: "dangerFullAccess" }, + summary: "detailed", + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } }, + }, + }, + }, + { + type: "emit_inbound", + frame: { id: 4, error: { code: -32603, message: "Start failed" } }, + }, + { + type: "expect_outbound", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: RESUME_CHILD_THREAD, turnId: RESUME_CHILD_TURN_1 }, + }, + }, + { type: "emit_inbound", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + frame: { + method: "turn/completed", + params: { + threadId: RESUME_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESUME_CHILD_TURN_1, status: "interrupted" }), + }, + }, + }, + ], + }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + (method, params) => + Effect.gen(function* () { + if (method === "turn/interrupt") interrupts.push(params); + if (method === "turn/start" && ++starts === 2) { + yield* Deferred.succeed(starting, undefined); + yield* Deferred.await(release); + } + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("capacity-start-fails-stop-child"), + text: RESUME_PROMPT, + }), + ); + yield* Deferred.await(retrying); + const root = harness.events.find((event) => event.type === "provider_turn.updated"); + assert.equal(root?.type, "provider_turn.updated"); + if (root?.type !== "provider_turn.updated") return; + yield* TestClock.adjust("2 seconds"); + yield* Deferred.await(starting); + const stopping = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: root.providerTurn.id, + }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(stopping); + assert.lengthOf(interrupts, 1); + assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("names a late child spawned without a prompt in its wake", () => Effect.scoped( Effect.gen(function* () { @@ -6722,6 +7411,96 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("Stop prevents a capacity retry while pausing its goal", () => + Effect.gen(function* () { + const nativeTurnId = "goal-capacity-stop"; + const prompt = "Keep going"; + const retrying = yield* Deferred.make(); + const pausing = yield* Deferred.make(); + const requests: Array = []; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "goal-capacity-stop", + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt }), + notification("goal active", "thread/goal/updated", { + threadId: nativeThreadId, + turnId: nativeTurnId, + goal: codexGoal("active", 10), + }), + notification("capacity failure", "turn/completed", { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: nativeTurnId, status: "failed" }), + error: { + message: "Selected model is at capacity.", + codexErrorInfo: "serverOverloaded", + }, + }, + }), + { + type: "expect_outbound", + frame: { + id: 4, + method: "thread/goal/set", + params: { threadId: nativeThreadId, status: "paused" }, + }, + }, + { + type: "emit_inbound", + afterMs: 3_000, + frame: { id: 4, result: { goal: codexGoal("paused", 10) } }, + }, + ], + }), + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.status === "running" + ? Deferred.succeed(retrying, undefined) + : Effect.void, + (method) => + Effect.gen(function* () { + requests.push(method); + if (method === "thread/goal/set") yield* Deferred.succeed(pausing, undefined); + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("goal-capacity-stop"), + text: prompt, + }), + ); + yield* Deferred.await(retrying); + const stopping = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: (yield* IdAllocator.IdAllocatorV2).derive.providerTurn({ + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeTurnId, + }), + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(pausing); + yield* TestClock.adjust("3 seconds"); + assert.lengthOf( + requests.filter((method) => method === "turn/start"), + 1, + ); + yield* Fiber.join(stopping); + yield* harness.awaitTerminal(1); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect("pauses an active goal before Stop interrupts its turn", () => Effect.gen(function* () { const prompt = "Keep going"; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index c83ba62748309..00a4e75585a4c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -86,6 +86,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as Function from "effect/Function"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -150,6 +151,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const CODEX_PROVIDER = ProviderDriverKind.make("codex"); +const CODEX_CAPACITY_MAX_RETRIES = 5; export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); @@ -1107,6 +1109,8 @@ function withCodexManagedFailure( } interface ActiveCodexTurnContext { + capacityRetryAttempt?: number; + capacityRetryPending?: Deferred.Deferred; closedCommandItemIds?: ReadonlySet; latestProviderFailure?: { readonly nativeMessage: string; @@ -1139,6 +1143,7 @@ interface ActiveCodexTurnContext { } interface ActiveCodexProviderRetry { + readonly context?: ActiveCodexTurnContext; readonly nativeMessage: string; readonly nativeCode: string | null; readonly retry: OrchestrationV2ProviderRetry; @@ -1147,6 +1152,30 @@ interface ActiveCodexProviderRetry { readonly itemOrdinal: number; } +function codexRetryTurnItem( + activeContext: ActiveCodexTurnContext, + state: ActiveCodexProviderRetry, + status: Parameters[0]["status"], + updatedAt: DateTime.Utc, +) { + // A continuation keeps the retry row attached to the native turn that failed. + const context = state.context ?? activeContext; + return makeProviderRetryTurnItem({ + driver: CODEX_PROVIDER, + threadId: context.projectionThreadId, + runId: context.projectionRunId, + nodeId: context.providerNodeId, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + itemOrdinal: state.itemOrdinal, + failure: state.failure, + retry: state.retry, + status, + startedAt: state.startedAt, + updatedAt, + }); +} + interface TrackedRunningCommandItem { readonly id: string; readonly command: string; @@ -1830,12 +1859,23 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const goalsByNativeThread = new Map(); const rootProviderThreads = new Map(); const goalHolds = new Map(); - const goalRuns = new Map>(); + const continuedRuns = new Map>(); + const capacityContinuations = new Map< + string, + { readonly context: ActiveCodexTurnContext; readonly retry: ActiveCodexProviderRetry } + >(); const goalActivations = new Map(); // Held runs must not complete while Stop resolves its target. const goalStops = new Map(); - const latestGoalTurnId = (providerTurnId: ProviderTurnId) => - goalRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId; + const latestRunTurnId = (providerTurnId: ProviderTurnId) => + continuedRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId; + const isContinuedRunDescendant = ( + candidate: ActiveCodexTurnContext, + context: ActiveCodexTurnContext, + ) => + (continuedRuns.get(context.providerTurnId) ?? [context]).some((ancestor) => + isDescendantCodexTurn(candidate, ancestor), + ); const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); @@ -1897,20 +1937,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: context.projectionThreadId, - runId: context.projectionRunId, - nodeId: context.providerNodeId, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - itemOrdinal: providerRetry.itemOrdinal, - failure: providerRetry.failure, - retry: providerRetry.retry, - status: "completed", - startedAt: providerRetry.startedAt, - updatedAt, - }), + turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt), }); }); @@ -1949,6 +1976,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( fileChangeKindsByItem: new Map(), runningCompactionItemIds: new Set(), }; + const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId; + const continuation = + nativeThreadId == null ? undefined : capacityContinuations.get(nativeThreadId); + if (continuation !== undefined) { + context.capacityRetryAttempt = continuation.context.capacityRetryAttempt ?? 0; + yield* Ref.update(providerRetries, (current) => + new Map(current).set(providerTurnId, continuation.retry), + ); + const turns = continuedRuns.get(continuation.context.providerTurnId) ?? [ + continuation.context, + ]; + turns.push(context); + continuedRuns.set(continuation.context.providerTurnId, turns); + continuedRuns.set(providerTurnId, turns); + if (continuation.context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(continuation.context.capacityRetryPending, context); + } + } yield* Ref.update(limitedTurnItems, (current) => { const next = new Map(current); next.delete(context.providerThread.id); @@ -1961,6 +2006,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updated.set(input.nativeTurnId, context); return updated; }); + if (continuation !== undefined && nativeThreadId != null) { + capacityContinuations.delete(nativeThreadId); + } yield* emitProviderEvent({ type: "provider_turn.updated", driver: CODEX_PROVIDER, @@ -4395,10 +4443,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( nativeTurnId: payload.turn.id, startedAt: codexTimestamp(payload.turn.startedAt), }); - const goalRun = goalRuns.get(goalHold.context.providerTurnId) ?? [goalHold.context]; + const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [ + goalHold.context, + ]; goalRun.push(next); - goalRuns.set(goalHold.context.providerTurnId, goalRun); - goalRuns.set(next.providerTurnId, goalRun); + continuedRuns.set(goalHold.context.providerTurnId, goalRun); + continuedRuns.set(next.providerTurnId, goalRun); yield* Deferred.succeed(goalHold.next, next); return; } @@ -4508,6 +4558,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( previous?.itemOrdinal ?? (yield* resolveItemOrdinal(context, `terminal-failure:${context.providerTurnId}`)); const state: ActiveCodexProviderRetry = { + ...(previous?.context === undefined ? {} : { context: previous.context }), nativeMessage: payload.error.message, nativeCode: code, retry, @@ -4523,20 +4574,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: context.projectionThreadId, - runId: context.projectionRunId, - nodeId: context.providerNodeId, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - itemOrdinal, - failure, - retry, - status: "running", - startedAt: state.startedAt, - updatedAt, - }), + turnItem: codexRetryTurnItem(context, state, "running", updatedAt), }); }).pipe(Effect.orDie), ); @@ -5632,8 +5670,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } : event; yield* emitProviderEvent(current); - for (const turn of goalRuns.get(context.providerTurnId) ?? []) { - goalRuns.delete(turn.providerTurnId); + const continued = continuedRuns.get(context.providerTurnId) ?? [context]; + if (current.status !== "completed") { + yield* Ref.update( + terminalizedNonCompletedNativeTurns, + (nativeTurns) => + new Set([...nativeTurns, ...continued.map((turn) => turn.nativeTurnId)]), + ); + } + for (const turn of continued) { + continuedRuns.delete(turn.providerTurnId); } const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId; const rootProviderThread = @@ -5671,7 +5717,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }) { const event = yield* makeRootTerminalEvent(input); const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some( - (candidate) => isDescendantCodexTurn(candidate, input.context), + (candidate) => isContinuedRunDescendant(candidate, input.context), ); // Failed roots leave native children working and must settle without them. // Interrupted roots wait for descendants because Stop interrupts the whole lineage. @@ -5796,7 +5842,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( for (const [nativeTurnId, deferred] of current) { if ( !activeTurnContexts.some((candidate) => - isDescendantCodexTurn(candidate, deferred.context), + isContinuedRunDescendant(candidate, deferred.context), ) ) { updated.delete(nativeTurnId); @@ -5818,8 +5864,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( readonly completedAt: DateTime.Utc; readonly failureMessage?: string; readonly failureErrorInfo?: CodexErrorInfo; + readonly continueRun?: boolean; }) => - turnTerminalizationPermit.withPermits(1)( + // Capacity continuation already holds the permit across finalization and restart. + (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))( Effect.gen(function* () { const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId); if (current !== input.context) { @@ -5837,31 +5885,33 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }); if ( providerRetry !== undefined && - (input.status !== "failed" || input.context.subagent !== null) + !input.continueRun && + (providerRetry.context !== undefined || + input.status !== "failed" || + input.context.subagent !== null) ) { yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: input.context.projectionThreadId, - runId: input.context.projectionRunId, - nodeId: input.context.providerNodeId, - providerThreadId: input.context.providerThread.id, - providerTurnId: input.context.providerTurnId, - itemOrdinal: providerRetry.itemOrdinal, - failure: providerRetry.failure, - retry: providerRetry.retry, - status: providerTurnStatusToTerminal(input.status), - startedAt: providerRetry.startedAt, - updatedAt: input.completedAt, - }), + turnItem: codexRetryTurnItem( + input.context, + providerRetry, + input.status === "failed" && + providerRetry.context !== input.context && + providerRetry.context !== undefined + ? "interrupted" + : providerTurnStatusToTerminal(input.status), + input.completedAt, + ), }); } - if (input.status !== "completed") { + if (input.status !== "completed" && !input.continueRun) { yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { const updated = new Set(current); - updated.add(input.nativeTurnId); + for (const turn of continuedRuns.get(input.context.providerTurnId) ?? [ + input.context, + ]) + updated.add(turn.nativeTurnId); return updated; }); } @@ -5982,11 +6032,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // Only direct children wake an idle root turn; grandchildren report to their // own parent, and running root turns receive results natively. const parent = input.context.subagent.parentContext; + const parentNativeThreadId = parent.providerThread.nativeThreadRef?.nativeId; if ( continuationRequests !== undefined && parent.subagent === null && (input.status === "completed" || input.status === "failed") && !(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) && + (parentNativeThreadId == null || + !capacityContinuations.has(parentNativeThreadId)) && ![...(yield* Ref.get(activeTurns)).values()].some( (active) => active.subagent === null && @@ -6032,7 +6085,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( input.completedAt, input.status === "interrupted" || input.status === "failed", ); - if (input.context.subagent === null) { + if (input.context.subagent === null && !input.continueRun) { yield* emitOrDeferRootTerminal({ ...input, ...(providerRetry === undefined ? {} : { providerRetry }), @@ -6066,6 +6119,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (input.context.nativeStartReady !== undefined) { yield* Deferred.succeed(input.context.nativeStartReady, undefined); } + if (!input.continueRun && input.context.capacityRetryPending !== undefined) { + yield* Deferred.succeed(input.context.capacityRetryPending, undefined); + } yield* flushReadyRootTerminals(); if (!retainSettledContext && !interruptInProgress) { yield* Ref.update(runningCommandItemsByTurn, (current) => { @@ -6107,6 +6163,141 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); + // Codex can end an overloaded turn without retry advice. Continue its + // persisted history with empty input instead of resending the user message. + const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* ( + terminal: Parameters[0], + ) { + const context = terminal.context; + const code = + terminal.failureErrorInfo == null + ? context.latestProviderFailure?.nativeCode + : codexErrorInfoCode(terminal.failureErrorInfo); + if ( + terminal.status !== "failed" || + code !== "serverOverloaded" || + context.subagent !== null || + (context.input.message.attachments.length === 0 && + context.input.message.text.trim().toLowerCase() === "/compact") + ) + return false; + + return yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ((yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context) return true; + if (context.capacityRetryPending !== undefined) return true; + if ( + (context.capacityRetryAttempt ?? 0) >= CODEX_CAPACITY_MAX_RETRIES || + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) + ) + return false; + + const attempt = (context.capacityRetryAttempt ?? 0) + 1; + context.capacityRetryAttempt = attempt; + const pending = yield* Deferred.make(); + context.capacityRetryPending = pending; + const retryDelayMs = Math.min(2_000 * 2 ** (attempt - 1), 30_000); + const now = yield* DateTime.now; + const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId); + const state: ActiveCodexProviderRetry = { + context: previous?.context ?? context, + nativeMessage: + terminal.failureMessage ?? + context.latestProviderFailure?.nativeMessage ?? + "Selected model is at capacity.", + nativeCode: code, + failure: makeProviderFailure({ + message: + context.latestProviderFailure?.failure.message ?? terminal.failureMessage, + code, + class: "provider_error", + retryable: true, + }), + retry: { attempt, maxAttempts: CODEX_CAPACITY_MAX_RETRIES, retryDelayMs }, + itemOrdinal: + previous?.itemOrdinal ?? + (yield* resolveItemOrdinal( + context, + `terminal-failure:${context.providerTurnId}`, + )), + startedAt: previous?.startedAt ?? now, + }; + yield* Ref.update(providerRetries, (current) => + new Map(current).set(context.providerTurnId, state), + ); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: codexRetryTurnItem(context, state, "running", now), + }); + yield* Effect.sleep(retryDelayMs).pipe( + Effect.andThen( + Effect.gen(function* () { + const nativeThreadId = yield* getNativeThreadId(context.providerThread); + const claimed = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ( + (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context || + goalStops.has(nativeThreadId) || + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId) + ) + return false; + capacityContinuations.set(nativeThreadId, { context, retry: state }); + yield* finalizeCodexTurn({ ...terminal, continueRun: true }); + return true; + }), + ); + if (!claimed) return; + // Notifications before the RPC response also take the terminalization permit. + yield* startNativeTurn( + { + ...context.input, + providerTurnOrdinal: context.providerTurnOrdinal + 1, + }, + [], + ).pipe( + Effect.catch((cause) => + Effect.gen(function* () { + if (yield* Deferred.isDone(pending)) return; + yield* Effect.logWarning( + "orchestration-v2.codex-capacity-retry-start-failed", + { + providerTurnId: context.providerTurnId, + cause, + }, + ); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: codexRetryTurnItem( + context, + state, + "failed", + yield* DateTime.now, + ), + }); + yield* Deferred.succeed(pending, undefined); + yield* turnTerminalizationPermit.withPermits(1)( + emitOrDeferRootTerminal({ ...terminal, providerRetry: state }), + ); + }), + ), + Effect.ensuring( + Effect.sync(() => { + if (capacityContinuations.get(nativeThreadId)?.context === context) + capacityContinuations.delete(nativeThreadId); + }), + ), + ); + }), + ), + Effect.forkIn(scope, { startImmediately: true }), + ); + return true; + }), + ); + }); + yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); @@ -6119,7 +6310,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( (yield* Ref.get(interruptingNativeTurns)).has(payload.turn.id) ? "interrupted" : nativeStatus; - yield* finalizeCodexTurn({ + const terminal = { context, nativeTurnId: payload.turn.id, status, @@ -6132,7 +6323,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ? {} : { failureErrorInfo: payload.turn.error.codexErrorInfo }), }), - }); + }; + if (!(yield* retryCapacityFailure(terminal))) yield* finalizeCodexTurn(terminal); }), ); @@ -6246,10 +6438,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( goalStops.set(goalThreadId, (goalStops.get(goalThreadId) ?? 0) + 1); return yield* Effect.gen(function* () { // The run can settle while the pause is in flight, which drops its record. - const requestedTurnId = latestGoalTurnId(turnInput.providerTurnId); + const requestedTurnId = latestRunTurnId(turnInput.providerTurnId); const heldBefore = goalHolds.get(goalThreadId)?.context; const turnsBefore = [ - ...(goalRuns.get(requestedTurnId) ?? []), + ...(continuedRuns.get(requestedTurnId) ?? []), ...Array.from((yield* Ref.get(activeTurns)).values()).filter( (context) => context.providerTurnId === requestedTurnId, ), @@ -6273,13 +6465,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } return yield* turnTerminalizationPermit.withPermits(1)( Effect.gen(function* () { - const providerTurnId = latestGoalTurnId(turnInput.providerTurnId); + const providerTurnId = latestRunTurnId(turnInput.providerTurnId); const hold = goalHolds.get(goalThreadId); const held = hold !== undefined && hold.context.providerTurnId === providerTurnId; const goalTurns: ReadonlyArray = Array.from( new Set([ ...turnsBefore, - ...(goalRuns.get(providerTurnId) ?? []), + ...(continuedRuns.get(providerTurnId) ?? []), ...(held ? [hold.context] : []), ]), ); @@ -6335,6 +6527,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return next; }); const started = yield* client.request("turn/start", turnStartParams); + // A turn/started notification may have registered and completed this turn already. + if ((yield* Ref.get(pendingRootTurns)).get(threadId) !== turnInput) return; yield* registerRootTurn({ turnInput, nativeTurnId: started.turn.id, @@ -6347,6 +6541,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( Effect.ensuring( Effect.flatMap(getNativeThreadId(turnInput.providerThread), (threadId) => Ref.update(pendingRootTurns, (current) => { + if (current.get(threadId) !== turnInput) return current; const updated = new Map(current); updated.delete(threadId); return updated; @@ -6781,15 +6976,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( steerTurn: Effect.fn("CodexAdapterV2.steerTurn")( function* (turnInput) { const threadId = yield* getNativeThreadId(turnInput.providerThread); - const latestTurnId = latestGoalTurnId(turnInput.providerTurnId); + const latestTurnId = latestRunTurnId(turnInput.providerTurnId); const held = heldGoalTurn(turnInput.providerThread, latestTurnId); const providerTurnId = held === undefined ? latestTurnId : ((yield* Deferred.await(held.hold.next))?.providerTurnId ?? latestTurnId); - const activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( + let activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( (candidate) => candidate.providerTurnId === providerTurnId, ); + const startingRetry = capacityContinuations.get(threadId)?.context; + if (activeTurn === undefined && startingRetry?.providerTurnId === providerTurnId) { + activeTurn = startingRetry; + } + while (activeTurn?.capacityRetryPending !== undefined) { + activeTurn = yield* Deferred.await(activeTurn.capacityRetryPending); + } if (activeTurn === undefined) { return yield* toProtocolError( `Provider turn ${turnInput.providerTurnId} is not active and cannot be steered.`, @@ -6839,6 +7041,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), interruptTurn: Effect.fn("CodexAdapterV2.interruptTurn")( function* (requestedInput) { + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId; + const capacityRetry = + nativeThreadId == null + ? undefined + : capacityContinuations.get(nativeThreadId)?.context; + const startingRetry = + capacityRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId) + ? capacityRetry + : undefined; + if (startingRetry?.capacityRetryPending !== undefined) { + const started = yield* Deferred.await(startingRetry.capacityRetryPending).pipe( + Effect.timeoutOption("10 seconds"), + ); + if (Option.isNone(started)) + return yield* toProtocolError( + "Codex did not start the capacity retry within 10 seconds; Stop could not be delivered.", + ); + } const { turnInput, goalTurns } = yield* resolveGoalStopTarget(requestedInput); const [activeTurnContexts, settledTurnContexts] = yield* turnTerminalizationPermit.withPermits(1)( @@ -6859,7 +7079,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ) : undefined); const lineageRoots = Array.from( - new Set([...(activeTurn === undefined ? [] : [activeTurn]), ...goalTurns]), + new Set([ + ...(activeTurn === undefined ? [] : [activeTurn]), + ...(startingRetry === undefined ? [] : [startingRetry]), + ...goalTurns, + ]), ); const inLineage = (context: ActiveCodexTurnContext) => lineageRoots.some( @@ -7070,6 +7294,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* Effect.gen(function* () { for (const target of interruptTargets) { const context = target.context; + if (context.capacityRetryPending !== undefined) { + yield* finalizeCodexTurn({ + context, + nativeTurnId: context.nativeTurnId, + status: "interrupted", + completedAt: yield* DateTime.now, + }); + continue; + } // Null start timestamps ACK queued turns; interruption must wait for turn/started. if (context.nativeStartReady !== undefined) { const ready = yield* Deferred.await(context.nativeStartReady).pipe( @@ -7210,7 +7443,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ); }, (effect, requestedInput) => - effect.pipe( + Effect.acquireUseRelease( + Effect.sync(() => { + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId != null) + goalStops.set(nativeThreadId, (goalStops.get(nativeThreadId) ?? 0) + 1); + return nativeThreadId; + }), + () => effect, + (nativeThreadId) => + Effect.sync(() => { + if (nativeThreadId == null) return; + const stops = (goalStops.get(nativeThreadId) ?? 1) - 1; + if (stops > 0) goalStops.set(nativeThreadId, stops); + else goalStops.delete(nativeThreadId); + }), + ).pipe( Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterInterruptError({ diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 381cfdff6f4ed..59d7dabcacd65 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -47,6 +47,7 @@ import { type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderThread, type OrchestrationV2ProviderTurn, + type OrchestrationV2RemoteTaskChild, type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2ThreadShell, @@ -8198,17 +8199,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.stoppedThere === undefined ? task.remoteChild : { ...task.remoteChild!, stoppedThere: true }; - // The follower may report a result twice; the first one stands, and the - // repeat records nothing, which the dispatcher refuses. A stop there that - // reached the thread after the result is still recorded, but a run that - // ended by itself does not settle a stop the task here still owes. + // The first result stands; a repeated follower report records nothing, which + // the dispatcher refuses. A stop there after the result is still recorded, but + // a run that ended by itself does not settle a stop the task here still owes. if (task.result !== null) { + // A Stop or cancel here that lost the race to the result still owes its stop. + if (command.stoppedThere === undefined) { + return yield* emitRemoteStopOwed(events, command, task, task.remoteChild!); + } const owesStop = task.status === "cancelled" || task.status === "interrupted" || task.remoteChild?.stoppedThere === false; if ( - command.stoppedThere === undefined || task.remoteChild?.stoppedThere === true || (owesStop && command.stoppedThere !== "stopped") ) @@ -8350,6 +8353,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Delegated task ${command.taskId} is not a finished remote task of thread ${command.parentThreadId}.`, }); } + yield* emitRemoteStopOwed(events, command, task, task.remoteChild); + }); + + const emitRemoteStopOwed = Effect.fn("orchestrationV2.dispatch.remoteStopOwed")(function* ( + events: Ref.Ref>, + command: Extract< + OrchestrationV2InternalCommand, + { readonly type: "delegated_task.remote.complete" | "delegated_task.remote.stop" } + >, + task: OrchestrationV2Subagent, + remoteChild: OrchestrationV2RemoteTaskChild, + ) { const now = yield* DateTime.now; yield* emit( events, @@ -8364,9 +8379,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: { ...task, remoteChild: { - ...task.remoteChild, + ...remoteChild, stoppedThere: false, - stopCount: (task.remoteChild.stopCount ?? 0) + 1, + stopCount: (remoteChild.stopCount ?? 0) + 1, }, updatedAt: now, }, diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index 6d2fcfbd19810..4f242b01db61e 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -135,7 +135,11 @@ export const layer: Layer.Layer< const interruptProviderThread = targetsRecordedSession ? providerThread : { ...providerThread, providerSessionId: input.providerSessionId }; - if (providerTurn.status !== "running") { + const continuingRun = + input.operation === "steer" && + context.run?.status === "running" && + context.run.activeAttemptId === providerTurn.runAttemptId; + if (providerTurn.status !== "running" && !continuingRun) { if (input.operation === "steer") { return yield* new ProviderTurnControlError({ threadId: input.threadId, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 926875e478a4a..7d9b8beef8f26 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -12,6 +12,7 @@ import { ProjectKnowledgeError, latestProviderTurnForAttempt, type OrchestrationV2DomainEvent, + type OrchestrationV2ContextHandoff, type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderThread, type OrchestrationV2Run, @@ -59,6 +60,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import { + ContextHandoffBackgroundWorkError, knownProviderFailureMessage, makeProviderFailure, } from "@t3tools/provider-core/server/failure"; @@ -767,6 +769,287 @@ export const layer: Layer.Layer< }); let effectiveHandoffs = handoffs; let resumeFellBack = false; + let userText = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + // Codex and Claude parse /goal from the prompt, so it goes verbatim and history waits a turn. + const goalDriver = session.driver === "codex" || session.driver === "claudeAgent"; + const goalCommand = goalDriver && parseGoalCommand(message) !== null; + // The provider turn marks cancellation work delivered; restart continuations use their own text or resume natively. + const noteContinuation = isRestartNoteContinuation( + run, + projection.runs, + projection.providerTurns, + projection.attempts, + ); + const restartCancelledWork = pendingRestartCancelledBackgroundWork({ + runs: projection.runs, + providerTurns: projection.providerTurns, + compactionMessageIds: new Set( + projection.messages + .filter( + (candidate) => + isCompactionCommandMessage(candidate) || + (goalDriver && parseGoalCommand(candidate) !== null), + ) + .map((candidate) => candidate.id), + ), + run, + attempts: projection.attempts, + }); + const restartNote = + restartCancelledWork.length === 0 + ? "" + : restartCancelledBackgroundWorkNote(restartCancelledWork); + const tokenCap = yield* handoffTokenCapConfig.pipe( + Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), + ); + const handoffContextForThread = (loadedProviderThread: OrchestrationV2ProviderThread) => { + const measuredContext = latestNativeContextUsage(projection, providerThread); + const previousSelection = + measuredContext?.modelSelection ?? + projection.runs.findLast( + (source) => + source.ordinal < run.ordinal && + source.startedAt !== null && + source.providerThreadId === providerThread.id, + )?.modelSelection; + const sameSelection = + previousSelection === undefined || + modelSelectionsEqual(previousSelection, run.modelSelection); + const sameNativeThread = + loadedProviderThread.nativeThreadRef?.nativeId === + providerThread.nativeThreadRef?.nativeId; + const threadUsage = loadedProviderThread.contextUsage ?? providerThread.contextUsage; + const previousUsage = measuredContext + ? { ...threadUsage, ...measuredContext.usage } + : threadUsage; + const reuseTelemetry = + sameSelection || + (previousSelection !== undefined && + session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); + const knownModelWindow = session.getModelContextWindow?.( + run.modelSelection, + resolvedRuntimePolicy.cwd, + ); + const handoffUsage = contextUsageForHandoff({ + sameNativeThread, + sameSelection, + reuseTelemetry, + previousUsage, + knownModelWindow, + }); + const settledHandoffs = projection.contextHandoffs.filter( + (handoff) => + handoff.toProviderThreadId === providerThread.id && + handoff.delivery?.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId && + handoff.delivery?.status !== "pending", + ); + const deliveredItemIds = new Set( + settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), + ); + const summarizedItemIds = new Set( + settledHandoffs.flatMap((handoff) => handoff.delivery?.summarizedItemIds ?? []), + ); + const coveredItemIds = new Set([ + ...summarizedItemIds, + ...deliveredItemIds, + ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), + ]); + const deliveredAttemptIds = new Set( + projection.providerTurns.map((turn) => turn.runAttemptId), + ); + const acceptedAttempts = projection.attempts.filter( + (source) => + source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), + ); + const nativeInputRunIds = new Set( + acceptedAttempts + .filter( + (source) => + source.nativeThreadId !== undefined && + source.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId, + ) + .map((source) => source.runId), + ); + const legacyInputRunIds = new Set( + acceptedAttempts + .filter((source) => source.nativeThreadId === undefined) + .map((source) => source.runId), + ); + const legacyRecoveredRunIds = new Set( + projection.runs + .filter( + (source) => + source.providerThreadId === providerThread.id && + settledHandoffs.some( + (handoff) => + handoff.strategy === "full_thread_summary" && + handoff.fromProviderThreadIds.includes(providerThread.id) && + source.ordinal >= handoff.coveredRunOrdinals.from && + source.ordinal <= handoff.coveredRunOrdinals.to, + ), + ) + .map((source) => source.id), + ); + // Row identity survives recovery; only accepted native input and delivered history occupy the new session. + const nativeContextEstimate = Effect.gen(function* () { + return sameNativeThread + ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { + const acceptedNatively = + item.providerThreadId === providerThread.id && + item.runId !== null && + (nativeInputRunIds.has(item.runId) || + (legacyInputRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + !legacyRecoveredRunIds.has(item.runId))); + if (item.runId === run.id || (!acceptedNatively && !deliveredItemIds.has(item.id))) + return sum; + const historical = historicalMessage(item); + const nativeAttachments = + item.type === "user_message" && acceptedNatively + ? attachmentTokenAllowance(item.attachments) + : 0; + return ( + sum + + (historical === null ? 0 : Buffer.byteLength(historical.text)) + + nativeAttachments + ); + }, 0) + : 0; + }); + const modelContextWindow = + knownModelWindow ?? + (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined); + const budgetProviderThread = { + ...loadedProviderThread, + contextUsage: handoffUsage, + }; + // The resume fallback summary already covers every earlier run in order. A + // retry copy after it would rank older missed requests as the newest history. + const missedRuns = projection.runs.filter( + (source) => + !resumeFellBack && + source.ordinal < run.ordinal && + source.providerThreadId === providerThread.id && + (source.status === "failed" || source.status === "interrupted") && + !deliveredAttemptIds.has(source.activeAttemptId), + ); + const missedRunIds = new Set(missedRuns.map((source) => source.id)); + return { + sameNativeThread, + handoffUsage, + deliveredItemIds, + coveredItemIds, + acceptedAttempts, + nativeInputRunIds, + legacyInputRunIds, + summarizedItemIds, + modelContextWindow, + budgetProviderThread, + nativeContextEstimate, + missedRuns, + missedRunIds, + budget: Effect.gen(function* () { + return handoffBudget({ + tokenCap, + modelContextWindow, + userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + attachments: message.attachments, + providerThread: budgetProviderThread, + nativeContextEstimate: + budgetProviderThread.contextUsage?.usedTokens === undefined + ? yield* nativeContextEstimate + : 0, + }); + }), + }; + }; + const prepareRetryHandoff = Effect.fn("ProviderTurnStartService.prepareRetryHandoff")( + function* (context: ReturnType) { + const { missedRunIds, coveredItemIds, missedRuns } = context; + const missedItems = + missedRunIds.size === 0 + ? [] + : (yield* projectionStore.getTurnStartHistory(input.threadId, [ + ...missedRunIds, + ])).filter( + (item) => + item.runId !== null && + missedRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + historicalMessage(item) !== null, + ); + const retryHandoff = + missedItems.length === 0 + ? [] + : [ + yield* contextHandoffService.prepareProviderHandoff({ + threadId: projection.thread.id, + targetRunId: run.id, + transferId: null, + fromProviderThreadIds: [providerThread.id], + toProviderThreadId: providerThread.id, + fromProviderInstanceId: run.providerInstanceId, + toProviderInstanceId: run.providerInstanceId, + coveredRunOrdinals: { + from: missedRuns[0]!.ordinal, + to: missedRuns.at(-1)!.ordinal, + }, + strategy: "delta_since_target_last_seen", + items: missedItems, + runs: projection.runs, + createdAt: yield* DateTime.now, + }), + ]; + return retryHandoff; + }, + ); + const prepareRecoveryHandoff = Effect.fn("ProviderTurnStartService.prepareRecoveryHandoff")( + function* () { + const transferId = yield* idAllocator.allocate.contextTransfer({ + sourceThreadId: projection.thread.id, + targetThreadId: projection.thread.id, + type: "provider_resume_fallback", + }); + const createdAt = yield* DateTime.now; + const handoff = yield* contextHandoffService.prepareProviderHandoff({ + threadId: projection.thread.id, + targetRunId: run.id, + transferId, + fromProviderThreadIds: [providerThread.id], + toProviderThreadId: providerThread.id, + fromProviderInstanceId: providerThread.providerInstanceId, + toProviderInstanceId: run.providerInstanceId, + coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + strategy: "full_thread_summary", + runs: projection.runs, + items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( + (item) => + item.runId === null || + projection.runs.some( + (source) => source.id === item.runId && source.ordinal < run.ordinal, + ), + ), + createdAt, + }); + return { handoff, transferId, createdAt }; + }, + ); + const recoveryHandoff = yield* Effect.cached(prepareRecoveryHandoff()); + const withRecoveryHandoff = (handoff: OrchestrationV2ContextHandoff) => { + // Delivery ranks later handoffs as newer history. Pending handoffs from + // earlier runs predate this summary; handoffs for this run (provider-switch + // delta, merge-back) are newer and must stay last. + return [ + ...effectiveHandoffs.filter((candidate) => candidate.targetRunId !== run.id), + handoff, + ...effectiveHandoffs.filter((candidate) => candidate.targetRunId === run.id), + ]; + }; + let resumedRetryHandoff: ReadonlyArray | undefined; + let resumedHandoffContext: ReturnType | undefined; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( @@ -870,15 +1153,71 @@ export const layer: Layer.Layer< }), ); if (resumed._tag === "Success") { - return resumed.success; + const context = handoffContextForThread(resumed.success); + context.budget = yield* Effect.cached(context.budget); + resumedHandoffContext = context; + resumedRetryHandoff = yield* prepareRetryHandoff(context); + const fits = yield* deliverContextHandoffs({ + handoffs: [...handoffs, ...resumedRetryHandoff], + providerThread: { ...resumed.success, id: providerThread.id }, + budget: context.budget, + deferInline: isCompactionCommandMessage(message) || goalCommand, + alreadyDeliveredItemIds: context.deliveredItemIds, + persist: () => Effect.void, + }).pipe( + Effect.as(true), + Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), + ); + if (fits) return resumed.success; + const freshThread = { ...resumed.success, nativeThreadRef: null }; + const freshContext = handoffContextForThread(freshThread); + const { handoff } = yield* recoveryHandoff; + const fitsFresh = yield* deliverContextHandoffs({ + handoffs: withRecoveryHandoff(handoff), + providerThread: freshThread, + budget: freshContext.budget, + alreadyDeliveredItemIds: freshContext.deliveredItemIds, + persist: () => Effect.void, + }).pipe( + Effect.as(true), + Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), + ); + if (!fitsFresh) return resumed.success; + const hasPendingWork = yield* ( + session.hasPendingBackgroundWorkForThread?.(resumed.success, { + includeSubagents: true, + }) ?? + session.hasPendingBackgroundWork ?? + Effect.succeed(false) + ); + if (hasPendingWork) { + return yield* loadFromProvider( + Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: session.driver, + threadId: projection.thread.id, + providerThreadId: providerThread.id, + runId, + cause: new ContextHandoffBackgroundWorkError(), + }), + ), + ); + } } + resumedHandoffContext = undefined; + resumedRetryHandoff = []; + if (!(yield* isCurrentAttemptInStatus("starting"))) return undefined; - yield* Effect.logWarning("Provider resume failed; attempting a fresh native session", { + yield* Effect.logWarning("Recovering provider context in a fresh native session", { driver: session.driver, providerThreadId: providerThread.id, runId, - reason: uncertainDelivery ? "uncertain_history_delivery" : "resume_failed", - errorTag: resumed.failure._tag, + reason: uncertainDelivery + ? "uncertain_history_delivery" + : resumed._tag === "Success" + ? "handoff_budget_exhausted" + : "resume_failed", + errorTag: resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", }); const replacement = yield* loadFromProvider( session.ensureThread({ @@ -889,43 +1228,12 @@ export const layer: Layer.Layer< providerSessionId, // Clear the native ref to force a fresh session while preserving row identity. existingProviderThread: { ...providerThread, nativeThreadRef: null }, + forceNewNativeThread: true, }), ); if (replacement === undefined) return undefined; - const transferId = yield* idAllocator.allocate.contextTransfer({ - sourceThreadId: projection.thread.id, - targetThreadId: projection.thread.id, - type: "provider_resume_fallback", - }); - const createdAt = yield* DateTime.now; - const handoff = yield* contextHandoffService.prepareProviderHandoff({ - threadId: projection.thread.id, - targetRunId: run.id, - transferId, - fromProviderThreadIds: [providerThread.id], - toProviderThreadId: providerThread.id, - fromProviderInstanceId: providerThread.providerInstanceId, - toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, - strategy: "full_thread_summary", - runs: projection.runs, - items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter( - (item) => - item.runId === null || - projection.runs.some( - (source) => source.id === item.runId && source.ordinal < run.ordinal, - ), - ), - createdAt, - }); - // Delivery ranks later handoffs as newer history. Pending handoffs from - // earlier runs predate this summary; handoffs for this run (provider-switch - // delta, merge-back) are newer and must stay last. - effectiveHandoffs = [ - ...effectiveHandoffs.filter((candidate) => candidate.targetRunId !== run.id), - handoff, - ...effectiveHandoffs.filter((candidate) => candidate.targetRunId === run.id), - ]; + const { handoff, transferId, createdAt } = yield* recoveryHandoff; + effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; yield* eventSink.write({ events: [ @@ -973,39 +1281,20 @@ export const layer: Layer.Layer< return; } const now = yield* DateTime.now; - const measuredContext = latestNativeContextUsage(projection, providerThread); - const previousSelection = - measuredContext?.modelSelection ?? - projection.runs.findLast( - (source) => - source.ordinal < run.ordinal && - source.startedAt !== null && - source.providerThreadId === providerThread.id, - )?.modelSelection; - const sameSelection = - previousSelection === undefined || - modelSelectionsEqual(previousSelection, run.modelSelection); - const sameNativeThread = - loadedProviderThread.nativeThreadRef?.nativeId === providerThread.nativeThreadRef?.nativeId; - const threadUsage = loadedProviderThread.contextUsage ?? providerThread.contextUsage; - const previousUsage = measuredContext - ? { ...threadUsage, ...measuredContext.usage } - : threadUsage; - const reuseTelemetry = - sameSelection || - (previousSelection !== undefined && - session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); - const knownModelWindow = session.getModelContextWindow?.( - run.modelSelection, - resolvedRuntimePolicy.cwd, - ); - const handoffUsage = contextUsageForHandoff({ + const handoffContext = resumedHandoffContext ?? handoffContextForThread(loadedProviderThread); + const { sameNativeThread, - sameSelection, - reuseTelemetry, - previousUsage, - knownModelWindow, - }); + handoffUsage, + deliveredItemIds, + acceptedAttempts, + nativeInputRunIds, + legacyInputRunIds, + summarizedItemIds, + modelContextWindow, + budgetProviderThread, + nativeContextEstimate, + } = handoffContext; + const retryHandoff = resumedRetryHandoff ?? (yield* prepareRetryHandoff(handoffContext)); const runningProviderThread: OrchestrationV2ProviderThread = { ...loadedProviderThread, contextUsage: handoffUsage, @@ -1203,7 +1492,7 @@ export const layer: Layer.Layer< receipt.nativeThreadId != null && receipt.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId, ); - const userText = alreadySent ? composer.baseText : composer.text; + userText = alreadySent ? composer.baseText : composer.text; // Keyed by attempt so a retried start updates the same record, while a replacement // attempt (account failover) keeps its own account of what its provider received. const recordKnowledgeDelivery = Effect.fn("ProviderTurnStartService.recordKnowledgeDelivery")( @@ -1238,151 +1527,6 @@ export const layer: Layer.Layer< }), ), ); - // Codex and Claude parse /goal from the prompt, so it goes verbatim and history waits a turn. - const goalDriver = session.driver === "codex" || session.driver === "claudeAgent"; - const goalCommand = goalDriver && parseGoalCommand(message) !== null; - // The provider turn marks cancellation work delivered; restart continuations use their own text or resume natively. - const noteContinuation = isRestartNoteContinuation( - run, - projection.runs, - projection.providerTurns, - projection.attempts, - ); - const restartCancelledWork = pendingRestartCancelledBackgroundWork({ - runs: projection.runs, - providerTurns: projection.providerTurns, - compactionMessageIds: new Set( - projection.messages - .filter( - (candidate) => - isCompactionCommandMessage(candidate) || - (goalDriver && parseGoalCommand(candidate) !== null), - ) - .map((candidate) => candidate.id), - ), - run, - attempts: projection.attempts, - }); - const restartNote = - restartCancelledWork.length === 0 - ? "" - : restartCancelledBackgroundWorkNote(restartCancelledWork); - const tokenCap = yield* handoffTokenCapConfig.pipe( - Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), - ); - const settledHandoffs = projection.contextHandoffs.filter( - (handoff) => - handoff.toProviderThreadId === providerThread.id && - handoff.delivery?.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId && - handoff.delivery?.status !== "pending", - ); - const deliveredItemIds = new Set( - settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), - ); - // Represented in the native thread by a delivered summary rather than verbatim text. - const summarizedItemIds = new Set( - settledHandoffs.flatMap((handoff) => handoff.delivery?.summarizedItemIds ?? []), - ); - const coveredItemIds = new Set([ - ...deliveredItemIds, - ...summarizedItemIds, - ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), - ]); - const deliveredAttemptIds = new Set( - projection.providerTurns.map((turn) => turn.runAttemptId), - ); - const acceptedAttempts = projection.attempts.filter( - (source) => - source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), - ); - const nativeInputRunIds = new Set( - acceptedAttempts - .filter( - (source) => - source.nativeThreadId !== undefined && - source.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId, - ) - .map((source) => source.runId), - ); - const legacyInputRunIds = new Set( - acceptedAttempts - .filter((source) => source.nativeThreadId === undefined) - .map((source) => source.runId), - ); - const legacyRecoveredRunIds = new Set( - projection.runs - .filter( - (source) => - source.providerThreadId === providerThread.id && - settledHandoffs.some( - (handoff) => - handoff.strategy === "full_thread_summary" && - handoff.fromProviderThreadIds.includes(providerThread.id) && - source.ordinal >= handoff.coveredRunOrdinals.from && - source.ordinal <= handoff.coveredRunOrdinals.to, - ), - ) - .map((source) => source.id), - ); - // Legacy attempts lack native ids; exclude recovered attachments absent from the replacement thread. - const nativeContextEstimate = Effect.gen(function* () { - return sameNativeThread - ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { - if ( - item.runId === run.id || - (item.runId !== null && - missedRunIds.has(item.runId) && - !deliveredItemIds.has(item.id)) || - (item.providerThreadId !== providerThread.id && !deliveredItemIds.has(item.id)) - ) - return sum; - const historical = historicalMessage(item); - const nativeAttachments = - item.type === "user_message" && - item.providerThreadId === providerThread.id && - item.runId !== null && - (nativeInputRunIds.has(item.runId) || - (legacyInputRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - !legacyRecoveredRunIds.has(item.runId))) - ? attachmentTokenAllowance(item.attachments) - : 0; - return ( - sum + - (historical === null ? 0 : Buffer.byteLength(historical.text)) + - nativeAttachments - ); - }, 0) - : 0; - }); - const modelContextWindow = - knownModelWindow ?? - (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined); - const budgetProviderThread = { - ...runningProviderThread, - contextUsage: handoffUsage, - }; - // The resume fallback summary already covers every earlier run in order. A - // retry copy after it would rank older missed requests as the newest history. - const missedRuns = projection.runs.filter( - (source) => - !resumeFellBack && - source.ordinal < run.ordinal && - source.providerThreadId === providerThread.id && - (source.status === "failed" || source.status === "interrupted") && - !deliveredAttemptIds.has(source.activeAttemptId), - ); - const missedRunIds = new Set(missedRuns.map((source) => source.id)); - const missedItems = - missedRunIds.size === 0 - ? [] - : (yield* projectionStore.getTurnStartHistory(input.threadId, [...missedRunIds])).filter( - (item) => - item.runId !== null && - missedRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - historicalMessage(item) !== null, - ); { knowledgeBudget = handoffBudgetAssumptions({ tokenCap, @@ -1444,28 +1588,6 @@ export const layer: Layer.Layer< cause: "Uncertain native history injection", }); } - const retryHandoff = - missedItems.length === 0 - ? [] - : [ - yield* contextHandoffService.prepareProviderHandoff({ - threadId: projection.thread.id, - targetRunId: run.id, - transferId: null, - fromProviderThreadIds: [providerThread.id], - toProviderThreadId: providerThread.id, - fromProviderInstanceId: run.providerInstanceId, - toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { - from: missedRuns[0]!.ordinal, - to: missedRuns.at(-1)!.ordinal, - }, - strategy: "delta_since_target_last_seen", - items: missedItems, - runs: projection.runs, - createdAt: yield* DateTime.now, - }), - ]; const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact || goalCommand, @@ -1621,7 +1743,7 @@ export const layer: Layer.Layer< ); const deliverySession = effectiveHandoffs.length === 0 && - missedItems.length === 0 && + retryHandoff.length === 0 && restartNote === "" && !noteContinuation && composer.byteLength === 0 && diff --git a/apps/server/src/orchestration-v2/RemoteDelegatedTask.test.ts b/apps/server/src/orchestration-v2/RemoteDelegatedTask.test.ts index d4f5232c58347..4bb8c103f6df7 100644 --- a/apps/server/src/orchestration-v2/RemoteDelegatedTask.test.ts +++ b/apps/server/src/orchestration-v2/RemoteDelegatedTask.test.ts @@ -292,6 +292,56 @@ it.layer(layerTest)("a delegated task in a linked environment", (it) => { }), ); + it.effect("owes a stop there for a Stop here that loses the race to its result", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const parent = yield* seedLiveParent("stop-after-result"); + yield* orchestrator.dispatch({ + type: "delegated_task.remote.request", + commandId: CommandId.make("command:remote-delegate:stop-after-result"), + parentThreadId: parent.threadId, + parentRunId: parent.runId, + parentNodeId: parent.rootNodeId, + task: "Run a check.", + driver: ProviderDriverKind.make("remote"), + modelSelection, + remoteChild, + }); + const task = (yield* orchestrator.getThreadProjection(parent.threadId)).subagents[0]!; + const complete = { + type: "delegated_task.remote.complete" as const, + parentThreadId: parent.threadId, + taskId: task.id, + }; + yield* orchestrator.dispatch({ + ...complete, + commandId: CommandId.make("command:result:stop-after-result"), + status: "completed", + result: "Done there.", + stoppedThere: "ran-out", + }); + // The parent's Stop read the task before its result landed. + yield* orchestrator.dispatch({ + ...complete, + commandId: CommandId.make("command:parent-stop:stop-after-result"), + status: "interrupted", + result: "Stopped with its parent.", + }); + + const stopped = (yield* orchestrator.getThreadProjection(parent.threadId)).subagents[0]!; + assert.equal(stopped.status, "completed"); + assert.equal(stopped.result, "Done there."); + assert.equal(stopped.remoteChild?.stoppedThere, false); + assert.equal(stopped.remoteChild?.stopCount, 1); + const open = yield* projections.getOpenRemoteDelegatedTasks; + assert.deepEqual( + open.filter((entry) => entry.parentThreadId === parent.threadId), + [{ parentThreadId: parent.threadId, taskId: task.id }], + ); + }), + ); + it.effect("wakes its parent with the result when it completes there", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -330,6 +380,7 @@ it.layer(layerTest)("a delegated task in a linked environment", (it) => { taskId: taskId!, status: "completed", result, + stoppedThere: "ran-out", }); yield* complete("All 412 tests pass on the box."); // A second report is refused, so the first result stands. diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 4c2d6712c5ad7..7481dcaa41b2c 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -42,6 +42,7 @@ it.effect.each( [true, "without native steering"], [true, "with interrupting native steering"], [true, "settled only"], + [false, "during continuation"], ] as const ).map(([mailbox, timing]) => ({ mailbox, @@ -135,7 +136,7 @@ it.effect.each( steerTurn: (turn) => Effect.gen(function* () { steerCalls += 1; - if (timing === "after delivery") return; + if (timing === "after delivery" || timing === "during continuation") return; yield* Deferred.succeed(steerEntered, undefined); yield* Deferred.await(rejectSteer); return yield* new ProviderAdapter.ProviderAdapterSteerRunError({ @@ -284,6 +285,29 @@ it.effect.each( : {}), }); if (timing !== "before dispatch") yield* dispatchSteer; + if (timing === "during continuation") { + const projection = yield* orchestrator.getThreadProjection(threadId); + const turn = projection.providerTurns[0]!; + const failed = yield* watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "failed", + ); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now }, + }); + yield* Fiber.join(failed); + yield* worker.drain(); + assert.equal(steerCalls, 1); + assert.lengthOf(started, 1); + const continued = yield* orchestrator.getThreadProjection(threadId); + assert.equal(continued.runs[0]?.status, "running"); + assert.equal( + continued.messages.find((message) => message.id === messageId)?.runId, + first.runId, + ); + return; + } if (timing === "after delivery") yield* worker.drain(); const delivery = timing === "during delivery" ? yield* worker.runOnce.pipe(Effect.forkScoped) : null; diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 840c081539971..97e403981dd51 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -1002,7 +1002,10 @@ const make = Effect.gen(function* () { creationSource: input.creationSource, }) .pipe( - // Agent retries retain their requested mode limits if the thread now has broader modes. + // An agent's launch can land on a thread that a replay, or a concurrent + // launch with the same key, created with broader modes. The orchestrator + // refuses the message under the thread's lock; the user's own clients + // launch into their own drafts, whatever modes those hold. input.createdBy === "agent" ? Effect.provideService(DispatchModeLimit, { runtimeMode: input.runtimeMode, diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index 010ea5a8a9e8a..dfd244ebe2f65 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -18,12 +18,14 @@ import { } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Option from "effect/Option"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -385,6 +387,63 @@ it.effect("waitForThread can wait past an ended run for the thread's delegated w }), ); +it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () => + Effect.gen(function* () { + const projectId = ProjectId.make("project:thread-management:wait-rolled-back"); + const threadId = ThreadId.make("thread:thread-management:wait-rolled-back"); + const runId = RunId.make("run:thread-management:wait-rolled-back"); + const rolledBack = { + id: runId, + ordinal: 1, + status: "rolled_back", + startedAt: DateTime.makeUnsafe(0), + completedAt: DateTime.makeUnsafe(1_000), + } as unknown as OrchestrationV2Run; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadEventSequence: () => Effect.succeed(0), + restartContinuationPending: () => Effect.succeed(false), + delegatedTaskResultPending: () => Effect.succeed(false), + streamStoredEventsFrom: () => Stream.never, + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, projectId, deletedAt: null }, + runs: [rolledBack], + messages: [], + subagents: [], + providerThreads: [], + } as unknown as OrchestrationV2ThreadProjection), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const service = yield* ThreadManagementService.ThreadManagementService.pipe( + Effect.provide(layerTest), + ); + const fiber = yield* service + .waitForThread({ + projectId, + threadId, + runId, + timeoutMs: 60_000, + untilDelegatedWorkSettled: true, + }) + .pipe(Effect.forkChild); + // Nothing will ever run there again, so a wait still open runs out here. + yield* TestClock.adjust(Duration.millis(60_000)); + const result = yield* Fiber.join(fiber); + + expect(result).toMatchObject({ + timedOut: false, + run: { id: runId, status: "rolled_back" }, + resultRun: { id: runId, status: "rolled_back" }, + }); + }), +); + it.effect("waitForThread returns the cut run once its continuation is declined", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-declined"); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 58d0c57f40b0f..aeeaf2ed82229 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -919,6 +919,13 @@ const make = Effect.gen(function* () { { messageRoles: ["user"] }, ); const progress = delegatedTaskProgress(records); + // A thread rolled back past all its work, with nothing left running, has no answer to wait for. + if ( + progress.resultRun === undefined && + records.runs.every((run) => isTerminalRunStatus(run.status)) + ) { + return latestRun(records); + } if (progress.state !== "result_available") return undefined; const pending = yield* orchestrator .delegatedTaskResultPending(input.threadId) diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 0fd201750464c..bb21d8ca9929a 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -115,6 +115,8 @@ function makeTestAdapter(input: { readonly failedRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; + readonly pendingBackgroundWork?: Ref.Ref; + readonly subagentOnlyBackgroundWork?: boolean; readonly holdFirstTurn?: Deferred.Deferred; readonly releaseFirstTurn?: Deferred.Deferred; }): ProviderAdapter.ProviderAdapterV2["Service"] { @@ -146,6 +148,14 @@ function makeTestAdapter(input: { providerSessionId: sessionInput.providerSessionId, providerSession, events: Stream.fromPubSub(events), + ...(input.pendingBackgroundWork === undefined + ? {} + : { + hasPendingBackgroundWorkForThread: (_thread, options) => + input.subagentOnlyBackgroundWork && options?.includeSubagents !== true + ? Effect.succeed(false) + : Ref.get(input.pendingBackgroundWork!), + }), ...(input.getModelContextWindow === undefined ? {} : { getModelContextWindow: input.getModelContextWindow }), @@ -419,6 +429,13 @@ describe("orchestration v2 provider switching", () => { "screenshot-fallback", "screenshot-reported-capacity-native", "screenshot-option-change-retry-native", + "screenshot-exhausted-target-native", + "screenshot-exhausted-target-fallback", + "screenshot-exhausted-target-estimate-native", + "screenshot-exhausted-target-oversized-native", + "screenshot-exhausted-target-tight-native", + "screenshot-exhausted-target-background-native", + "screenshot-exhausted-target-subagent-native", "prior-images-reasoning-change-replacement-small-native", "imported-prior-images-fallback", "prior-images-telemetry-native", @@ -438,11 +455,16 @@ describe("orchestration v2 provider switching", () => { const failStartOnce = yield* Ref.make(false); const failResumeOnce = yield* Ref.make(false); const generation = yield* Ref.make(0); + const pendingBackgroundWork = yield* Ref.make(false); const priorImages = scenario.includes("prior-images"); + const exhaustedTarget = scenario.includes("exhausted-target"); const replaceNative = scenario.includes("replacement"); const capacityScenario = modelScenario || scenario.includes("reported-capacity"); const returning = - scenario === "large-current-input" || scenario.includes("-change-") || priorImages; + scenario === "large-current-input" || + scenario.includes("-change-") || + priorImages || + exhaustedTarget; const targetSelection: ModelSelection = !modelScenario ? CLAUDE_MODEL_SELECTION : { @@ -467,10 +489,19 @@ describe("orchestration v2 provider switching", () => { driver: CLAUDE_DRIVER, capabilities: ClaudeProviderCapabilitiesV2, modelSelection: CLAUDE_MODEL_SELECTION, - responseByRunOrdinal: {}, + responseByRunOrdinal: scenario.includes("target-estimate") + ? { 2: "x".repeat(10_000) } + : {}, capturedTurns, ...(scenario.includes("retry") || scenario.includes("unsent") ? { failStartOnce } : {}), ...(replaceNative ? { failResumeOnce, nativeThreadGeneration: generation } : {}), + ...(exhaustedTarget ? { nativeThreadGeneration: generation } : {}), + ...(scenario.includes("target-background") || scenario.includes("target-subagent") + ? { + pendingBackgroundWork, + subagentOnlyBackgroundWork: scenario.includes("target-subagent"), + } + : {}), ...(scenario.includes("reported-capacity") ? { initialContextUsage: { usedTokens: 999_999, maxTokens: 1_000_000 } } : {}), @@ -498,8 +529,11 @@ describe("orchestration v2 provider switching", () => { ? 1_000_000 : 32_000, } - : scenario === "large-current-input" || priorImages - ? { getModelContextWindow: () => 32_000 } + : scenario === "large-current-input" || priorImages || exhaustedTarget + ? { + getModelContextWindow: () => + scenario.includes("target-tight") ? 25_121 : 32_000, + } : {}), ...(scenario.endsWith("-native") || scenario === "large-current-input" ? { injectedHistory } @@ -546,7 +580,9 @@ describe("orchestration v2 provider switching", () => { })) : turnUsageScenario && ordinal >= targetOrdinal ? [screenshot, { ...screenshot, id: "current-2" }] - : scenario.startsWith("screenshot") && ordinal >= targetOrdinal + : scenario.startsWith("screenshot") && + ordinal >= targetOrdinal && + (!exhaustedTarget || ordinal === targetOrdinal) ? screenshots : priorImages && ordinal === (scenario.startsWith("imported") ? 1 : 2) ? [screenshot] @@ -602,19 +638,49 @@ describe("orchestration v2 provider switching", () => { yield* Effect.addFinalizer(() => Effect.sync(() => spy?.mockRestore())); const current = scenario.startsWith("compact") ? "/compact" - : capacityScenario && !turnUsageScenario - ? "x".repeat(70_000) - : scenario === "large-current-input" - ? "x".repeat(9_000) - : "Continue work"; + : scenario.includes("target-oversized") + ? "x".repeat(40_000) + : capacityScenario && !turnUsageScenario + ? "x".repeat(70_000) + : scenario === "large-current-input" + ? "x".repeat(9_000) + : "Continue work"; if (returning) { if (scenario.includes("unsent")) yield* Ref.set(failStartOnce, true); yield* dispatch( 2, - turnUsageScenario ? "x".repeat(27_460) : "Establish target", + turnUsageScenario + ? "x".repeat(27_460) + : scenario.includes("target-estimate") + ? "x".repeat(12_000) + : "Establish target", CLAUDE_MODEL_SELECTION, ); yield* wait(2); + if (exhaustedTarget) { + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + } + if (exhaustedTarget && !scenario.includes("target-estimate")) { + const target = (yield* orchestrator.getThreadProjection( + threadId, + )).providerThreads.find( + (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + )!; + yield* eventSink.write({ + events: [ + { + id: EventId.make("exhausted-target-usage"), + type: "provider-thread.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...target, contextUsage: { usedTokens: 30_000, maxTokens: 32_000 } }, + }, + ], + }); + } if (modelScenario && !turnUsageScenario) { const target = (yield* orchestrator.getThreadProjection( threadId, @@ -695,6 +761,8 @@ describe("orchestration v2 provider switching", () => { yield* Ref.set(failResumeOnce, true); } if (scenario.includes("retry")) yield* Ref.set(failStartOnce, true); + if (scenario.includes("target-background") || scenario.includes("target-subagent")) + yield* Ref.set(pendingBackgroundWork, true); yield* dispatch(targetOrdinal, current, targetSelection); yield* wait(targetOrdinal); if (scenario.includes("retry")) { @@ -780,7 +848,73 @@ describe("orchestration v2 provider switching", () => { yield* wait(targetOrdinal + 3); } const projection = yield* orchestrator.getThreadProjection(threadId); + if ( + scenario.includes("target-oversized") || + scenario.includes("target-tight") || + scenario.includes("target-background") || + scenario.includes("target-subagent") + ) { + assert.equal(projection.runs.at(-1)?.status, "failed"); + assert.equal(yield* Ref.get(generation), 1); + assert.equal(projection.messages.at(-1)!.text, current); + assert.deepEqual(projection.messages.at(-1)!.attachments, screenshots); + if (scenario.includes("target-background") || scenario.includes("target-subagent")) { + assert.isTrue(yield* Ref.get(pendingBackgroundWork)); + const failure = projection.turnItems.findLast((item) => item.type === "error"); + assert.equal( + failure?.type === "error" ? failure.failure.message : null, + "The target provider still has background work running. Wait for it to finish or stop it before retrying the handoff.", + ); + } + return; + } assert.equal(projection.runs.at(-1)?.status, "completed"); + if (exhaustedTarget) { + assert.equal(yield* Ref.get(generation), 2); + const turns = (yield* Ref.get(capturedTurns)).filter( + (turn) => turn.driver === CLAUDE_DRIVER, + ); + assert.notEqual(turns[0]!.nativeThreadId, turns[1]!.nativeThreadId); + const latestRun = projection.runs.at(-1)!; + assert.equal( + projection.attempts.find((attempt) => attempt.id === latestRun.activeAttemptId)! + .nativeThreadId, + turns[1]!.nativeThreadId, + ); + const context = scenario.endsWith("-native") + ? yield* encodeJson(yield* Ref.get(injectedHistory)) + : turns[1]!.text; + assert.include(context, "Original request with constraints"); + assert.include(context, "Original partial work"); + assert.include(context, "New source constraint"); + assert.include(turns[1]!.text, current); + assert.deepEqual(turns[1]!.attachments, screenshots); + const historyBeforeFollowup = yield* Ref.get(injectedHistory); + yield* dispatch(5, "Continue after automatic recovery", CLAUDE_MODEL_SELECTION); + yield* wait(5); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + assert.equal(yield* Ref.get(generation), 2); + assert.equal( + (yield* Ref.get(capturedTurns)).at(-1)!.text, + "Continue after automatic recovery", + ); + assert.deepEqual(yield* Ref.get(injectedHistory), historyBeforeFollowup); + if (scenario.includes("target-estimate")) { + yield* dispatch(6, "Another source turn", CODEX_MODEL_SELECTION); + yield* wait(6); + yield* dispatch(7, "Return after recovery", CLAUDE_MODEL_SELECTION); + yield* wait(7); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + assert.equal(yield* Ref.get(generation), 2); + } + return; + } if (priorImages) { const handoff = projection.contextHandoffs.at(-1)!; const context = scenario.endsWith("-native") @@ -901,7 +1035,12 @@ describe("orchestration v2 provider switching", () => { }), ), ); - it.live.each(["turn-start", "injection", "large-missed-request"] as const)( + it.live.each([ + "turn-start", + "injection", + "large-missed-request", + "exhausted-target-retry", + ] as const)( "recovers %s failure without duplicating history in the same native thread", (failure) => Effect.scoped( @@ -999,6 +1138,22 @@ describe("orchestration v2 provider switching", () => { failure !== "injection" ? "injected" : "pending", ); const historyBeforeRetry = (yield* Ref.get(injectedHistory)).length; + if (failure === "exhausted-target-retry") { + const target = failed.providerThreads.find( + (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + )!; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("exhausted-retry-usage"), + type: "provider-thread.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...target, contextUsage: { usedTokens: 30_000, maxTokens: 32_000 } }, + }, + ], + }); + } const retryText = failure === "large-missed-request" ? "y".repeat(10_000) : "Retry target request"; yield* dispatch(3, retryText, CLAUDE_MODEL_SELECTION); @@ -1006,7 +1161,7 @@ describe("orchestration v2 provider switching", () => { const retried = yield* orchestrator.getThreadProjection(threadId); const lastTurn = (yield* Ref.get(capturedTurns)).at(-1)!; assert.equal(lastTurn.text, retryText); - if (failure !== "injection") { + if (failure !== "injection" && failure !== "exhausted-target-retry") { const delta = yield* encodeJson( (yield* Ref.get(injectedHistory)).slice(historyBeforeRetry), ); diff --git a/apps/server/src/peer/PeerForwarding.test.ts b/apps/server/src/peer/PeerForwarding.test.ts index 05591a053743c..03f69c5c85293 100644 --- a/apps/server/src/peer/PeerForwarding.test.ts +++ b/apps/server/src/peer/PeerForwarding.test.ts @@ -112,7 +112,14 @@ const boxThreads = (seen: Ref.Ref, linkSession: Ref.Ref) => { { sequence: 1, commandId: command.commandId, - event: { type: "run.updated", runId: boxRun }, + event: { + type: "turn-item.updated", + runId: boxRun, + payload: { + type: "run_interrupt_request", + providerTurnId: "provider-turn:box-auto", + }, + }, }, ], } as never), @@ -198,9 +205,10 @@ const serveBox = (seen: Ref.Ref, linkSession: Ref.Ref) => box, Layer.mergeAll( McpHttpServer.layerOrchestratorToolkit, - McpHttpServer.layerProjectRegistration.pipe(Layer.provide(DocumentText.layer)), + McpHttpServer.layerProjectRegistration, McpHttpServer.layerThreadToolkit, ).pipe( + Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provide(NodeCrypto.layer), Layer.provide(boxThreads(seen, linkSession)), Layer.provide(Layer.mock(ThreadSearch.ThreadSearch)({})), @@ -249,9 +257,10 @@ const makeLaptop = Effect.gen(function* () { const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); const layerHere = Layer.mergeAll( McpHttpServer.layerOrchestratorToolkit, - McpHttpServer.layerProjectRegistration.pipe(Layer.provide(DocumentText.layer)), + McpHttpServer.layerProjectRegistration, McpHttpServer.layerThreadToolkit, ).pipe( + Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provideMerge(McpServer.McpServer.layer), Layer.provide(Layer.mock(ThreadSearch.ThreadSearch)({})), Layer.provide(Layer.mock(ThreadLaunch.ThreadLaunchService)({})), diff --git a/apps/server/src/peer/PeerForwarding.ts b/apps/server/src/peer/PeerForwarding.ts index 84c6a5f2a6e2f..227fd26e34ba2 100644 --- a/apps/server/src/peer/PeerForwarding.ts +++ b/apps/server/src/peer/PeerForwarding.ts @@ -80,6 +80,13 @@ export class PeerForwarding extends Context.Service< } >()("t3/peer/PeerForwarding") {} +/** What an agent hears when it names another machine while the beta is off here. */ +export const linkedEnvironmentsOff = new OrchestratorMcpFailure({ + code: "capability_denied", + message: + "Linked environments is off on this machine, so no tool can act on another machine. Leave out environmentId to work here; the user can turn it on in Settings → Labs.", +}); + /** The linked environment a call names, or nothing when it names this one. */ export const remoteTarget = (scope: Scope, environmentId: EnvironmentId | undefined) => environmentId === scope.environmentId ? undefined : environmentId; @@ -196,25 +203,37 @@ const make = Effect.gen(function* () { } }); - const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params) => - tool.name === "t3_thread_wait" + // While the beta is off, agents start nothing over links. Tasks already running there keep + // following through readThread, waitForThread and interruptThread, so they still finish. + const requireEnabled = links.enabled.pipe( + Effect.flatMap((enabled) => (enabled ? Effect.void : Effect.fail(linkedEnvironmentsOff))), + ); + + const call: PeerForwarding["Service"]["call"] = (scope, tool, environmentId, params) => { + const forwarded: Effect.Effect< + Tool.Success, + OrchestratorMcpFailure + > = tool.name === "t3_thread_wait" ? (waitForThread( scope, environmentId, params as OrchestratorMcpThreadWaitInput, ) as Effect.Effect, OrchestratorMcpFailure>) : forward(scope, tool.name, tool.parametersSchema, tool.successSchema, environmentId, params); + return Effect.andThen(requireEnabled, forwarded); + }; /** * The next step for an agent whose machine is not linked: without it, an * empty list reads as a dead end and agents go looking elsewhere. */ const NOT_LISTED = - "A machine the user means that is not listed here is not linked yet: call t3_environment_link with a hint from their words (or the environmentId of a machine they mentioned), and the user picks it and its access in a card."; + "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."; const listLinks: PeerForwarding["Service"]["links"] = Effect.fn("PeerForwarding.listLinks")( function* (scope) { yield* callerLimits(scope); + yield* requireEnabled; const listed = yield* links.list.pipe( Effect.mapError( (error) => diff --git a/apps/server/src/peer/PeerLinkRequests.test.ts b/apps/server/src/peer/PeerLinkRequests.test.ts index 5ec7aff82b460..b7353beb64992 100644 --- a/apps/server/src/peer/PeerLinkRequests.test.ts +++ b/apps/server/src/peer/PeerLinkRequests.test.ts @@ -178,6 +178,7 @@ const makeLaptop = Effect.gen(function* () { const server = Context.get(here, McpServer.McpServer); return { links: Context.get(linking, PeerLinks.PeerLinks), + linksSql: Context.get(linking, SqlClient.SqlClient), requests: Context.get(here, PeerLinkRequests.PeerLinkRequests), orchestrator: Context.get(here, Orchestrator.OrchestratorV2), sink: Context.get(here, EventSink.EventSinkV2), @@ -938,6 +939,19 @@ it.live("a link that expired goes through the card, so the user can link it agai ), ); +it.live("a link made before links were fenced goes through the card, as an expired one", () => + withBoth((a, b) => + Effect.gen(function* () { + yield* seedThread(a); + yield* linkTo(a.links, b); + yield* a.linksSql`UPDATE peer_environment_links SET fenced = 0`; + const asked = yield* a.call({ environmentId: box.environmentId }); + expect(asked.structuredContent).toMatchObject({ status: "pending" }); + yield* pendingCard(a); + }), + ), +); + it.live( "a card for a machine named by id keeps it, and the user may keep a link it already has", () => diff --git a/apps/server/src/peer/PeerLinkRequests.ts b/apps/server/src/peer/PeerLinkRequests.ts index e3ba0b6e39431..cdae666138355 100644 --- a/apps/server/src/peer/PeerLinkRequests.ts +++ b/apps/server/src/peer/PeerLinkRequests.ts @@ -1,6 +1,8 @@ /** - * Pairing codes go directly to PeerLinks.link, never into transcripts or projections. - * User approval may outlive a provider tool timeout, so requests return before the card is answered. + * Links an agent asks the user to make. The user picks the machine and access in a card, and + * their client answers with its addresses and a pairing code that goes straight to + * `PeerLinks.link`, never into the transcript or the agent's context. Asking does not wait + * (provider CLIs abandon a long tool call): the answer reaches the agent as a notification. */ import { CommandId, @@ -141,6 +143,12 @@ const make = Effect.gen(function* () { "t3_environment_link", ); const threadId = threadScope.thread.threadId; + if (!(yield* peerLinks.enabled)) { + return yield* new OrchestratorMcpFailure({ + code: "capability_denied", + message: PeerLinks.LINKED_ENVIRONMENTS_OFF, + }); + } const unreadable = (error: unknown) => new OrchestratorMcpFailure({ code: "orchestration_error", @@ -378,7 +386,7 @@ const outcomeNotice = ( case "linked": { const label = outcome.linkedLabel ?? target; return { - text: `The user linked ${label} with ${outcome.linkedAccess} access. Pass environmentId ${outcome.linkedEnvironmentId} to act there; t3_environment_links lists it.`, + text: `The user linked ${label} with ${outcome.linkedAccess} access. Pass environmentId ${outcome.linkedEnvironmentId} to act there.`, notification: { source: { kind: "background_task" }, outcome: "completed", diff --git a/apps/server/src/peer/PeerLinks.test.ts b/apps/server/src/peer/PeerLinks.test.ts index 2ea6ead464501..f16a91f26c0ad 100644 --- a/apps/server/src/peer/PeerLinks.test.ts +++ b/apps/server/src/peer/PeerLinks.test.ts @@ -19,10 +19,12 @@ import { HttpServerResponse, } from "effect/http"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import * as McpToolAccess from "../mcp/McpToolAccess.ts"; import * as PeerLinks from "./PeerLinks.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { descriptorOf, layerLinkingEnvironment, @@ -73,13 +75,17 @@ const serveB = (descriptor: ExecutionEnvironmentDescriptor) => servePeer(descriptor, McpHttpServer.toolkitRegistration(ProbeToolkit, probeHandlers)); /** Environment A, which links to B. */ -const makeA = (descriptor: ExecutionEnvironmentDescriptor) => - layerLinkingEnvironment(descriptor).pipe( +const makeA = ( + descriptor: ExecutionEnvironmentDescriptor, + secrets?: typeof ServerSecretStore.layer, +) => + layerLinkingEnvironment(descriptor, secrets).pipe( Layer.build, Effect.map((context) => ({ links: Context.get(context, PeerLinks.PeerLinks), peer: Context.get(context, PeerMcpClient.PeerMcpClient), sql: Context.get(context, SqlClient.SqlClient), + settings: Context.get(context, ServerSettings.ServerSettingsService), })), ); @@ -151,6 +157,83 @@ it.effect("links with a pairing code, and the peer holds its agents to the limit ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect( + "each machine's Linked environments beta gates links, and a link waits while it is off", + () => + Effect.scoped( + Effect.gen(function* () { + const b = yield* serveB(box); + const a = yield* makeA(laptop); + const pairing = yield* b.auth.issuePairingCredential(); + const link = a.links.link({ url: b.url, pairingCode: pairing.credential, access: "auto" }); + + yield* a.settings.updateSettings({ linkedEnvironmentsEnabled: false }); + expect(yield* link.pipe(Effect.flip)).toMatchObject({ + reason: "disabled", + message: PeerLinks.LINKED_ENVIRONMENTS_OFF, + }); + expect(yield* Ref.get(b.bearers)).toEqual([]); + + yield* a.settings.updateSettings({ linkedEnvironmentsEnabled: true }); + yield* b.settings.updateSettings({ linkedEnvironmentsEnabled: false }); + const refusedThere = yield* link.pipe(Effect.flip); + expect(refusedThere.reason).toBe("pairing_rejected"); + expect(refusedThere.message).toContain("Linked environments is off on that machine"); + expect(yield* linkedSessions(b)).toEqual([]); + + yield* b.settings.updateSettings({ linkedEnvironmentsEnabled: true }); + yield* link; + yield* b.settings.updateSettings({ linkedEnvironmentsEnabled: false }); + // Not final, so a task already running there picks up again once it is back on. + const waiting = yield* probe(a).pipe(Effect.flip); + expect(waiting.code).toBe("orchestration_error"); + expect(waiting.message).toContain("has Linked environments turned off"); + yield* b.settings.updateSettings({ linkedEnvironmentsEnabled: true }); + expect(yield* probe(a)).toEqual({ caller: "T3 Code · Laptop", modes: "auto/default" }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a link whose row cannot be stored keeps none of its token", () => + Effect.scoped( + Effect.gen(function* () { + const b = yield* serveB(box); + const held = yield* Ref.make>(new Set()); + const noted = Layer.effect( + ServerSecretStore.ServerSecretStore, + Effect.map(ServerSecretStore.make, (store) => + ServerSecretStore.ServerSecretStore.of({ + ...store, + set: (name, value) => + store + .set(name, value) + .pipe(Effect.andThen(Ref.update(held, (names) => new Set([...names, name])))), + remove: (name) => + store + .remove(name) + .pipe( + Effect.andThen( + Ref.update(held, (names) => new Set([...names].filter((at) => at !== name))), + ), + ), + }), + ), + ); + const a = yield* makeA(laptop, noted); + yield* a.sql` + CREATE TRIGGER refuse_link BEFORE INSERT ON peer_environment_links + BEGIN SELECT RAISE(ABORT, 'database or disk is full'); END + `; + const pairing = yield* b.auth.issuePairingCredential(); + const failed = yield* a.links + .link({ url: b.url, pairingCode: pairing.credential, access: "auto" }) + .pipe(Effect.flip); + expect(failed.reason).toBe("storage"); + expect(yield* Ref.get(held)).toEqual(new Set()); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("two links racing to one peer leave the stored access matching the token", () => Effect.scoped( Effect.gen(function* () { @@ -306,6 +389,8 @@ it.effect("tells the agent to link again once the peer revokes the link or it ex // A link stored before links were marked: its session there is not a // link's, so the peer would not fence its work even though it can. yield* a.sql`UPDATE peer_environment_links SET fenced = 0`; + // Linking again is its only way back, as for an expired link. + expect((yield* a.links.list).map((link) => link.status)).toEqual(["expired"]); const legacy = yield* probe(a).pipe(Effect.flip); expect(legacy.code).toBe("capability_denied"); expect(legacy.message).toContain("Link it again"); diff --git a/apps/server/src/peer/PeerLinks.testkit.ts b/apps/server/src/peer/PeerLinks.testkit.ts index 39e86c8b38659..0f7e39355b95d 100644 --- a/apps/server/src/peer/PeerLinks.testkit.ts +++ b/apps/server/src/peer/PeerLinks.testkit.ts @@ -28,6 +28,7 @@ import * as McpHttpServer from "../mcp/McpHttpServer.ts"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as Sqlite from "../persistence/Sqlite.ts"; import * as PeerLinks from "./PeerLinks.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as PeerMcpClient from "./PeerMcpClient.ts"; class PeerTestApi extends HttpApi.make("environment") @@ -70,6 +71,7 @@ export const servePeer = ( getDescriptor: Ref.get(descriptor), }); const authContext = yield* EnvironmentAuth.layer.pipe( + Layer.provideMerge(ServerSettings.layerTest({ linkedEnvironmentsEnabled: true })), Layer.provide(Sqlite.layerMemory), Layer.provideMerge(ServerSecretStore.layer), Layer.provideMerge(ServerEnvironment.layerIdentity), @@ -103,7 +105,8 @@ export const servePeer = ( Layer.provideMerge(McpHttpServer.layerMcpTransport), Layer.provide( Layer.mock(McpSessionRegistry.McpSessionRegistry)({ - resolve: () => Effect.sync(() => undefined), + // @effect-diagnostics-next-line effectSucceedWithVoid:off - This optional result requires undefined, not the wider void type. + resolve: () => Effect.succeed(undefined), }), ), Layer.provide(McpOAuth.layerMcpClientAuthenticator), @@ -126,6 +129,8 @@ export const servePeer = ( return { url, auth, + /** Its settings, as for turning its Linked environments beta off. */ + settings: Context.get(authContext, ServerSettings.ServerSettingsService), descriptor, bearers, /** How often its descriptor was asked for, as a probe of whether it answers. */ @@ -147,11 +152,15 @@ export type ServedPeer = Effect.Success>; * The environment that links: its own database and secret store, reaching * peers over plain HTTP. Provide it to anything built on its links. */ -export const layerLinkingEnvironment = (descriptor: ExecutionEnvironmentDescriptor) => +export const layerLinkingEnvironment = ( + descriptor: ExecutionEnvironmentDescriptor, + secrets: typeof ServerSecretStore.layer = ServerSecretStore.layer, +) => PeerMcpClient.layer.pipe( Layer.provideMerge(PeerLinks.layer), + Layer.provideMerge(ServerSettings.layerTest({ linkedEnvironmentsEnabled: true })), Layer.provideMerge(Sqlite.layerMemory), - Layer.provide(Layer.mergeAll(ServerSecretStore.layer, FetchHttpClient.layer)), + Layer.provide(Layer.mergeAll(secrets, FetchHttpClient.layer)), // Services that act for this environment, such as delegation, name it too. Layer.provideMerge( Layer.succeed(ServerEnvironment.ServerEnvironment, { diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 9e481819a452a..54f6af780d71d 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -31,6 +31,7 @@ import { isLocalLoopbackHost, isTailnetHost } from "@t3tools/shared/hostClassifi import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ServerSettings from "../serverSettings.ts"; /** The redirect registered for a link. Nothing ever listens there: the code comes back in the response. */ const LINK_REDIRECT_URI = "http://127.0.0.1/t3code-peer-link"; @@ -49,6 +50,9 @@ export interface ResolvedPeerLink { readonly token: string; } +export const LINKED_ENVIRONMENTS_OFF = + "Linked environments is off on this machine. Turn it on in Settings → Labs."; + /** * Links from this environment to others. Linking signs this environment in to * the peer's `/mcp` as an outside agent with a pairing code from the peer, the @@ -58,6 +62,8 @@ export interface ResolvedPeerLink { export class PeerLinks extends Context.Service< PeerLinks, { + /** Whether the Linked environments beta (Settings → Labs) is on here. */ + readonly enabled: Effect.Effect; readonly link: (input: PeerLinkCreateInput) => Effect.Effect; /** * The environment answering at `url`, or nothing when it is not a usable @@ -159,6 +165,11 @@ const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const httpClient = yield* HttpClient.HttpClient; const environment = yield* ServerEnvironment.ServerEnvironment; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const enabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.linkedEnvironmentsEnabled), + Effect.orElseSucceed(() => false), + ); /** A fresh name for one link's token; the row records it. */ const newSecretName = crypto.randomBytes(16).pipe( @@ -192,7 +203,9 @@ const make = Effect.gen(function* () { urls: yield* decodeUrls(row.urls_json), access: yield* decodeAccess(row.access), linkedAt: DateTime.makeUnsafe(row.linked_at), - expiresAt: DateTime.makeUnsafe(row.expires_at), + // Its token predates link fencing, so the peer would not fence its work: + // like an expired link, only a new pairing code brings it back. + expiresAt: DateTime.makeUnsafe(row.fenced === 1 ? row.expires_at : row.linked_at), lastReachedAt: row.last_reached_at === null ? null : DateTime.makeUnsafe(row.last_reached_at), lastError: row.last_error, } satisfies PeerLink; @@ -386,6 +399,9 @@ const make = Effect.gen(function* () { }); const link: PeerLinks["Service"]["link"] = Effect.fn("PeerLinks.link")(function* (input) { + if (!(yield* enabled)) { + return yield* new PeerLinkError({ reason: "disabled", message: LINKED_ENVIRONMENTS_OFF }); + } const urls = [input.url, ...(input.alternateUrls ?? [])]; // The user picked a machine: link through the addresses that prove to // be it, so a reused LAN address never receives the pairing code. @@ -428,9 +444,11 @@ const make = Effect.gen(function* () { yield* secrets .set(secretName, new TextEncoder().encode(issued.access_token)) .pipe(Effect.catch(failStorage)); - const urlsJson = yield* encodeUrls(origins).pipe(Effect.catch(failStorage)); - const previous = yield* readRow(peer.environmentId); - yield* sql` + // A token no row names could never be removed. + const previous = yield* Effect.gen(function* () { + const urlsJson = yield* encodeUrls(origins).pipe(Effect.catch(failStorage)); + const previous = yield* readRow(peer.environmentId); + yield* sql` INSERT INTO peer_environment_links ( environment_id, secret_name, label, urls_json, access, linked_at, expires_at, last_reached_at, last_error, fenced @@ -450,6 +468,8 @@ const make = Effect.gen(function* () { last_reached_at = excluded.last_reached_at, last_error = NULL `.pipe(Effect.catch(failStorage)); + return previous; + }).pipe(Effect.onError(() => secrets.remove(secretName).pipe(Effect.ignore))); // Linking again replaces the old token. A racing link may leave its own // behind; it is unused either way. if (Option.isSome(previous) && previous.value.secret_name !== secretName) { @@ -593,6 +613,7 @@ const make = Effect.gen(function* () { ); return PeerLinks.of({ + enabled, link, describe: describeUrl, list, diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index c9ab38e6ab82d..18c704c0ad82e 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -129,6 +129,12 @@ const readResponse = Effect.fn("PeerMcpClient.readResponse")(function* ( return decoded; }); +const linkedEnvironmentsOffThere = new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "The linked environment has Linked environments turned off. Turn it on in its Settings → Labs.", +}); + const make = Effect.gen(function* () { const links = yield* PeerLinks.PeerLinks; const httpClient = yield* HttpClient.HttpClient; @@ -179,6 +185,7 @@ const make = Effect.gen(function* () { }, limitHeader, ); + if (initialized.status === 403) return yield* linkedEnvironmentsOffThere; const sessionId = initialized.headers["mcp-session-id"]; if (initialized.status !== 200 || sessionId === undefined) { return yield* new OrchestratorMcpFailure({ @@ -237,6 +244,9 @@ const make = Effect.gen(function* () { "The linked environment no longer accepts this link. It may have been revoked there; link it again.", }); } + if (response.status === 403) { + return yield* linkedEnvironmentsOffThere; + } if (response.status !== 200) { return yield* new OrchestratorMcpFailure({ code: "orchestration_error", diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 913fc371fbc25..1e2d29f9469da 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -23,15 +23,15 @@ import { expect, it } from "@effect/vitest"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; -import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; +import * as Queue from "effect/Queue"; +import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; -import * as TestClock from "effect/testing/TestClock"; import type * as SqlClient from "effect/sql/SqlClient"; import { McpSchema, McpServer } from "effect/ai"; @@ -129,6 +129,8 @@ interface BoxThread { readonly releaseStop: Deferred.Deferred; /** Messages sent to the thread there after its launch, each starting a later run. */ readonly sent: Ref.Ref>; + /** Runs as the launch there is answered, before the laptop hears back. */ + readonly afterLaunch: Ref.Ref>; } const makeBoxThread = Effect.gen(function* () { @@ -163,6 +165,7 @@ const makeBoxThread = Effect.gen(function* () { stopStarted: yield* Deferred.make(), releaseStop: yield* Deferred.make(), sent: yield* Ref.make>([]), + afterLaunch: yield* Ref.make>(Effect.void), } satisfies BoxThread; }); @@ -435,6 +438,7 @@ const boxToolkitLayer = (thread: BoxThread) => { interactionMode: input.interactionMode, }), ), + Effect.andThen(Effect.flatten(Ref.get(thread.afterLaunch))), Effect.as({ threadId: input.threadId, projection: { @@ -463,8 +467,9 @@ const boxToolkitLayer = (thread: BoxThread) => { }); return Layer.merge( McpHttpServer.layerOrchestratorToolkit, - McpHttpServer.layerProjectRegistration.pipe(Layer.provide(DocumentText.layer)), + McpHttpServer.layerProjectRegistration, ).pipe( + Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), Layer.provide(NodeCrypto.layer), Layer.provide(layerBoxThreads), Layer.provide(layerBoxLaunches), @@ -604,6 +609,7 @@ const makeLaptop = ( const server = Context.get(here, McpServer.McpServer); return { links: Context.get(linking, PeerLinks.PeerLinks), + settings: Context.get(linking, ServerSettings.ServerSettingsService), orchestrator: Context.get(here, Orchestrator.OrchestratorV2), remote: Context.get(here, RemoteDelegation.RemoteDelegation), threads: Context.get(here, ThreadManagement.ThreadManagementService), @@ -1699,6 +1705,16 @@ it.effect("stopping the parent stops its task there", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +/** A follower retry schedule the test steps: each failed attempt is offered, then waits for `retry`. */ +const steppedRetries = Effect.gen(function* () { + const failed = yield* Queue.unbounded(); + const retry = yield* Queue.unbounded(); + const schedule = Schedule.forever.pipe( + Schedule.tap(() => Queue.offer(failed, undefined).pipe(Effect.andThen(Queue.take(retry)))), + ); + return { failed, retry, schedule }; +}); + /** Waits for a recorded remote Stop, optionally limited to one stop generation. */ const stoppedThere = ( laptopEnv: Effect.Success>, @@ -1927,25 +1943,19 @@ it.effect("a parent stopped while the box is unreachable still stops its task th ReadonlyArray >([]); const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); - const offline = yield* Deferred.make(); - const unreachable = yield* Deferred.make(); - const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); - const client = Context.get(linking, PeerMcpClient.PeerMcpClient); - const observed = Context.add(linking, PeerMcpClient.PeerMcpClient, { - call: (input) => - (input.tool === "t3_thread_wait" ? Deferred.await(offline) : Effect.void).pipe( - Effect.andThen(client.call(input)), - Effect.tapError(() => Deferred.succeed(unreachable, undefined)), - ), - }); - const a = yield* makeLaptop(database, offers, observed); + const retries = yield* steppedRetries; + const a = yield* makeLaptop(database, offers).pipe( + Effect.provideService(RemoteDelegation.FollowRetrySchedule, retries.schedule), + ); yield* linkTo(a.links, b, "full-access"); yield* seedParent(a); - // The box sleeps as soon as the task is there, so its follower backs off. + // The box sleeps as soon as the task is there, so its follower's first attempt fails. + yield* Ref.set( + thread.afterLaunch, + Ref.set(b.descriptor, descriptorOf("environment-elsewhere", "Elsewhere")), + ); const delegated = yield* delegateToBox(a); - yield* Ref.set(b.descriptor, descriptorOf("environment-elsewhere", "Elsewhere")); - yield* Deferred.succeed(offline, undefined); - yield* Deferred.await(unreachable); + yield* Queue.take(retries.failed); // The parent is stopped meanwhile. yield* a.threads.stopDelegatedTasks({ threadId: parentThreadId, @@ -1955,7 +1965,7 @@ it.effect("a parent stopped while the box is unreachable still stops its task th // Once the box is back, the follower's next attempt stops the task there. yield* Ref.set(b.descriptor, box); - yield* TestClock.adjust(Duration.minutes(5)); + yield* Queue.offer(retries.retry, undefined); yield* Deferred.await(thread.interruptedOnce); expect(yield* Ref.get(thread.interrupted)).toBe(1); }), @@ -2048,3 +2058,49 @@ it.effect("a local task runs in this thread's project", () => }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + +it.effect( + "with the beta off here, a task already running there still ends, and no new one starts", + () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const a = yield* makeLaptop(database, offers); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const delegated = yield* delegateToBox(a); + + yield* a.settings.updateSettings({ linkedEnvironmentsEnabled: false }); + const refused = yield* a.call("delegate_task", { + task: "Run the linter on the box.", + target: { + environmentId: box.environmentId, + providerInstanceId: instanceId, + model: "gpt-5.4", + }, + mode: "async", + clientRequestId: "lint-on-box", + }); + expect(refused.isError).toBe(true); + expect(JSON.stringify(refused.content)).toContain( + PeerForwarding.linkedEnvironmentsOff.message, + ); + expect((yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents).toHaveLength( + 1, + ); + + yield* Ref.set(thread.reply, "All 412 tests pass on the box."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ + status: "completed", + result: "All 412 tests pass on the box.", + }); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 5ecef3a6f1876..03a8b12cc4194 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -50,10 +50,17 @@ import * as PeerLinks from "./PeerLinks.ts"; const FOLLOW_WAIT_MS = 10 * 60 * 1_000; /** A linked environment that stops answering is retried, backing off to this. */ const FOLLOW_MAX_BACKOFF = Duration.minutes(5); -const backoff = Schedule.exponential(Duration.seconds(2)).pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, FOLLOW_MAX_BACKOFF)), - ), +/** How a follower retries; tests step it by hand. */ +export const FollowRetrySchedule = Context.Reference>( + "t3/peer/RemoteDelegation/FollowRetrySchedule", + { + defaultValue: () => + Schedule.exponential(Duration.seconds(2)).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, FOLLOW_MAX_BACKOFF)), + ), + ), + }, ); type ThreadScope = McpInvocationContext.McpThreadInvocationScope; @@ -112,6 +119,7 @@ const isFinal = (error: OrchestratorMcpFailure) => error.code === "run_not_found"; const make = Effect.gen(function* () { + const followRetry = yield* FollowRetrySchedule; const forwarding = yield* PeerForwarding.PeerForwarding; const links = yield* PeerLinks.PeerLinks; const threads = yield* ThreadManagement.ThreadManagementService; @@ -557,7 +565,7 @@ const make = Effect.gen(function* () { yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount); }), }), - Effect.retry(backoff), + Effect.retry(followRetry), Effect.ignoreCause({ log: true }), ); diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index c0a113f3e4153..3ace2a97f3f79 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -62,6 +62,9 @@ const makeHarness = Effect.fn(function* (options: { getThreadShell: () => Effect.succeed(null), }), Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadShell: () => Effect.succeed(null), + }), ServerSettings.layerTest(), NodeCrypto.layer, ), diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts index 94bdc6ffc5792..6723e4d9b741c 100644 --- a/apps/server/src/provider/AgentContextBudget.test.ts +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -22,6 +22,10 @@ import { ProviderAdapterRegistryV2 } from "../orchestration-v2/ProviderAdapterRe import { ThreadLaunchService } from "../orchestration-v2/ThreadLaunchService.ts"; import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts"; import { ThreadSearch } from "../orchestration-v2/ThreadSearch.ts"; +import { PeerForwarding } from "../peer/PeerForwarding.ts"; +import { PeerLinkRequests } from "../peer/PeerLinkRequests.ts"; +import { PeerLinks } from "../peer/PeerLinks.ts"; +import { RemoteDelegation } from "../peer/RemoteDelegation.ts"; import { PreviewManager } from "../preview/Manager.ts"; import { PreviewBrowser } from "../preview/PreviewBrowser.ts"; import { CodeIntelligence } from "../project/knowledge/CodeIntelligence.ts"; @@ -29,10 +33,6 @@ import { FileHistory } from "../project/knowledge/FileHistory.ts"; import { ProjectKnowledge } from "../project/knowledge/ProjectKnowledge.ts"; import { ProjectLoom } from "../project/loom/ProjectLoom.ts"; import { ConversationSummaries } from "../project/conversationSummaries/ConversationSummaries.ts"; -import { PeerForwarding } from "../peer/PeerForwarding.ts"; -import { PeerLinkRequests } from "../peer/PeerLinkRequests.ts"; -import { PeerLinks } from "../peer/PeerLinks.ts"; -import { RemoteDelegation } from "../peer/RemoteDelegation.ts"; import { ManagedProjectFolders } from "../project/ManagedProjectFolders.ts"; import { ProjectService } from "../project/ProjectService.ts"; import { ProjectSetupScriptRunner } from "../project/ProjectSetupScriptRunner.ts"; diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index c9449d2450054..03b86d9df8705 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6298, "codexAdditionalContextBytes": 6848, - "mcpToolListJsonBytes": 63335 + "mcpToolListJsonBytes": 61320 } diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 9a43fe8b0cb0a..5819b2edb322e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -775,7 +775,9 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide( PullRequestReviewGuideService.layer.pipe(Layer.provide(PullRequestReviewGuides.layer)), ), + // delegate_task to a linked environment, from /mcp and the follower above. Layer.provide(RemoteDelegation.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + // Links agents ask for: shown in a thread from /mcp, answered over WebSocket. Layer.provide(PeerLinkRequests.layer), // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches. Layer.provide(layerPullRequestService), diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 01d7502bcb4b9..8f9d71bbbd3dd 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -11,6 +11,7 @@ import { } from "@t3tools/contracts"; import { acpRegistryClient } from "@t3tools/provider-acp-registry/client"; import { cursorClient } from "@t3tools/provider-cursor/client"; +import { fxClient } from "@t3tools/provider-fx/client"; import { grokClient } from "@t3tools/provider-grok/client"; import { museClient } from "@t3tools/provider-muse/client"; import { openCodeClient } from "@t3tools/provider-opencode/client"; @@ -33,9 +34,15 @@ const providerSettingsSchemas = [ ["codex", CodexSettings], ["claudeAgent", ClaudeSettings], ["antigravity", AntigravitySettings], - ...[cursorClient, grokClient, museClient, openCodeClient, piClient, acpRegistryClient].map( - (definition) => [definition.driverKind, definition.settingsSchema] as const, - ), + ...[ + cursorClient, + fxClient, + grokClient, + museClient, + openCodeClient, + piClient, + acpRegistryClient, + ].map((definition) => [definition.driverKind, definition.settingsSchema] as const), ] as const; const providerFieldPolicies = new Map( diff --git a/apps/web/src/components/CommandPaletteProjectSearch.browser.test.tsx b/apps/web/src/components/CommandPaletteProjectSearch.browser.test.tsx index 18076c358517f..5972c831c798b 100644 --- a/apps/web/src/components/CommandPaletteProjectSearch.browser.test.tsx +++ b/apps/web/src/components/CommandPaletteProjectSearch.browser.test.tsx @@ -216,7 +216,7 @@ it("keeps each group independent when one source fails, and opens symbols and co { router: "/env/thread-a" }, ); - await userEvent.type(screen.getByRole("combobox"), "refresh token"); + await userEvent.fill(screen.getByRole("combobox"), "refresh token"); // Facts fail on their own; the other groups still render their results. await expect.element(screen.getByText("Fact storage is offline.")).toBeVisible(); diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index ba5b46252590d..af5a200e264af 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -1,4 +1,4 @@ -import { documentIconColorForFileName } from "~/pierre-icons"; +import { documentChipKindForFileName } from "~/pierre-icons"; import { isProviderSkillUserInvocable } from "@t3tools/client-runtime/providerSkills"; import { formatProviderSkillDisplayName } from "@t3tools/shared/inlineSkills"; import { @@ -46,7 +46,6 @@ import { useLayoutEffect, useRef, useState, - type CSSProperties, type KeyboardEvent as ReactKeyboardEvent, type RefObject, } from "react"; @@ -232,7 +231,7 @@ function resolvedThemeFromDocument(): "light" | "dark" { } const CHIP_NODE_SELECTION_CLASS_NAME = - "relative inline-flex select-none items-center align-middle leading-none data-[composer-chip-selected]:after:pointer-events-none data-[composer-chip-selected]:after:absolute data-[composer-chip-selected]:after:inset-0 data-[composer-chip-selected]:after:rounded-sm data-[composer-chip-selected]:after:bg-[Highlight] data-[composer-chip-selected]:after:opacity-30 data-[composer-chip-selected]:after:content-['']"; + "relative inline-flex min-w-0 max-w-full select-none items-baseline data-[composer-chip-selected]:after:pointer-events-none data-[composer-chip-selected]:after:absolute data-[composer-chip-selected]:after:inset-0 data-[composer-chip-selected]:after:rounded-sm data-[composer-chip-selected]:after:bg-[Highlight] data-[composer-chip-selected]:after:opacity-30 data-[composer-chip-selected]:after:content-['']"; const ComposerMentionExtension = Node.create({ name: "composer-mention", @@ -260,11 +259,9 @@ const ComposerMentionExtension = Node.create({ function ComposerMentionNodeView({ node }: NodeViewProps) { const actions = use(ComposerContextActionsContext); const path = (node.attrs.path as string) ?? ""; - const accent = documentIconColorForFileName(path); const chip = ( } onClick={() => actions.openMention(path)} @@ -442,7 +439,7 @@ function ComposerCitationNodeView({ node, editor, getPos }: NodeViewProps) { return ( ; +} + +describe("ContextChip", () => { + it("sets its label on the sentence baseline at the sentence size without growing the line", async () => { + await using screen = await render( +

+ Open + the{" "} + }> + + + src/ + + Typography.tsx + + {" "} + file +

, + ); + const paragraph = screen.container.querySelector("p")!; + const label = screen.container.querySelector('[data-slot="context-chip-label"]')!; + const baseline = (id: string) => + screen.container.querySelector(`[data-baseline="${id}"]`)!.getBoundingClientRect().bottom; + + expect(getComputedStyle(label).fontSize).toBe(getComputedStyle(paragraph).fontSize); + expect(Math.abs(baseline("chip") - baseline("text"))).toBeLessThanOrEqual(0.5); + expect(paragraph.getBoundingClientRect().height).toBeCloseTo( + Number.parseFloat(getComputedStyle(paragraph).lineHeight), + 0, + ); + }); +}); diff --git a/apps/web/src/components/ContextChip.tsx b/apps/web/src/components/ContextChip.tsx index ec1481079076b..74542ce9debbb 100644 --- a/apps/web/src/components/ContextChip.tsx +++ b/apps/web/src/components/ContextChip.tsx @@ -6,33 +6,39 @@ import type * as React from "react"; import { cn } from "~/lib/utils"; const contextChipVariants = cva( - "inline-flex h-[1.41em] max-w-full items-center gap-[0.33em] rounded-[0.5em] border px-[0.5em] align-middle font-medium text-[0.86em] leading-none [&_svg]:block [&_svg]:size-[1.17em] [&_svg]:shrink-0 [&_svg]:self-center [button&,a&,[data-popup-open]&]:cursor-pointer [button&,a&]:transition-colors [button&,a&]:motion-reduce:transition-none focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-foreground disabled:cursor-default", + "inline-flex max-w-full items-center gap-[0.2em] font-medium leading-[normal] text-(--context-chip-ink) [--context-chip-ink:var(--context-chip-accent)] [--document-icon-stroke:var(--context-chip-ink)] [&_svg]:block [&_svg]:size-[1em] [&_svg]:shrink-0 [&_svg]:self-center [button&,a&,[data-popup-open]&]:cursor-pointer focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-current rounded-sm disabled:cursor-default", { defaultVariants: { kind: "neutral" }, variants: { kind: { - neutral: "border-border/70 bg-accent/40 text-foreground", - image: "[--context-chip-accent:oklch(0.62_0.16_16)]", - video: "[--context-chip-accent:oklch(0.62_0.16_48)]", - file: "[--context-chip-accent:oklch(0.62_0.136_237)]", - mention: "[--context-chip-accent:oklch(0.62_0.11_215)]", - terminal: "[--context-chip-accent:oklch(0.62_0.134_163)]", - element: "[--context-chip-accent:oklch(0.62_0.134_70)]", - "preview-annotation": "[--context-chip-accent:oklch(0.62_0.134_70)]", - "review-comment": "[--context-chip-accent:oklch(0.62_0.16_292)]", - "pull-request": "[--context-chip-accent:oklch(0.62_0.16_277)]", - "pr-open": "[--context-chip-accent:oklch(0.62_0.134_163)]", - "pr-draft": "[--context-chip-accent:oklch(0.62_0.02_259)]", - "pr-merged": "[--context-chip-accent:oklch(0.62_0.16_292)]", - "pr-closed": "[--context-chip-accent:oklch(0.62_0.16_16)]", - skill: "[--context-chip-accent:oklch(0.62_0.16_322)]", - thread: "[--context-chip-accent:oklch(0.62_0.12_190)]", - environment: "[--context-chip-accent:oklch(0.62_0.11_145)]", - citation: "[--context-chip-accent:oklch(0.62_0.16_259)]", + neutral: "[--context-chip-accent:currentColor]", + image: "[--context-chip-accent:var(--context-chip-image)]", + video: "[--context-chip-accent:var(--context-chip-video)]", + file: "[--context-chip-accent:var(--context-chip-file)]", + mention: "[--context-chip-accent:var(--context-chip-mention)]", + terminal: "[--context-chip-accent:var(--context-chip-terminal)]", + element: "[--context-chip-accent:var(--context-chip-element)]", + "preview-annotation": "[--context-chip-accent:var(--context-chip-preview-annotation)]", + "review-comment": "[--context-chip-accent:var(--context-chip-review-comment)]", + "pull-request": "[--context-chip-accent:var(--context-chip-pull-request)]", + "pr-open": "[--context-chip-accent:var(--context-chip-pr-open)]", + "pr-draft": "[--context-chip-accent:var(--context-chip-pr-draft)]", + "pr-merged": "[--context-chip-accent:var(--context-chip-pr-merged)]", + "pr-closed": "[--context-chip-accent:var(--context-chip-pr-closed)]", + skill: "[--context-chip-accent:var(--context-chip-skill)]", + thread: "[--context-chip-accent:var(--context-chip-thread)]", + environment: "[--context-chip-accent:var(--context-chip-environment)]", + citation: "[--context-chip-accent:var(--context-chip-citation)]", + "doc-pdf": "[--context-chip-accent:var(--context-chip-doc-pdf)]", + "doc-word": "[--context-chip-accent:var(--context-chip-doc-word)]", + "doc-spreadsheet": "[--context-chip-accent:var(--context-chip-doc-spreadsheet)]", + "doc-slides": "[--context-chip-accent:var(--context-chip-doc-slides)]", + "doc-epub": "[--context-chip-accent:var(--context-chip-doc-epub)]", }, state: { - unresolved: "border-dashed", - invalid: "", + unresolved: "data-[state=unresolved]:[--context-chip-accent:currentColor]", + invalid: + "data-[state=invalid]:[--context-chip-accent:var(--context-chip-invalid,var(--destructive-foreground))]", }, }, compoundVariants: [ @@ -88,7 +94,10 @@ function ContextChip({ className, kind, state, render, ...props }: ContextChipPr function ContextChipLabel({ className, ...props }: React.ComponentProps<"span">) { return ( &]:decoration-current", + className, + )} data-slot="context-chip-label" {...props} /> @@ -98,7 +107,7 @@ function ContextChipLabel({ className, ...props }: React.ComponentProps<"span">) function ContextChipAction({ className, render, ...props }: useRender.ComponentProps<"button">) { const defaultProps = { className: cn( - "ml-[0.17em] inline-flex size-[1.17em] shrink-0 cursor-pointer items-center justify-center rounded-sm text-current transition-colors hover:bg-(--context-chip-accent,var(--color-foreground))/17 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ring motion-reduce:transition-none [&_svg]:size-[0.85em]", + "ml-[0.17em] inline-flex size-[1em] shrink-0 cursor-pointer items-center justify-center rounded-sm text-current transition-colors hover:bg-(--context-chip-accent,var(--color-foreground))/17 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ring motion-reduce:transition-none [&_svg]:size-[0.85em]", className, ), "data-slot": "context-chip-action", diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 9a4893cb4a874..5e83743695967 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -812,6 +812,7 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve projectGroupByScopeKey, projectScopeItems, } = useSidebarLookups(); + const delegatedUnderParent = useDelegatedUnderParent(); const router = useRouter(); const { isMobile, setOpenMobile } = useSidebar(); const keybindings = useAtomValue(primaryServerKeybindingsAtom); @@ -993,7 +994,6 @@ export default function Sidebar({ routeReveal }: { routeReveal: SidebarRouteReve ), ); const threads = useQuietThreadShells(); - const delegatedUnderParent = useDelegatedUnderParent(); const routeRowThreadKey = routeThreadRef === null ? null diff --git a/apps/web/src/components/ThreadContextChip.tsx b/apps/web/src/components/ThreadContextChip.tsx index 5f1a4202f1d36..715a210f4e917 100644 --- a/apps/web/src/components/ThreadContextChip.tsx +++ b/apps/web/src/components/ThreadContextChip.tsx @@ -30,7 +30,7 @@ export function ThreadContextChip(props: { className="no-underline" > {project ? ( - + ) : ( )} diff --git a/apps/web/src/components/ThreadNotificationCoordinator.tsx b/apps/web/src/components/ThreadNotificationCoordinator.tsx index f75fd901c0e97..9513ec7e1826a 100644 --- a/apps/web/src/components/ThreadNotificationCoordinator.tsx +++ b/apps/web/src/components/ThreadNotificationCoordinator.tsx @@ -240,32 +240,38 @@ export function startThreadNotifications( if (state) present(environmentId, state, kind); }), }; - const observe = () => { - const shell = registry.get(environmentShell.stateValueAtom(environmentId)); - const threads = - shell.status === "live" && Option.isSome(shell.snapshot) - ? shell.snapshot.value.threads - : null; - if (threads !== null) { - observeThreads(environmentId, entry, threads); - return; - } - entry.gate.clear(); - entry.previous = new Map(); + let threads: ReadonlyArray | null = null; + const unsubscribeShell = registry.subscribe( + environmentShell.stateValueAtom(environmentId), + (shell) => { + threads = + shell.status === "live" && Option.isSome(shell.snapshot) + ? shell.snapshot.value.threads + : null; + if (threads !== null) { + observeThreads(environmentId, entry, threads); + return; + } + entry.gate.clear(); + entry.previous = new Map(); + }, + { immediate: true }, + ); + const reobserve = () => { + if (threads !== null) observeThreads(environmentId, entry, threads); }; - // Linked environments change what this one's threads ask and which rows sit under a parent. - const unsubscribes = [ - registry.subscribe(environmentShell.stateValueAtom(environmentId), observe, { - immediate: true, - }), - registry.subscribe( - environmentThreadShells.pendingSubagentRequestsAtom(environmentId), - observe, - ), - registry.subscribe(environmentThreadShells.delegatedUnderParentAtom, observe), - ]; + const unsubscribeRequests = registry.subscribe( + environmentThreadShells.pendingSubagentRequestsAtom(environmentId), + reobserve, + ); + const unsubscribeDelegated = registry.subscribe( + environmentThreadShells.delegatedUnderParentAtom, + reobserve, + ); entry.unsubscribe = () => { - for (const unsubscribe of unsubscribes) unsubscribe(); + unsubscribeShell(); + unsubscribeRequests(); + unsubscribeDelegated(); entry.gate.clear(); }; return entry; diff --git a/apps/web/src/components/chat/AssistantCitationChip.tsx b/apps/web/src/components/chat/AssistantCitationChip.tsx index dd21f7c466844..1e97f00319c2e 100644 --- a/apps/web/src/components/chat/AssistantCitationChip.tsx +++ b/apps/web/src/components/chat/AssistantCitationChip.tsx @@ -124,7 +124,7 @@ export function AssistantCitationChip({ const composerSourceLink = (