From 2bd3203c17833b631ff4ff7e79db355585e571fd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 00:58:11 -0800 Subject: [PATCH 1/3] improvement(computer-use): fold empty layout groups and compact rows in the model's accessibility tree Wrapper groups with no text, state or action no longer spend the observation budget; their children point at the nearest shown ancestor. Rows drop the selected window id, JSON array syntax and fractional coordinates, and a one-line legend explains the format. --- .../tools/client/computer-tool-result.test.ts | 29 +++++++ .../tools/client/computer-tool-result.ts | 79 +++++++++++++++---- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts index 4eb5a6c0d70..bb3f17823de 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts @@ -198,6 +198,35 @@ describe('native computer model projection', () => { expect(result).toMatchObject({ dispatched: true, sequence: { completedSteps: 1 } }) }) + it('folds empty layout groups so their controls fit and point at a shown parent', () => { + const state = snapshot() + state.nodes = [ + { elementId: 'window', role: 'AXWindow', label: 'Fixture', actions: [], windowId: '1' }, + ...Array.from({ length: 300 }, (_, index) => ({ + elementId: `wrap-${index}`, + parentId: index === 0 ? 'window' : `wrap-${index - 1}`, + role: 'AXGroup', + actions: [], + windowId: '1', + })), + ...Array.from({ length: 150 }, (_, index) => ({ + elementId: `button-${index}`, + parentId: 'wrap-299', + role: 'AXButton', + label: `Action ${index}`, + actions: ['AXPress'], + windowId: '1', + })), + ] + const result = computerToolResultForModel(state) + if (!('accessibilityTree' in result)) throw new Error('Missing projected tree') + const rows = result.accessibilityTree.split('\n') + expect(rows.some((row) => row.includes('AXGroup'))).toBe(false) + expect(rows.filter((row) => row.includes('AXButton'))).toHaveLength(150) + expect(rows.find((row) => row.startsWith('button-0 '))).toContain('parent=window') + expect(result.omittedNodeCount).toBe(0) + }) + it('does not split an emoji when bounding text for the model', () => { const state = snapshot() state.nodes[0] = { diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 8fb6de95690..a9284252c70 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -24,13 +24,37 @@ function jsonBytes(value: unknown): number { return encoder.encode(JSON.stringify(value)).length } +type SnapshotNode = ComputerUseSnapshot['nodes'][number] + +/** Layout wrappers that carry no text, state or action; their children are shown under the nearest meaningful ancestor. */ +const WRAPPER_ROLES = new Set([ + 'AXGroup', + 'AXSplitGroup', + 'AXLayoutArea', + 'AXLayoutItem', + 'AXUnknown', +]) + +function isWrapper(node: SnapshotNode): boolean { + return ( + WRAPPER_ROLES.has(node.role) && + !node.label && + !node.value && + !node.placeholder && + !node.focused && + !node.editable && + node.actions.length === 0 + ) +} + function nodeRow( - node: ComputerUseSnapshot['nodes'][number], + node: SnapshotNode, + parentId: string | undefined, + selectedWindowId: string, selected?: ReadonlySet ): string { const fields = [node.elementId, node.role] - if (node.parentId && (!selected || selected.has(node.parentId))) - fields.push(`parent=${node.parentId}`) + if (parentId && (!selected || selected.has(parentId))) fields.push(`parent=${parentId}`) if (node.label) fields.push(`label=${JSON.stringify(truncateAtCodePoint(node.label, 512, ''))}`) if (node.value) fields.push(`value=${JSON.stringify(truncateAtCodePoint(node.value, 512, ''))}`) if (node.placeholder) @@ -38,15 +62,17 @@ function nodeRow( if (node.focused) fields.push('focused') if (node.editable) fields.push('editable') if (node.enabled === false) fields.push('disabled') - if (node.actions.length) fields.push(`actions=${JSON.stringify(node.actions.slice(0, 16))}`) - if (node.windowId) fields.push(`window=${node.windowId}`) + if (node.actions.length) fields.push(`actions=${node.actions.slice(0, 16).join(',')}`) + if (node.windowId && node.windowId !== selectedWindowId) fields.push(`window=${node.windowId}`) if ( node.x !== undefined && node.y !== undefined && node.width !== undefined && node.height !== undefined ) - fields.push(`screenRect=${JSON.stringify([node.x, node.y, node.width, node.height])}`) + fields.push( + `screenRect=${[node.x, node.y, node.width, node.height].map((value) => Math.round(value)).join(',')}` + ) return fields.join(' ') } @@ -70,8 +96,10 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { : {}), windows, omittedWindowCount: allWindows.length - windows.length, + treeFormat: + 'Rows: elementId role [parent=id] [label|value|placeholder] [focused] [editable] [disabled] [actions=...] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.', accessibilityTree: '', - omittedNodeCount: nodes.length, + omittedNodeCount: nodes.filter((node) => !isWrapper(node)).length, nodeTextLimit: 512, ...(image ? { @@ -87,8 +115,25 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { model.omittedWindowCount += 1 } const byId = new Map(nodes.map((node) => [node.elementId, node])) - const rows = new Map(nodes.map((node) => [node.elementId, nodeRow(node)])) - const priority = (node: ComputerUseSnapshot['nodes'][number]) => { + const shown = nodes.filter((node) => !isWrapper(node)) + /** Each shown node's nearest shown ancestor, skipping wrappers (and stopping on a parent cycle). */ + const shownParent = new Map() + for (const node of shown) { + const seen = new Set([node.elementId]) + let parent = node.parentId ? byId.get(node.parentId) : undefined + while (parent && isWrapper(parent) && !seen.has(parent.elementId)) { + seen.add(parent.elementId) + parent = parent.parentId ? byId.get(parent.parentId) : undefined + } + shownParent.set(node.elementId, parent && !isWrapper(parent) ? parent.elementId : undefined) + } + const rows = new Map( + shown.map((node) => [ + node.elementId, + nodeRow(node, shownParent.get(node.elementId), state.windowId), + ]) + ) + const priority = (node: SnapshotNode) => { const visible = selectedWindow && node.windowId === selectedWindow.windowId && @@ -133,14 +178,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { bytes += cost return true } - for (const node of [...nodes].sort((left, right) => priority(right) - priority(left))) { + for (const node of [...shown].sort((left, right) => priority(right) - priority(left))) { const ancestry = [node.elementId] const seen = new Set(ancestry) - let parent = node.parentId - while (parent && byId.has(parent) && !seen.has(parent)) { + let parent = shownParent.get(node.elementId) + while (parent && !seen.has(parent)) { ancestry.push(parent) seen.add(parent) - parent = byId.get(parent)?.parentId + parent = shownParent.get(parent) } /** Keep the editor itself even if an unusually deep ancestry cannot fit. */ if (!add(ancestry.reverse())) add([node.elementId]) @@ -155,14 +200,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { ) || allWindows.some((window) => window.title.length > 256) return { ...model, - accessibilityTree: nodes + accessibilityTree: shown .filter((node) => selected.has(node.elementId)) - .map((node) => nodeRow(node, selected)) + .map((node) => nodeRow(node, shownParent.get(node.elementId), state.windowId, selected)) .join('\n'), - omittedNodeCount: nodes.length - selected.size, + omittedNodeCount: shown.length - selected.size, truncated: state.truncated || - selected.size < nodes.length || + selected.size < shown.length || windows.length < allWindows.length || clipped || screenshotOmitted, From d9e91f87b1457f0b207c8a5632b1b57a4cdbb2e7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 05:12:48 -0800 Subject: [PATCH 2/3] fix(computer-use): keep disabled layout groups in the model's accessibility tree --- .../tools/client/computer-tool-result.test.ts | 28 +++++++++++++++++++ .../tools/client/computer-tool-result.ts | 3 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts index bb3f17823de..a3c6780ddaa 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts @@ -227,6 +227,34 @@ describe('native computer model projection', () => { expect(result.omittedNodeCount).toBe(0) }) + it('keeps an empty layout group that reports itself disabled', () => { + const state = snapshot() + state.nodes = [ + { elementId: 'window', role: 'AXWindow', label: 'Fixture', actions: [], windowId: '1' }, + { + elementId: 'pane', + parentId: 'window', + role: 'AXGroup', + enabled: false, + actions: [], + windowId: '1', + }, + { + elementId: 'save', + parentId: 'pane', + role: 'AXButton', + label: 'Save', + actions: ['AXPress'], + windowId: '1', + }, + ] + const result = computerToolResultForModel(state) + if (!('accessibilityTree' in result)) throw new Error('Missing projected tree') + const rows = result.accessibilityTree.split('\n') + expect(rows.find((row) => row.startsWith('pane '))).toContain('disabled') + expect(rows.find((row) => row.startsWith('save '))).toContain('parent=pane') + }) + it('does not split an emoji when bounding text for the model', () => { const state = snapshot() state.nodes[0] = { diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index a9284252c70..937e7f631de 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -26,7 +26,7 @@ function jsonBytes(value: unknown): number { type SnapshotNode = ComputerUseSnapshot['nodes'][number] -/** Layout wrappers that carry no text, state or action; their children are shown under the nearest meaningful ancestor. */ +/** Layout wrappers that carry no text, state (including disabled) or action; their children are shown under the nearest meaningful ancestor. */ const WRAPPER_ROLES = new Set([ 'AXGroup', 'AXSplitGroup', @@ -43,6 +43,7 @@ function isWrapper(node: SnapshotNode): boolean { !node.placeholder && !node.focused && !node.editable && + node.enabled !== false && node.actions.length === 0 ) } From d3aa47e392daeef5f2c6e76a4c29c29a963a7d02 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 05:22:43 -0800 Subject: [PATCH 3/3] fix(computer-use): keep JSON-encoded action names so custom actions stay exact --- apps/sim/lib/mothership/tools/client/computer-tool-result.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 937e7f631de..0f53a975af9 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -63,7 +63,7 @@ function nodeRow( if (node.focused) fields.push('focused') if (node.editable) fields.push('editable') if (node.enabled === false) fields.push('disabled') - if (node.actions.length) fields.push(`actions=${node.actions.slice(0, 16).join(',')}`) + if (node.actions.length) fields.push(`actions=${JSON.stringify(node.actions.slice(0, 16))}`) if (node.windowId && node.windowId !== selectedWindowId) fields.push(`window=${node.windowId}`) if ( node.x !== undefined && @@ -98,7 +98,7 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { windows, omittedWindowCount: allWindows.length - windows.length, treeFormat: - 'Rows: elementId role [parent=id] [label|value|placeholder] [focused] [editable] [disabled] [actions=...] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.', + 'Rows: elementId role [parent=id] [label|value|placeholder] [focused] [editable] [disabled] [actions=JSON array of exact names] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.', accessibilityTree: '', omittedNodeCount: nodes.filter((node) => !isWrapper(node)).length, nodeTextLimit: 512,