improvement(computer-use): fold empty layout groups and compact the accessibility tree shown to the model (#9997)

* improvement(computer-use): fold empty layout groups and compact rows in the model's accessibility tree

Wrapper groups with no text, state or action no longer spend the observation budget;
their children point at the nearest shown ancestor. Rows drop the selected window id,
JSON array syntax and fractional coordinates, and a one-line legend explains the format.

* fix(computer-use): keep disabled layout groups in the model's accessibility tree

* fix(computer-use): keep JSON-encoded action names so custom actions stay exact

Author: waleedlatif1

apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts

@@ -198,6 +198,63 @@ describe('native computer model projection', () => {
     expect(result).toMatchObject({ dispatched: true, sequence: { completedSteps: 1 } })
   })
 
+  it('folds empty layout groups so their controls fit and point at a shown parent', () => {
+    const state = snapshot()
+    state.nodes = [
+      { elementId: 'window', role: 'AXWindow', label: 'Fixture', actions: [], windowId: '1' },
+      ...Array.from({ length: 300 }, (_, index) => ({
+        elementId: `wrap-${index}`,
+        parentId: index === 0 ? 'window' : `wrap-${index - 1}`,
+        role: 'AXGroup',
+        actions: [],
+        windowId: '1',
+      })),
+      ...Array.from({ length: 150 }, (_, index) => ({
+        elementId: `button-${index}`,
+        parentId: 'wrap-299',
+        role: 'AXButton',
+        label: `Action ${index}`,
+        actions: ['AXPress'],
+        windowId: '1',
+      })),
+    ]
+    const result = computerToolResultForModel(state)
+    if (!('accessibilityTree' in result)) throw new Error('Missing projected tree')
+    const rows = result.accessibilityTree.split('\n')
+    expect(rows.some((row) => row.includes('AXGroup'))).toBe(false)
+    expect(rows.filter((row) => row.includes('AXButton'))).toHaveLength(150)
+    expect(rows.find((row) => row.startsWith('button-0 '))).toContain('parent=window')
+    expect(result.omittedNodeCount).toBe(0)
+  })
+
+  it('keeps an empty layout group that reports itself disabled', () => {
+    const state = snapshot()
+    state.nodes = [
+      { elementId: 'window', role: 'AXWindow', label: 'Fixture', actions: [], windowId: '1' },
+      {
+        elementId: 'pane',
+        parentId: 'window',
+        role: 'AXGroup',
+        enabled: false,
+        actions: [],
+        windowId: '1',
+      },
+      {
+        elementId: 'save',
+        parentId: 'pane',
+        role: 'AXButton',
+        label: 'Save',
+        actions: ['AXPress'],
+        windowId: '1',
+      },
+    ]
+    const result = computerToolResultForModel(state)
+    if (!('accessibilityTree' in result)) throw new Error('Missing projected tree')
+    const rows = result.accessibilityTree.split('\n')
+    expect(rows.find((row) => row.startsWith('pane '))).toContain('disabled')
+    expect(rows.find((row) => row.startsWith('save '))).toContain('parent=pane')
+  })
+
   it('does not split an emoji when bounding text for the model', () => {
     const state = snapshot()
     state.nodes[0] = {

apps/sim/lib/mothership/tools/client/computer-tool-result.ts

@@ -24,13 +24,38 @@ function jsonBytes(value: unknown): number {
   return encoder.encode(JSON.stringify(value)).length
 }
 
+type SnapshotNode = ComputerUseSnapshot['nodes'][number]
+
+/** Layout wrappers that carry no text, state (including disabled) or action; their children are shown under the nearest meaningful ancestor. */
+const WRAPPER_ROLES = new Set([
+  'AXGroup',
+  'AXSplitGroup',
+  'AXLayoutArea',
+  'AXLayoutItem',
+  'AXUnknown',
+])
+
+function isWrapper(node: SnapshotNode): boolean {
+  return (
+    WRAPPER_ROLES.has(node.role) &&
+    !node.label &&
+    !node.value &&
+    !node.placeholder &&
+    !node.focused &&
+    !node.editable &&
+    node.enabled !== false &&
+    node.actions.length === 0
+  )
+}
+
 function nodeRow(
-  node: ComputerUseSnapshot['nodes'][number],
+  node: SnapshotNode,
+  parentId: string | undefined,
+  selectedWindowId: string,
   selected?: ReadonlySet<string>
 ): string {
   const fields = [node.elementId, node.role]
-  if (node.parentId && (!selected || selected.has(node.parentId)))
-    fields.push(`parent=${node.parentId}`)
+  if (parentId && (!selected || selected.has(parentId))) fields.push(`parent=${parentId}`)
   if (node.label) fields.push(`label=${JSON.stringify(truncateAtCodePoint(node.label, 512, ''))}`)
   if (node.value) fields.push(`value=${JSON.stringify(truncateAtCodePoint(node.value, 512, ''))}`)
   if (node.placeholder)
@@ -39,14 +64,16 @@ function nodeRow(
   if (node.editable) fields.push('editable')
   if (node.enabled === false) fields.push('disabled')
   if (node.actions.length) fields.push(`actions=${JSON.stringify(node.actions.slice(0, 16))}`)
-  if (node.windowId) fields.push(`window=${node.windowId}`)
+  if (node.windowId && node.windowId !== selectedWindowId) fields.push(`window=${node.windowId}`)
   if (
     node.x !== undefined &&
     node.y !== undefined &&
     node.width !== undefined &&
     node.height !== undefined
   )
-    fields.push(`screenRect=${JSON.stringify([node.x, node.y, node.width, node.height])}`)
+    fields.push(
+      `screenRect=${[node.x, node.y, node.width, node.height].map((value) => Math.round(value)).join(',')}`
+    )
   return fields.join(' ')
 }
 
@@ -70,8 +97,10 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
       : {}),
     windows,
     omittedWindowCount: allWindows.length - windows.length,
+    treeFormat:
+      'Rows: elementId role [parent=id] [label|value|placeholder] [focused] [editable] [disabled] [actions=JSON array of exact names] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.',
     accessibilityTree: '',
-    omittedNodeCount: nodes.length,
+    omittedNodeCount: nodes.filter((node) => !isWrapper(node)).length,
     nodeTextLimit: 512,
     ...(image
       ? {
@@ -87,8 +116,25 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
     model.omittedWindowCount += 1
   }
   const byId = new Map(nodes.map((node) => [node.elementId, node]))
-  const rows = new Map(nodes.map((node) => [node.elementId, nodeRow(node)]))
-  const priority = (node: ComputerUseSnapshot['nodes'][number]) => {
+  const shown = nodes.filter((node) => !isWrapper(node))
+  /** Each shown node's nearest shown ancestor, skipping wrappers (and stopping on a parent cycle). */
+  const shownParent = new Map<string, string | undefined>()
+  for (const node of shown) {
+    const seen = new Set([node.elementId])
+    let parent = node.parentId ? byId.get(node.parentId) : undefined
+    while (parent && isWrapper(parent) && !seen.has(parent.elementId)) {
+      seen.add(parent.elementId)
+      parent = parent.parentId ? byId.get(parent.parentId) : undefined
+    }
+    shownParent.set(node.elementId, parent && !isWrapper(parent) ? parent.elementId : undefined)
+  }
+  const rows = new Map(
+    shown.map((node) => [
+      node.elementId,
+      nodeRow(node, shownParent.get(node.elementId), state.windowId),
+    ])
+  )
+  const priority = (node: SnapshotNode) => {
     const visible =
       selectedWindow &&
       node.windowId === selectedWindow.windowId &&
@@ -133,14 +179,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
     bytes += cost
     return true
   }
-  for (const node of [...nodes].sort((left, right) => priority(right) - priority(left))) {
+  for (const node of [...shown].sort((left, right) => priority(right) - priority(left))) {
     const ancestry = [node.elementId]
     const seen = new Set(ancestry)
-    let parent = node.parentId
-    while (parent && byId.has(parent) && !seen.has(parent)) {
+    let parent = shownParent.get(node.elementId)
+    while (parent && !seen.has(parent)) {
       ancestry.push(parent)
       seen.add(parent)
-      parent = byId.get(parent)?.parentId
+      parent = shownParent.get(parent)
     }
     /** Keep the editor itself even if an unusually deep ancestry cannot fit. */
     if (!add(ancestry.reverse())) add([node.elementId])
@@ -155,14 +201,14 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
     ) || allWindows.some((window) => window.title.length > 256)
   return {
     ...model,
-    accessibilityTree: nodes
+    accessibilityTree: shown
       .filter((node) => selected.has(node.elementId))
-      .map((node) => nodeRow(node, selected))
+      .map((node) => nodeRow(node, shownParent.get(node.elementId), state.windowId, selected))
       .join('\n'),
-    omittedNodeCount: nodes.length - selected.size,
+    omittedNodeCount: shown.length - selected.size,
     truncated:
       state.truncated ||
-      selected.size < nodes.length ||
+      selected.size < shown.length ||
       windows.length < allWindows.length ||
       clipped ||
       screenshotOmitted,