From b4a119f96ad62e0e9bfd237ce5eee77e542d8dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 20:21:47 +0200 Subject: [PATCH] feat(ios): decide the presented surface from a captured tree (#3565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the rule that answers whether an iOS capture read a modal presentation over the app's own content, the golden table that pins it, and the publication that carries its answer, so the rule has a production consumer the moment it lands. The rule reads a dimming view without its enablement, unlike the occlusion cut that shares the same container class: a sheet resting at an undimmed detent keeps a dimming view that takes no touches and is still presented. `absent` is the strong answer and never the default, because a consumer arms a dismissal confirmation on it. It requires that the producer stated the capture was not capped, that every node carries the UIKit class name the rule reads, that each node is numbered once so the parent links the walk follows name the node they mean, and that the one content window under the app root holds exactly one presentation container. Positive evidence — the dimming view a presentation owns, an alert's own root class, or an element type that names a presentation — answers `presented` from a single node, so a capped capture still reports what it read. Producers that publish no class names, which is every XCTest runner tier, answer `presented` or `unknown` and never `absent`. The rule lives in `geometry-policy.ts` beside the occlusion cut rather than in `engine.ts`, because that module already owns the UIKit presentation vocabulary both readers consume and `geometry.ts` already imports it on the engine's eager path, so the placement costs nothing and leaves one module to change when UIKit does. Both readers now match class names through the same normalization, so a producer that spells a class with a module prefix cannot reach one answer in the occlusion cut and another here. Evidence is matched on the node occurrence a walk stood on, not on the number it carries, so a keyboard host's subtree cannot swallow an app window's own dimming view; only `absent`, a claim about every window the producer read, requires numbering that identifies each node once. The publication is in this layer rather than one up because `check:production-exports` refuses an exported rule with no production consumer, and a layer that ships a decision nobody can read is not a layer that passes. The Apple runner's Swift twin and its own consumer land in the next layer, where the runner acquires trees; both languages read the one fixture published here. --- contracts/fixtures/ios-presented-surface.json | 343 ++++++++++++++++++ .../src/ios-snapshot-engine/engine.test.ts | 118 ++++++ .../src/ios-snapshot-engine/engine.ts | 21 ++ .../geometry-policy-presented-surface.test.ts | 103 ++++++ .../ios-snapshot-engine/geometry-policy.ts | 267 +++++++++++++- ...snapshot-runtime-presented-surface.test.ts | 69 ++++ .../src/snapshot/ios-snapshot-runtime.ts | 1 + packages/contracts/src/ios-snapshot.ts | 10 + .../src/snapshot-source/tree.test.ts | 40 ++ 9 files changed, 963 insertions(+), 9 deletions(-) create mode 100644 contracts/fixtures/ios-presented-surface.json create mode 100644 packages/capture-kit/src/ios-snapshot-engine/geometry-policy-presented-surface.test.ts create mode 100644 packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-presented-surface.test.ts diff --git a/contracts/fixtures/ios-presented-surface.json b/contracts/fixtures/ios-presented-surface.json new file mode 100644 index 0000000000..c49844c2d4 --- /dev/null +++ b/contracts/fixtures/ios-presented-surface.json @@ -0,0 +1,343 @@ +{ + "description": "The iOS rule that decides whether a capture read a modal presentation over the app's own content (issue #3565), and the tree shapes it is pinned on. Source of truth shared by the TypeScript classifier (collectPresentedSurface in packages/capture-kit/src/ios-snapshot-engine/engine.ts) and the Swift one (SnapshotPresentedSurface in apple/snapshot-presentation); a change here must keep both parity tests green, and each side mirrors the vocabulary below by hand the way contracts/fixtures/ios-system-surface-hosts.json does. Every case states the answer for a capture whose producer reports UIKit class names (`expected`) and for the same tree read by a producer that reports none (`classBlindExpected`) — both tests run both readings. `presented` comes only from positive evidence: a dimming view, an alert-root class, or an element type that names a presentation itself. `absent` is the claim that every window the capture read showed only its own content, so it needs class facts, a completeness statement, one content window, and one container in it; anything else answers `unknown`. Each case's `note` says whether the shape was read from a device or is anticipated, and the anticipated ones are why `unknown` exists.", + "vocabulary": { + "presentationContainerClass": "UITransitionView", + "presentationContainerNote": "UIKit appends each modal presentation as a later sibling of the container it presents over, directly under the window, and presents the app's own content inside one too. A window carrying more than one container therefore holds a presentation of some kind; one container is just the window's content.", + "dimmingViewClass": "UIDimmingView", + "dimmingViewNote": "The dimming view a presentation dims the content behind it with. Only a presentation installs one, so wherever a capture reads one inside the app's own windows, something is presented. Its depth is not a fact this rule can rely on: the capture of the lab app's sheet holds one as a direct child of the presenting container AND one two levels deep under the app's own container's drop shadow, so the rule reads a dimming view as evidence without asking which container it sits in or whether it is a direct child of anything. A sheet resting at an undimmed detent keeps one with user interaction disabled, so this rule must NOT read its enablement — the occlusion cut in geometry-policy.ts is the reader that cares about touches.", + "alertRootClassPrefix": "_UIAlertController", + "alertRootClassNote": "Private root classes of a UIAlertController. An iPhone alert arrives in its own UIWindow with no dimming view anywhere, so the class is the only evidence that screen holds.", + "systemWindowClasses": ["UITextEffectsWindow", "UIRemoteKeyboardWindow"], + "systemWindowNote": "Windows UIKit hosts outside the app's scene, so they can carry none of its presentations and their internal shape says nothing about one. A capture taken with the keyboard up still reads as one content window. A denylist of the hosts iOS is known to name, deliberately not an allowlist of app content: an app hosting its scene in a UIWindow subclass stays readable, while an unlisted host carrying a dimming view of its own would read as presented. No such host has been observed.", + "presentingElementTypes": [ + { "name": "Sheet", "elementType": 5 }, + { "name": "Drawer", "elementType": 6 }, + { "name": "Alert", "elementType": 7 }, + { "name": "Dialog", "elementType": 8 } + ], + "presentingElementTypesNote": "Element types that name a presentation wherever they appear, keyed by the element-type numbering every iOS producer reads. A producer whose name table stops short of a name publishes `Element()` for the same node, so both spellings count and both are derived from these numbers rather than listed twice. The number must be written bare: `Element(+7)` is not a spelling of Alert, because a lenient numeric parse would let each language's integer parser decide the answer.", + "classNameMatchingNote": "Class names are matched after normalization, not byte for byte: trim, drop every case-insensitive `XCUIElementType`, drop a leading `AX`, lowercase, then keep only what follows the last `.` or `/`. That is `normalizeType` in packages/contracts/src/snapshot-text.ts, which the occlusion cut already matches UIKit classes through, and the Swift rule mirrors it. Producers spell one class differently — the bridge prefixes it with its module, the runner prefixes roles with AX — so a reader that compared raw bytes would answer a different question about the same capture than one that normalized. presentationContainerClass, dimmingViewClass, alertRootClassPrefix and systemWindowClasses are all read through it, so the spellings below name those classes however a producer writes them.", + "absentRequirementsNote": "absent additionally requires: `truncated` stated false (a producer that never observed completeness proves nothing about what it may have capped), a class name on every node (a producer that reports none can show no window structure), exactly one window outside systemWindowClasses (a second is a presentation UIKit hosts in its own window, or an overlay window, and the tree cannot tell those apart), and exactly one presentation container in that window (more than one is a full-screen or custom presentation, which this rule has no positive read on, and none means there is no container structure to read), and an index that identifies each node once (window structure is read through parent links, which cannot tell two nodes sharing a number apart). Positive evidence is exempt from that last requirement because it is keyed on the node occurrence it was read off, not on its number: a repeated index costs this rule the answer it declines to make, never a false absent." + }, + "cases": [ + { + "name": "home, one container per window", + "note": "Read from a device: iPhone 17 simulator, RN lab app, `snapshot --raw` over the simulator AX bridge. The app window carries a single UITransitionView with only a drop shadow, beside the keyboard host window.", + "truncated": false, + "expected": "absent", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "Other", "role": "UIDropShadowView" }, + { "index": 4, "parentIndex": 3, "type": "StaticText", "role": "RCTParagraphComponentView" }, + { "index": 5, "parentIndex": 0, "type": "Application", "role": "UITextEffectsWindow" }, + { "index": 6, "parentIndex": 5, "type": "Other", "role": "UIKeyboardLayoutStar" } + ] + }, + { + "name": "RN pageSheet modal, second container carrying a dimming view", + "note": "Read from a device: the same screen with the lab app's RN Modal (pageSheet) open. The window's second UITransitionView carries a UIDimmingView that takes touches. Through the XCTest runner the same sheet shows no positive evidence at all, which is the coverage gap this fact discloses rather than hides.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "RCTParagraphComponentView" }, + { "index": 4, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { + "index": 5, + "parentIndex": 4, + "type": "Other", + "role": "UIDimmingView", + "userInteractionEnabled": true + }, + { "index": 6, "parentIndex": 4, "type": "Other", "role": "UIDropShadowView" }, + { "index": 7, "parentIndex": 6, "type": "StaticText", "role": "RCTParagraphComponentView" }, + { "index": 8, "parentIndex": 0, "type": "Application", "role": "UITextEffectsWindow" } + ] + }, + { + "name": "sheet resting at an undimmed detent", + "note": "Anticipated, not read from a device: a sheet pulled down to a detent keeps its dimming view with user interaction disabled, so touches reach the content under it while the sheet is still presented. A dimming view's enablement must not change the answer.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { + "index": 5, + "parentIndex": 4, + "type": "Other", + "role": "UIDimmingView", + "userInteractionEnabled": false + }, + { "index": 6, "parentIndex": 4, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "RN alert in its own window, alert-root class, no dimming view", + "note": "Read from a device: the lab app's RN Alert.alert over UIAlertController. A second content window appears under the app root holding the alert, and no UIDimmingView exists anywhere in the capture, so only the alert's own class names say what it is.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "RCTParagraphComponentView" }, + { "index": 4, "parentIndex": 0, "type": "Application", "role": "UITextEffectsWindow" }, + { "index": 5, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 6, "parentIndex": 5, "type": "Other", "role": "UITransitionView" }, + { "index": 7, "parentIndex": 6, "type": "Other", "role": "UIDropShadowView" }, + { "index": 8, "parentIndex": 5, "type": "Other", "role": "UITransitionView" }, + { "index": 9, "parentIndex": 8, "type": "Group", "role": "_UIAlertControllerGradientView" }, + { + "index": 10, + "parentIndex": 9, + "type": "Button", + "role": "_UIAlertControllerLayoutObservingContainerView" + } + ] + }, + { + "name": "the same alert through a producer whose name table stops short", + "note": "Read from a device: that alert through the XCTest runner, which reports no class names and names raw element type 7 nothing, so the presenting node arrives as `Element(7)` — the shared numbering's spelling for Alert.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "presented", + "nodes": [ + { "index": 0, "type": "Application" }, + { "index": 1, "parentIndex": 0, "type": "Window" }, + { "index": 2, "parentIndex": 1, "type": "Other" }, + { "index": 3, "parentIndex": 0, "type": "Window" }, + { "index": 4, "parentIndex": 3, "type": "Element(7)" }, + { "index": 5, "parentIndex": 4, "type": "Button" }, + { "index": 6, "parentIndex": 4, "type": "Button" } + ] + }, + { + "name": "presentation named by its element type alone", + "note": "Anticipated, not read from a device: a host that reports the Sheet element type on the presenting node, in a tree whose window shape is otherwise unremarkable. The vocabulary test pins that Alert, Dialog and Drawer count the same way.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "presented", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "Sheet", "role": "RCTModalHostViewComponentView" } + ] + }, + { + "name": "second container with no dimming view", + "note": "Anticipated, not read from a device: a full-screen or custom presentation appends a container without dimming anything, and nothing else in the tree says whether it presents. Neither `presented` nor `absent` is supported here, which is why this case exists.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UIDropShadowView" }, + { "index": 6, "parentIndex": 5, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a window with no presentation container at all", + "note": "Anticipated, not read from a device: a window whose content sits directly under it has no container structure for this rule to read, so it cannot be called un-presented either.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a second content window this rule cannot name", + "note": "Anticipated, not read from a device: a React Native `` host window and an RN debug-build overlay window both arrive as a second ordinary window holding its own content, so a capture with two of them cannot be called un-presented.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UITransitionView" }, + { "index": 6, "parentIndex": 5, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a system window with internal presentation containers", + "note": "Anticipated, not read from a device: the keyboard host window's own view tree can hold containers of its own. It is outside the app's scene, so it neither presents the app nor stops absence, and a rule that counted containers across every root would be wrong here.", + "truncated": false, + "expected": "absent", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 0, "type": "Application", "role": "UIRemoteKeyboardWindow" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UITransitionView" }, + { "index": 6, "parentIndex": 5, "type": "Other", "role": "UIDimmingView" }, + { "index": 7, "parentIndex": 4, "type": "Other", "role": "UITransitionView" } + ] + }, + { + "name": "a capture whose producer never stated completeness", + "note": "Anticipated, not read from a device: an Appium page source or Limrun element tree reports nothing about truncation, so nothing about its windows can add up to absence even when the shape is ordinary. Positive evidence still answers, as the capped dimming-view case shows.", + "truncated": null, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a capture its producer capped", + "note": "Anticipated, not read from a device: a truncated tree proves nothing about the windows or containers it dropped, so it cannot be called un-presented even when everything it kept looks ordinary.", + "truncated": true, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a capped capture over a dimming view", + "note": "Anticipated, not read from a device: truncation blocks absence, not the positive half — a dimming view the producer did read is evidence of what it read.", + "truncated": true, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { + "index": 3, + "parentIndex": 2, + "type": "Other", + "role": "UIDimmingView", + "userInteractionEnabled": true + } + ] + }, + { + "name": "one node the producer reported no class name for", + "note": "Read from a device, then degraded: the lab app's ordinary home screen, with one node's class name dropped. The bridge omits the class name when the attribute is absent (snapshot-source/tree.ts), so a capture can be partly class-blind while looking ordinary. The bridge publishes the element type from the accessibility raw value while omitting a class name it did not read (snapshot-source/tree.ts: type without role), so a capture can be partly class-blind and still look ordinary. Absence is the claim that every window the capture read showed only its own content, and a tree that cannot name one of its own nodes cannot support it, however unremarkable the rest reads.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "Other", "role": "UIDropShadowView" }, + { "index": 4, "parentIndex": 3, "type": "StaticText" }, + { "index": 5, "parentIndex": 0, "type": "Application", "role": "UITextEffectsWindow" } + ] + }, + { + "name": "an element type spelled with a sign", + "note": "Malformed producer output, not read from a device: the same shape as an ordinary screen, with one node typed `Element(+7)`. A parser that reads a leading plus as a number would call this an Alert and answer presented; the answer has to come from the table's numbers, not from how lenient one language's integer parser is.", + "truncated": false, + "expected": "absent", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "Element(+7)", "role": "UIView" }, + { "index": 4, "parentIndex": 3, "type": "StaticText", "role": "UIView" } + ] + }, + { + "name": "a producer whose repeated index closes a cycle", + "note": "Malformed producer output, not read from a device: under the keyboard host window, a node repeats the index of an ancestor of itself, so following parent links walks that loop forever. Both languages have to terminate on it: the Swift classification runs outside the runner's bounded main-thread work, so no timeout would catch a spin there. The repeated index is also the answer: once two nodes share one, parent links no longer say which node is which, so the tree cannot show that every window holds only its own content.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 0, "type": "Application", "role": "UITextEffectsWindow" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UIKeyboardLayoutView" }, + { "index": 6, "parentIndex": 5, "type": "Other", "role": "UIKeyboardKeyView" }, + { "index": 7, "parentIndex": 6, "type": "Other", "role": "UIKeyboardKeyView" }, + { "index": 5, "parentIndex": 7, "type": "Other", "role": "UIKeyboardKeyView" } + ] + }, + { + "name": "an app node sharing its number with a node inside the keyboard host window", + "note": "Anticipated malformed output, not read from a device: the app window holds a real dimming view whose index repeats a number used inside the keyboard host window. Excluding that host reaches the node the walk stood on, not this one, so the app window's own read survives and answers presented — keying the exclusion on node numbers would discard a fact the producer plainly reported. Absence is still out of reach on a tree that numbers a node twice, which is what the second case below pins.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 0, "type": "Application", "role": "UIRemoteKeyboardWindow" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UIKeyboardLayoutView" }, + { "index": 5, "parentIndex": 1, "type": "Other", "role": "UIDimmingView" } + ] + }, + { + "name": "a dimming view hanging under a number two nodes carry", + "note": "Anticipated malformed output, not read from a device: the same repeated number one step further — the app window's dimming view reports the shared number as its parent, so the walk from the keyboard host reaches it through that link and cannot know it is standing on the app's own node. The exclusion does cover it here, and the tree answers unknown: it numbers a node twice, so window structure is unreadable in either direction.", + "truncated": false, + "expected": "unknown", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 0, "type": "Application", "role": "UIRemoteKeyboardWindow" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "UIKeyboardLayoutView" }, + { "index": 5, "parentIndex": 1, "type": "Other", "role": "UIDropShadowView" }, + { "index": 6, "parentIndex": 5, "type": "Other", "role": "UIDimmingView" } + ] + }, + { + "name": "a page sheet whose classes are spelled with module and AX prefixes", + "note": "Anticipated producer output, not read from a device: a page-sheet capture in which every UIKit class is written a little differently from the way this table names it — the app window carries its module prefix, the dimming view carries the AX prefix the runner adds, and the keyboard host window is lowercased. Normalization is what lets this read as presented with two containers in the app window and that host window excluded; a rule comparing class names byte for byte would call the same tree undecided, and the two languages would then disagree about one capture.", + "truncated": false, + "expected": "presented", + "classBlindExpected": "unknown", + "nodes": [ + { "index": 0, "type": "Application", "role": "UIApplication" }, + { "index": 1, "parentIndex": 0, "type": "Window", "role": "com.apple.internal.UIWindow" }, + { "index": 2, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 3, "parentIndex": 2, "type": "StaticText", "role": "UIView" }, + { "index": 4, "parentIndex": 1, "type": "Other", "role": "UITransitionView" }, + { "index": 5, "parentIndex": 4, "type": "Other", "role": "AXUIDimmingView" }, + { "index": 6, "parentIndex": 0, "type": "Application", "role": "uitexteffectswindow" }, + { "index": 7, "parentIndex": 6, "type": "Other", "role": "UIKeyboardLayoutStar" } + ] + } + ] +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index 86ed6bcbb7..7abb7f5f27 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -397,6 +397,77 @@ function acquisition( }; } +test("the published presented-surface fact is the acquisition's, for every projection", () => { + const requests = [ + createIosSnapshotRequest(), + createIosSnapshotRequest({ interactiveOnly: true }), + createIosSnapshotRequest({ raw: true, depth: 1 }), + createIosSnapshotRequest({ scope: 'Automation lab' }), + ]; + const published = requests.map((request) => + publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request), + ); + + // One screen answers one value: the fact is read on the acquired tree, so a projection that dropped + // the presentation from what it shows still reports the presentation the producer saw. + assert.deepEqual( + published.map((result) => result.presentedSurface), + ['presented', 'presented', 'presented', 'presented'], + ); + + // The depth-1 cut here projects a full acquisition: the presentation sits at depth 2 and 3, so what + // that capture publishes holds no trace of it. (A live `--raw --depth 1` bounds the acquisition + // instead and answers `unknown`.) Were the fact read off the published nodes, this projection would + // have to answer `absent` — which is the failure mode the fact exists to close. + const depthCut = published[2]!; + const publishedRoles = (depthCut.payload.nodes ?? []).map((node) => node.role); + assert.ok( + !publishedRoles.includes('UIDimmingView') && !publishedRoles.includes('UITransitionView'), + `expected the depth-1 cut to drop the presentation, saw ${JSON.stringify(publishedRoles)}`, + ); + assert.equal(depthCut.presentedSurface, 'presented'); +}); + +test('an acquisition that states no completeness cannot be read as un-presented', () => { + const request = createIosSnapshotRequest(); + const unstated: IosSnapshotAcquisition = { + ...acquisition(request, ordinaryTreeNodes()), + truncated: undefined, + }; + + assert.equal( + publishIosSnapshot({ stage: 'acquired', acquisition: unstated }, request).presentedSurface, + 'unknown', + ); + assert.equal( + publishIosSnapshot(acquiredInput(request, ordinaryTreeNodes()), request).presentedSurface, + 'absent', + ); +}); + +test('a runner-declared publication states no presented-surface fact of its own', () => { + const request = createIosSnapshotRequest(); + const input: IosSnapshotInput = { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { nodes: presentedTreeNodes(), truncated: false }, + }, + validation: validationFacts(request), + }; + + // The runner classifies its own pre-presentation acquisition and publishes the reading beside the + // payload; re-deriving one from the presented tree here would be a second, weaker answer. The same tree + // answers at the acquired stage, so this pins the stage gate and not a classifier that reads neither: + // a publication path that stopped classifying at all would fail the second assertion, not this one. + assert.equal(publishIosSnapshot(input, request).presentedSurface, undefined); + assert.equal( + publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request).presentedSurface, + 'presented', + ); +}); + function acquiredInput(request: IosSnapshotRequest, nodes: RawSnapshotNode[]): IosSnapshotInput { return { stage: 'acquired', acquisition: acquisition(request, nodes) }; } @@ -421,6 +492,53 @@ function nestedNodes(): RawSnapshotNode[] { ]; } +/** A UIKit-shaped tree over which a presentation is up: a second container carrying a dimming view. */ +function presentedTreeNodes(): RawSnapshotNode[] { + return [ + appNode(0, 'UIApplication', 'Application', viewport, undefined), + appNode(1, 'UIWindow', 'Window', viewport, 0), + appNode(2, 'UITransitionView', 'Other', viewport, 1, 2), + node(3, 'StaticText', 'Automation lab', { x: 10, y: 20, width: 200, height: 20 }, 2, 3), + appNode(4, 'UITransitionView', 'Other', viewport, 1, 2), + { + ...appNode(5, 'UIDimmingView', 'Other', viewport, 4, 3), + userInteractionEnabled: true, + }, + node(6, 'StaticText', 'Automation sheet', { x: 10, y: 120, width: 200, height: 20 }, 4, 3), + ]; +} + +/** A UIKit-shaped tree with nothing presenting: one window, its own single container. */ +function ordinaryTreeNodes(): RawSnapshotNode[] { + return [ + appNode(0, 'UIApplication', 'Application', viewport, undefined), + appNode(1, 'UIWindow', 'Window', viewport, 0), + appNode(2, 'UITransitionView', 'Other', viewport, 1), + appNode( + 3, + 'RCTParagraphComponentView', + 'StaticText', + { x: 10, y: 20, width: 200, height: 20 }, + 2, + ), + ]; +} + +/** A node shaped like the simulator AX bridge reports one: UIKit class name in `role`. */ +function appNode( + index: number, + role: string, + type: string, + rect: Rect, + parentIndex: number | undefined, + depth?: number, +): RawSnapshotNode { + return { + ...node(index, type, undefined, rect, parentIndex, depth), + role, + }; +} + function scopedNodes(): RawSnapshotNode[] { return [ node(0, 'Application', 'App', viewport), diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index 918960f2e3..551920f4a0 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -13,6 +13,7 @@ import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot' import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; import { validateIosSnapshotGraph } from './graph.ts'; import { foldIosSnapshot } from './geometry.ts'; +import { collectPresentedSurface, type PresentedSurfaceCompleteness } from './geometry-policy.ts'; import { resolveIosViewport, validateIosPayload } from './invariants.ts'; import { projectIosQualitySnapshot, projectIosSnapshot } from './projection.ts'; import { presentIosRunnerSnapshot } from './runner-presentation.ts'; @@ -49,9 +50,29 @@ export function publishIosSnapshot( ...(presentation.validatedViewport ? { validatedViewport: presentation.validatedViewport } : {}), + ...(input.stage === 'acquired' + ? { + presentedSurface: collectPresentedSurface( + input.acquisition.nodes, + acquisitionCompleteness(input.acquisition), + ), + } + : {}), }; } +/** + * What an acquisition claims about its own completeness, from the producer's truncation statement and + * never from the tree: a producer that reports nothing about truncation has stated nothing, and a + * presented-surface classification may not read a silent field as "complete" (#2188 invariant 5). + */ +function acquisitionCompleteness( + acquisition: IosSnapshotAcquisition, +): PresentedSurfaceCompleteness { + if (acquisition.truncated === undefined) return 'unstated'; + return acquisition.truncated ? 'capped' : 'complete'; +} + export function presentIosSnapshot( input: IosSnapshotInput, request: IosSnapshotRequest, diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy-presented-surface.test.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy-presented-surface.test.ts new file mode 100644 index 0000000000..cea22b0c2b --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy-presented-surface.test.ts @@ -0,0 +1,103 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { + ALERT_ROOT_CLASS_PREFIX, + PRESENTATION_CONTAINER_CLASS, + PRESENTATION_DIMMING_VIEW_CLASS, + PRESENTING_ELEMENT_TYPES, + SYSTEM_WINDOW_CLASSES, + collectPresentedSurface, + type PresentedSurfaceCompleteness, +} from './geometry-policy.ts'; + +const FIXTURE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'ios-presented-surface.json', +); + +type FixtureNode = { + index: number; + parentIndex?: number; + type: string; + role?: string; + label?: string; + userInteractionEnabled?: boolean; +}; + +type FixtureCase = { + name: string; + truncated: boolean | null; + expected: PresentedSurface; + classBlindExpected: PresentedSurface; + nodes: FixtureNode[]; +}; + +type Fixture = { + vocabulary: { + presentationContainerClass: string; + dimmingViewClass: string; + alertRootClassPrefix: string; + systemWindowClasses: string[]; + presentingElementTypes: Array<{ name: string; elementType: number }>; + }; + cases: FixtureCase[]; +}; + +function readFixture(): Fixture { + return JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as Fixture; +} + +// Cross-language parity: the classifier must mirror the canonical table the Swift +// SnapshotPresentedSurface also reads, and the vocabulary it mirrors by hand. A drift on either side +// fails here or in the Swift test. +test('the classifier mirrors the canonical vocabulary exactly', () => { + const vocabulary = readFixture().vocabulary; + expect(vocabulary.presentationContainerClass).toBe(PRESENTATION_CONTAINER_CLASS); + expect(vocabulary.dimmingViewClass).toBe(PRESENTATION_DIMMING_VIEW_CLASS); + expect(vocabulary.alertRootClassPrefix).toBe(ALERT_ROOT_CLASS_PREFIX); + expect(new Set(vocabulary.systemWindowClasses)).toEqual(SYSTEM_WINDOW_CLASSES); + expect(Object.fromEntries(PRESENTING_ELEMENT_TYPES)).toEqual( + Object.fromEntries( + vocabulary.presentingElementTypes.map((entry) => [entry.name, entry.elementType]), + ), + ); +}); + +// Every case runs twice: once with the UIKit class names the table spells, once with every class name +// stripped, which is what a capture from a producer that reports none — every XCTest runner tier — +// actually looks like. That reading is what `classBlindExpected` states, and it is why the table can +// hold a case a bridge capture answers `presented` and a runner capture answers `unknown`. +test('every golden case classifies as the table states', () => { + for (const testCase of readFixture().cases) { + const completeness = completenessOf(testCase); + expect(collectPresentedSurface(nodesOf(testCase), completeness)).toBe(testCase.expected); + expect(collectPresentedSurface(classBlindNodesOf(testCase), completeness)).toBe( + testCase.classBlindExpected, + ); + } +}); + +/** + * The producer's own completeness claim, which the table states as `truncated`: a null is a producer + * that observes nothing about truncation, and cannot be read as complete. + */ +function completenessOf(testCase: FixtureCase): PresentedSurfaceCompleteness { + if (testCase.truncated === null) return 'unstated'; + return testCase.truncated ? 'capped' : 'complete'; +} + +function nodesOf(testCase: FixtureCase): RawSnapshotNode[] { + return testCase.nodes.map((node) => ({ ...node })); +} + +function classBlindNodesOf(testCase: FixtureCase): RawSnapshotNode[] { + return testCase.nodes.map(({ role: _role, ...node }) => ({ ...node })); +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts index 8d1ffd155f..55b9d5bc37 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.ts @@ -1,4 +1,4 @@ -import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PresentedSurface, Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { isPositiveFiniteRect, rectContains } from '@agent-device/kernel/rect'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { collectChildrenByParent, collectSubtreeByParentLinks } from './tree.ts'; @@ -9,12 +9,62 @@ const VISIBILITY_CARRIER_TYPES = new Set(['application', 'window']); const NEGLIGIBLE_DECORATION_TOLERANCE = 1; /** - * Private UIKit classes are matched by exact name, unlike the substring role tests the occlusion - * pass uses: this rule decides node membership, so a near miss on a class name has to fail closed - * rather than widen the cut. + * The container UIKit appends per presentation, and the one it presents app content inside. This is the + * one place the name is written: the occlusion cut reads a LATER sibling of this class carrying a dimming + * view as the presentation that dims everything before it, and the presented-surface rule reads a window + * holding MORE than one of them as a capture it cannot call un-presented. A UIKit version that stopped + * using this class would degrade one reader to `unknown` and the other to no cut at all, and both + * failures are silent, so the pairing lives in the golden table's vocabulary and in both parity tests. */ -const PRESENTATION_CONTAINER_ROLE = 'uitransitionview'; -const PRESENTATION_DIMMING_ROLE = 'uidimmingview'; +export const PRESENTATION_CONTAINER_CLASS = 'UITransitionView'; + +/** + * The dimming view a presentation dims the content behind it with. Only a presentation installs one. The + * occlusion cut reads this node's `userInteractionEnabled` to decide what the user can still reach; the + * presented-surface rule deliberately does not, because a sheet resting at an undimmed detent keeps a + * dimming view that takes no touches and is still presented. + */ +export const PRESENTATION_DIMMING_VIEW_CLASS = 'UIDimmingView'; + +/** + * The private root classes of a UIAlertController. An iPhone alert arrives in its own window with no + * dimming view anywhere in the capture, so these classes are the only evidence that screen holds. + */ +export const ALERT_ROOT_CLASS_PREFIX = '_UIAlertController'; + +/** + * Windows UIKit hosts outside the app's own scene. They can carry none of the app's presentations, so + * neither the window nor anything under its subtree counts as evidence in either direction: a capture + * taken with the keyboard up still reads as one content window. + * + * This is a denylist of the hosts iOS is known to name, not a way to recognize app content. The direction + * is deliberate: an app that hosts its scene in a `UIWindow` subclass stays readable, while an allowlist + * would answer `unknown` for every screen it did not recognize. The cost is that an unlisted host which + * carries a dimming view of its own would read as `presented`; no such host has been observed. + */ +export const SYSTEM_WINDOW_CLASSES: ReadonlySet = new Set([ + 'UITextEffectsWindow', + 'UIRemoteKeyboardWindow', +]); + +/** + * Element types that name a presentation wherever they appear, keyed by the shared element-type + * numbering every iOS producer reads. A producer whose own name table stops short of the name publishes + * `Element()` for the same node, so both spellings count; the raw-value spelling is derived from + * these numbers rather than listed beside them (#3565). + * + * The numbers belong to `ELEMENT_TYPE_NAMES` in `packages/platform-apple/src/snapshot-source/tree.ts`, + * which is the table that turns an accessibility raw value into one of these names, and nothing imports + * across that boundary — capture-kit sits below platform-apple. The golden table's vocabulary is the tie + * that keeps the two in agreement: its parity test is read both by that package's own test, which + * re-derives each name's index from the owner's table, and by both classifier parity tests. + */ +export const PRESENTING_ELEMENT_TYPES: ReadonlyMap = new Map([ + ['Sheet', 5], + ['Drawer', 6], + ['Alert', 7], + ['Dialog', 8], +]); export type TraversalState = Readonly<{ projectedOut: boolean; @@ -292,13 +342,212 @@ function isDimmedByPresentation(covering: CoveringPresentation, sibling: RawSnap } function isPresentationContainer(node: RawSnapshotNode): boolean { - return hasRole(node, PRESENTATION_CONTAINER_ROLE); + return hasRole(node, PRESENTATION_CONTAINER_CLASS); } function isPresentationDimmingView(node: RawSnapshotNode): boolean { - return hasRole(node, PRESENTATION_DIMMING_ROLE); + return hasRole(node, PRESENTATION_DIMMING_VIEW_CLASS); } +/** + * Roles are matched through `normalizeType`, which is what lets the occlusion cut and the + * presented-surface rule agree on a class a producer spelled with a module prefix or in another case: + * one predicate, so neither reader can drift to accepting a name the other refuses. + */ function hasRole(node: RawSnapshotNode, role: string): boolean { - return normalizeType(node.role ?? '') === role; + return normalizeType(node.role ?? '') === normalizeType(role); +} + +/** + * What the capture's producer claims about its own completeness, stated by the caller from the + * producer's truncation evidence instead of inferred from the tree: `complete` when the producer reports + * it is not capped, `capped` when it reports a cap, `unstated` when it observes nothing either way. + */ +export type PresentedSurfaceCompleteness = 'complete' | 'capped' | 'unstated'; + +/** + * Whether a capture read a modal presentation over the app's own content (#3565), decided on the + * acquired tree so the interactive, full, depth-cut and scoped projections of one acquisition all + * answer the same value. A raw `--depth` bounds the acquisition itself; one too shallow to reach the + * presentation's windows answers `unknown`, never `absent`. + * + * UIKit appends each modal presentation as a later sibling of the container it presents over, directly + * under the window, and dims what is behind it — the same shape `findCoveringPresentation` above rests + * on when it drops dimmed content. That reader decides which nodes a user can still reach and so reads a + * dimming view's enablement; this one decides only whether something is presented and must NOT read it: a + * sheet resting at an undimmed detent keeps a dimming view that takes no touches and is still presented. + * + * `absent` is the strong answer and is never the default, because a consumer arms a dismissal + * confirmation on it. It requires that the producer stated the capture was not capped, that every node + * carries the UIKit class name this rule reads, that each node is numbered once so the parent links this + * walk follows name the node they mean, and that the one content window under the app root holds exactly + * one presentation container. Producers that report no class names — every XCTest runner tier — answer + * `presented` or `unknown`, never `absent`. + * + * The cross-language rule and the tree shapes it is pinned on are the golden table + * `contracts/fixtures/ios-presented-surface.json`; the Swift `SnapshotPresentedSurface` mirrors this + * vocabulary and each side has a parity test against that table. + */ +export function collectPresentedSurface( + nodes: readonly RawSnapshotNode[], + completeness: PresentedSurfaceCompleteness, +): PresentedSurface { + const windowPositions = appWindowPositions(nodes); + const examinable = positionsOutsideSystemWindows(nodes, windowPositions); + // Positive evidence is evidence of what the producer did read, so a cap does not defeat it; absence + // is what a cap defeats, along with any capture that cannot state its own window structure. + if (nodes.some((node, position) => examinable.has(position) && isPresentingEvidence(node))) { + return 'presented'; + } + if (completeness !== 'complete') return 'unknown'; + if (windowPositions.length === 0 || !nodes.every((node) => node.role !== undefined)) { + return 'unknown'; + } + // `absent` is a claim about every window the producer read, and window structure is read through + // parent links, which cannot tell two nodes sharing a number apart. Positive evidence is already + // reported above, keyed on the node it was read off rather than on its number, so the most a + // collision can cost this rule is the answer it declines to make — never a false `absent`. + if (!identifiesEachNodeOnce(nodes)) return 'unknown'; + + const contentWindows = windowPositions.filter((position) => !isSystemWindow(nodes[position]!)); + if (contentWindows.length !== 1) return 'unknown'; + return presentationContainersIn(nodes[contentWindows[0]!]!, nodes).length === 1 + ? 'absent' + : 'unknown'; +} + +/** + * Positive evidence of a presentation: the dimming view a presentation owns, an alert's own root class, + * or an element type that names a presentation. Each is a per-node fact, so a tree that holds one + * answers without any window structure to read. + */ +function isPresentingEvidence(node: RawSnapshotNode): boolean { + return isPresentationDimmingView(node) || isAlertRoot(node) || isPresentingElementType(node); +} + +function isAlertRoot(node: RawSnapshotNode): boolean { + const role = node.role; + if (role === undefined) return false; + const normalized = normalizeType(role); + return normalized.startsWith(normalizeType(ALERT_ROOT_CLASS_PREFIX)); +} + +function isPresentingElementType(node: RawSnapshotNode): boolean { + const type = node.type; + if (!type) return false; + if (PRESENTING_ELEMENT_TYPES.has(type)) return true; + const reportedRawValue = UNNAMED_ELEMENT_TYPE.exec(type); + return reportedRawValue !== null && PRESENTING_RAW_VALUES.has(Number(reportedRawValue[1])); +} + +/** The name a producer publishes for an element type its own table does not cover. */ +const UNNAMED_ELEMENT_TYPE = /^Element\((\d+)\)$/; + +const PRESENTING_RAW_VALUES: ReadonlySet = new Set(PRESENTING_ELEMENT_TYPES.values()); + +function isSystemWindow(node: RawSnapshotNode): boolean { + const role = node.role; + if (role === undefined) return false; + const normalized = normalizeType(role); + for (const systemWindowClass of SYSTEM_WINDOW_CLASSES) { + if (normalizeType(systemWindowClass) === normalized) return true; + } + return false; +} + +/** + * Whether every node carries its own number, which is what lets a parent-link walk name the node it is + * standing on. Positive evidence survives a repeated number — it is keyed on the node it was read off — + * but window structure is read through those links, so a producer that numbered a node twice cannot + * support a claim about every window it read, and such a tree is answered here rather than walked + * further. The engine's own fold refuses this tree outright, so the regular route never reaches here with + * one; the guard is what keeps the answer honest when the classifier is called on a tree that has not + * passed through it — which is the Apple runner's route, where the classification runs on the acquisition + * and no graph validation happens first. Both languages answer `unknown` for the same reason, which is + * the parity the golden table pins. + */ +function identifiesEachNodeOnce(nodes: readonly RawSnapshotNode[]): boolean { + return new Set(nodes.map((node) => node.index)).size === nodes.length; +} + +/** + * The app's windows: the nodes directly under its root, as the places in `nodes` they occupy. Both iOS + * acquisition tiers report the app root and then one node per window, so finding them needs no class + * facts — which is what lets a class-blind capture still be examined for the positive evidence it does + * report. + */ +function appWindowPositions(nodes: readonly RawSnapshotNode[]): number[] { + const rootPositions: number[] = []; + for (const [position, node] of nodes.entries()) { + if (node.parentIndex === undefined) rootPositions.push(position); + } + if (rootPositions.length !== 1) return []; + const rootIndex = nodes[rootPositions[0]!]!.index; + const windows: number[] = []; + for (const [position, node] of nodes.entries()) { + if (node.parentIndex === rootIndex) windows.push(position); + } + return windows; +} + +/** + * The nodes to read evidence from: every node except the ones the walk reached from a window UIKit hosts + * outside the app's scene, keyed by the place in `nodes` each occupies rather than by its number. A walk + * has to reach the node it is standing on, and two nodes sharing a number cannot be told apart by the + * links that reach them; keying on the number would let a keyboard host's subtree swallow an app window's + * own evidence, which is a direct read the rule would then throw away. Keying on the occurrence keeps that + * read, and the numbering guard above is what stops such a tree from earning absence. A capture with no + * class facts cannot single any window out, which is the honest reading: nothing in it is excludable, and + * nothing in it can earn absence either. + */ +function positionsOutsideSystemWindows( + nodes: readonly RawSnapshotNode[], + windowPositions: readonly number[], +): ReadonlySet { + const positionsByParentIndex = new Map(); + for (const [position, node] of nodes.entries()) { + if (typeof node.parentIndex !== 'number') continue; + const siblings = positionsByParentIndex.get(node.parentIndex) ?? []; + siblings.push(position); + positionsByParentIndex.set(node.parentIndex, siblings); + } + const examinable = new Set(nodes.map((_node, position) => position)); + for (const windowPosition of windowPositions) { + if (!isSystemWindow(nodes[windowPosition]!)) continue; + for (const occurrence of subtreePositions(windowPosition, nodes, positionsByParentIndex)) { + examinable.delete(occurrence); + } + } + return examinable; +} + +/** + * A node and everything reachable from it through parent links, as the places in `nodes` they occupy. + * Reported depths are not needed. Visited is keyed on the occurrence and seeded with the root: a producer + * that closes a loop deeper in the tree is one this walk would follow forever, and the classification runs + * outside the Apple runner's bounded main-thread work, where no timeout would catch the spin. The golden + * table's cycle case is pinned on exactly that shape, in both languages. + */ +function subtreePositions( + rootPosition: number, + nodes: readonly RawSnapshotNode[], + positionsByParentIndex: ReadonlyMap, +): number[] { + const collected: number[] = [rootPosition]; + const visited = new Set([rootPosition]); + for (let walked = 0; walked < collected.length; walked += 1) { + for (const position of positionsByParentIndex.get(nodes[collected[walked]!]!.index) ?? []) { + if (visited.has(position)) continue; + visited.add(position); + collected.push(position); + } + } + return collected; +} + +function presentationContainersIn( + window: RawSnapshotNode, + nodes: readonly RawSnapshotNode[], +): RawSnapshotNode[] { + return nodes.filter((node) => node.parentIndex === window.index && isPresentationContainer(node)); } diff --git a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-presented-surface.test.ts b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-presented-surface.test.ts new file mode 100644 index 0000000000..93e7d44c10 --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-presented-surface.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { presentIosSnapshotAcquisition } from '../ios-snapshot-runtime.ts'; +import { createIosSnapshotRequest, deriveIosCaptureHint } from '../../ios-snapshot-planning.ts'; +import type { IosSnapshotAcquisition } from '@agent-device/contracts/ios-snapshot'; +import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +// #3565: this seam is where an acquisition becomes the snapshot result the daemon, the Linux provider +// and the macOS desktop consume, so it is where the presented-surface fact enters `SnapshotResult`. +// Nothing else reaches it: the engine's own tests stop at the publication it builds, and the carriage +// tests enter above it through a backend, so a projection that dropped the field here would leave a +// settle confirmation reading silence as "no producer spoke". + +const SCREEN = { x: 0, y: 0, width: 390, height: 844 }; + +/** One window, one presentation container, and the dimming view that container owns. */ +const PRESENTED_NODES: readonly RawSnapshotNode[] = [ + { index: 0, type: 'Application', role: 'UIApplication', rect: SCREEN }, + { index: 1, parentIndex: 0, type: 'Window', role: 'UIWindow', rect: SCREEN }, + { index: 2, parentIndex: 1, type: 'Other', role: 'UITransitionView', rect: SCREEN }, + { + index: 3, + parentIndex: 2, + type: 'Other', + role: 'UIDimmingView', + rect: SCREEN, + userInteractionEnabled: true, + }, +]; + +/** The same tree as a producer that reports no UIKit class names publishes it. */ +const CLASS_BLIND_NODES: readonly RawSnapshotNode[] = PRESENTED_NODES.map( + ({ role: _role, ...node }) => node, +); + +/** + * Both cases state that nothing was capped, so the answers below differ by class evidence alone and + * not by a completeness statement the producer never made. + */ +function acquiring(nodes: readonly RawSnapshotNode[]): SnapshotRuntimeAcquiredResult { + const hint = deriveIosCaptureHint(createIosSnapshotRequest({})); + const acquisition: IosSnapshotAcquisition = { + producer: 'simulator-ax-bridge', + intent: 'full', + hint: { ...hint, acquisitionIntent: 'full' }, + nodes, + truncated: false, + viewport: { kind: 'reported', rect: SCREEN }, + lineage: { targetId: 'simulator-1', generation: 'generation-1' }, + residue: [], + }; + return { stage: 'acquired', acquisition }; +} + +test('the snapshot result carries the presented-surface fact of the tree it projects (#3565)', () => { + const presented = presentIosSnapshotAcquisition(acquiring(PRESENTED_NODES)); + + assert.equal(presented.presentedSurface, 'presented'); +}); + +// A capture whose producer reports no class names can show no window structure, so it cannot earn +// `absent`. It answers `unknown` rather than nothing, which is what lets a consumer tell "this +// producer could not read it" apart from "no producer classified this capture at all". +test('a class-blind acquisition answers unknown rather than silence (#3565)', () => { + const presented = presentIosSnapshotAcquisition(acquiring(CLASS_BLIND_NODES)); + + assert.equal(presented.presentedSurface, 'unknown'); +}); diff --git a/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts b/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts index 04d1c8a02c..7aa8d66a3b 100644 --- a/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts +++ b/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts @@ -55,6 +55,7 @@ export function presentIosSnapshotAcquisition( ? {} : { truncated: acquired.acquisition.truncated }), ...(validatedViewport ? { viewport: validatedViewport } : {}), + ...(presentation.presentedSurface ? { presentedSurface: presentation.presentedSurface } : {}), ...snapshotWarnings(acquired.acquisition.residue), }; } catch (error) { diff --git a/packages/contracts/src/ios-snapshot.ts b/packages/contracts/src/ios-snapshot.ts index a608372385..0e1a717c54 100644 --- a/packages/contracts/src/ios-snapshot.ts +++ b/packages/contracts/src/ios-snapshot.ts @@ -1,4 +1,5 @@ import type { + PresentedSurface, RawSnapshotNode, Rect, SnapshotNode, @@ -230,6 +231,15 @@ export type IosSnapshotPublication = Readonly<{ * that measured against it. A raw projection validates no box and carries none. */ validatedViewport?: Rect; + /** + * Whether this capture read a modal presentation over the app (#3565), classified on the acquired + * tree so every projection of one acquisition — interactive, full, depth-cut, scoped — answers the + * same value. A raw `--depth` bounds the acquisition itself, so it can read too little and answer + * `unknown`. Only an acquisition the engine itself classified carries it: a runner-declared capture + * arrives already presented, and the runner's own reading of its pre-presentation acquisition + * travels on the payload beside it. + */ + presentedSurface?: PresentedSurface; }>; export type IosSnapshotEngine = Readonly<{ diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index d8c29df4eb..5766b63daa 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; import { SnapshotSourceError } from './errors.ts'; import { decodeSnapshotBridgeTree } from './tree.ts'; @@ -650,3 +652,41 @@ test('the AX bridge publishes native switch and text entry facts without inventi assert.deepEqual([tree[3]!.enabled, tree[3]!.editable, tree[3]!.password], [false, true, false]); assert.equal(tree[4]!.checked, undefined); }); + +/** + * The presented-surface rule reads Sheet, Drawer, Alert and Dialog as positive evidence and derives the + * `Element()` spelling from the same numbers (#3565). Those numbers are the name table above, and the + * rule lives in capture-kit below this package, so nothing imports across that boundary. This decodes + * each raw value the way a real capture does and holds the published name against the golden table both + * classifier parity tests read, so a renumbering that leaves the rule behind fails at the owner. + */ +test('the element-type names the presented-surface rule reads are the ones this table publishes', () => { + const fixture = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'ios-presented-surface.json', + ); + const { presentingElementTypes } = JSON.parse(fs.readFileSync(fixture, 'utf8')).vocabulary as { + presentingElementTypes: Array<{ name: string; elementType: number }>; + }; + assert.ok(presentingElementTypes.length > 0); + + for (const { name, elementType: rawValue } of presentingElementTypes) { + const decoded = decodeSnapshotBridgeTree( + { + [application]: 'Application', + [frame]: { X: 0, Y: 0, Width: 390, Height: 844 }, + [children]: [{ [automationType]: rawValue, [children]: [] }], + }, + { truncated: false }, + limits, + ); + + assert.equal(decoded.nodes[1]?.type, name, `raw element type ${rawValue}`); + } +});