diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ea8d6d04..1ab58f835c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,5 @@ ## Unreleased +- Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - Carry provider keyboard facts through snapshots. Appium captures, including wait polls and fill pre-captures, add one sequential keyboard visibility probe with a 500 ms timeout. Probe results preserve tap-time docked-tree checks; unavailable probes leave visibility unknown. diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts index a42176d710..dd8525151b 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts @@ -169,6 +169,30 @@ test('Limrun carries docked tree evidence and never infers absence from missing ).toEqual({ kind: 'tree-rejected', derivation: 'docked-tree' }); }); +test('Limrun acquires a system photo picker whose tabs report numeric values', async () => { + // PHPickerViewController reports AXValue 1/0 as numbers, and marks a tapped photo with the + // Selected trait. Both are variant JSON the raw tree hands over untyped. + const result = await captureLimrunIosSnapshot( + createLimrunSnapshotSession({ + elementType: 'Other', + children: [ + { elementType: 'RadioButton', AXLabel: 'Photos', AXValue: 1 }, + { elementType: 'RadioButton', AXLabel: 'Collections', AXValue: 0 }, + { + elementType: 'Image', + AXLabel: 'Photo', + AXUniqueId: 12, + traits: ['Scrollable', 'Selected', 'Image'], + }, + ], + }), + ); + const [photos, collections, photo] = result.acquisition.nodes.slice(1); + expect(photos).toMatchObject({ value: '1', checked: true }); + expect(collections).toMatchObject({ value: '0', checked: false }); + expect(photo).toMatchObject({ selected: true, label: 'Photo', identifier: '12' }); +}); + test('limrun acquisition shares RN interpretation and preserves native values', async () => { const result = await captureLimrunIosSnapshot( createLimrunSnapshotSession({ diff --git a/packages/provider-limrun/src/snapshot.test.ts b/packages/provider-limrun/src/snapshot.test.ts index 02e0cfd493..cfc9827f33 100644 --- a/packages/provider-limrun/src/snapshot.test.ts +++ b/packages/provider-limrun/src/snapshot.test.ts @@ -15,6 +15,39 @@ test('visible text targets the provider label field, which is where the tree exp expect(toIosSelector({ key: 'text', value: 'Sign In' })).toEqual({ AXLabel: 'Sign In' }); }); +test('variant scalar attributes become the text every snapshot producer publishes', () => { + // The system photo picker's segmented tabs report the numbers 1/0 as AXValue, and the + // provider's JSON can carry a variant scalar in any text attribute. + const [selected, unselected, boolean, text, empty, missing, fallback, label, identifier] = + flattenIosTree([ + { type: 'RadioButton', AXLabel: 'Photos', AXValue: 1 }, + { type: 'RadioButton', AXLabel: 'Collections', AXValue: 0 }, + { type: 'Switch', value: true }, + { type: 'TextField', AXValue: 'qa@example.com' }, + { type: 'TextField', value: '' }, + { type: 'TextField', AXValue: null }, + { type: 'TextField', value: '', AXValue: 1 }, + { type: 'Other', label: 42, AXLabel: 'fallback' }, + { type: 'Other', identifier: true, AXUniqueId: 'fallback' }, + ]); + expect([selected?.value, unselected?.value, boolean?.value]).toEqual(['1', '0', 'true']); + expect(text?.value).toBe('qa@example.com'); + expect([empty?.value, missing?.value]).toEqual([undefined, undefined]); + expect(fallback?.value).toBe('1'); + expect([label?.label, identifier?.identifier]).toEqual(['42', 'true']); +}); + +test('the Selected trait publishes selection in the shape the Apple tree uses', () => { + const [selected, unselected, explicit] = flattenIosTree([ + { type: 'Image', traits: ['Scrollable', 'Selected', 'Image'] }, + { type: 'Image', traits: ['Image'] }, + { type: 'Image', selected: true }, + ]); + expect(selected).toHaveProperty('selected', true); + expect(unselected?.selected).toBeUndefined(); + expect(explicit).toHaveProperty('selected', true); +}); + test('Limrun raw mapping forwards observed Header traits and SDK role descriptions', () => { const [heading, unknown] = flattenIosTree([ { type: 'Heading', traits: ['Scrollable', 'Header'], role_description: 'heading' }, diff --git a/packages/provider-limrun/src/snapshot.ts b/packages/provider-limrun/src/snapshot.ts index d2f68845bc..b209372448 100644 --- a/packages/provider-limrun/src/snapshot.ts +++ b/packages/provider-limrun/src/snapshot.ts @@ -2,15 +2,23 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; type LimrunSelector = { key: 'id' | 'label' | 'text' | 'value'; value: string }; +/** + * One accessibility text attribute as the instance's JSON reports it. `AXValue` is a variant + * attribute — the picker's segmented tabs report the numbers `1`/`0` — and the other text + * attributes arrive over the same untyped channel, so every one of them is read through + * {@link scalarText} before it reaches a snapshot node. + */ +type IosTextAttribute = string | number | boolean | null; + export type IosTreeNode = { elementType?: string; type?: string; - label?: string; - AXLabel?: string | null; - identifier?: string; - AXUniqueId?: string | null; - value?: string; - AXValue?: string | null; + label?: IosTextAttribute; + AXLabel?: IosTextAttribute; + identifier?: IosTextAttribute; + AXUniqueId?: IosTextAttribute; + value?: IosTextAttribute; + AXValue?: IosTextAttribute; frame?: { x?: number; y?: number; width?: number; height?: number }; rect?: { x?: number; y?: number; width?: number; height?: number }; enabled?: boolean; @@ -53,7 +61,7 @@ function mapIosNode( identifier: readIosNodeIdentifier(node), rect: readIosNodeRect(node), enabled: node.enabled, - selected: node.selected, + selected: readIosNodeSelected(node), password: node.subrole === 'AXSecureTextField' ? true : undefined, heading: node.traits?.includes('Header') === true ? true : undefined, roleDescription: node.role_description, @@ -67,20 +75,43 @@ function readIosNodeType(node: IosTreeNode): string | undefined { return node.elementType ?? node.type; } +/** + * Selection from an explicit field or the `Selected` trait the picker reports on a tapped + * photo. Published as `true` only when selected and omitted otherwise, the shape the Apple + * tree publishes, so a `selected:` selector cannot tell the producers apart. + */ +function readIosNodeSelected(node: IosTreeNode): true | undefined { + if (node.selected === true || node.traits?.includes('Selected') === true) return true; + return undefined; +} + function readIosNodeRole(node: IosTreeNode): string | undefined { return node.role ?? readIosNodeType(node); } function readIosNodeLabel(node: IosTreeNode): string | undefined { - return node.label ?? node.AXLabel ?? undefined; + return scalarText(node.label) ?? scalarText(node.AXLabel); } function readIosNodeValue(node: IosTreeNode): string | undefined { - return node.value ?? node.AXValue ?? undefined; + return scalarText(node.value) ?? scalarText(node.AXValue); +} + +/** + * One scalar text attribute of the snapshot-node contract. iOS `AXValue` is a variant + * attribute — the photo picker's segmented tabs report the numbers `1`/`0` — and every + * attribute arrives over the same untyped JSON channel, so each is read through here + * before it reaches a node. This mirrors what the Apple tree's scalar reader publishes; + * unlike that trusted-guest reader, an unexpected type here stringifies rather than + * throwing, because failing one picker screen is not worth failing the whole snapshot. + */ +function scalarText(value: string | number | boolean | null | undefined): string | undefined { + if (value === undefined || value === null || value === '') return undefined; + return typeof value === 'string' ? value : String(value); } function readIosNodeIdentifier(node: IosTreeNode): string | undefined { - return node.identifier ?? node.AXUniqueId ?? undefined; + return scalarText(node.identifier) ?? scalarText(node.AXUniqueId); } /** The node's screen-space rect, or undefined when the provider omitted a complete one. */