Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@

## Unreleased

- Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one.
- Carry provider keyboard facts through snapshots. Appium captures, including wait polls and fill pre-captures, add one sequential keyboard visibility probe with a 500 ms timeout. Probe results preserve tap-time docked-tree checks; unavailable probes leave visibility unknown.
24 changes: 24 additions & 0 deletions packages/provider-limrun/src/ios-snapshot-adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,30 @@ test('Limrun carries docked tree evidence and never infers absence from missing
).toEqual({ kind: 'tree-rejected', derivation: 'docked-tree' });
});

test('Limrun acquires a system photo picker whose tabs report numeric values', async () => {
// PHPickerViewController reports AXValue 1/0 as numbers, and marks a tapped photo with the
// Selected trait. Both are variant JSON the raw tree hands over untyped.
const result = await captureLimrunIosSnapshot(
createLimrunSnapshotSession({
elementType: 'Other',
children: [
{ elementType: 'RadioButton', AXLabel: 'Photos', AXValue: 1 },
Comment thread
thymikee marked this conversation as resolved.
{ elementType: 'RadioButton', AXLabel: 'Collections', AXValue: 0 },
{
elementType: 'Image',
AXLabel: 'Photo',
AXUniqueId: 12,
traits: ['Scrollable', 'Selected', 'Image'],
},
],
}),
);
const [photos, collections, photo] = result.acquisition.nodes.slice(1);
expect(photos).toMatchObject({ value: '1', checked: true });
expect(collections).toMatchObject({ value: '0', checked: false });
expect(photo).toMatchObject({ selected: true, label: 'Photo', identifier: '12' });
});

test('limrun acquisition shares RN interpretation and preserves native values', async () => {
const result = await captureLimrunIosSnapshot(
createLimrunSnapshotSession({
Expand Down
33 changes: 33 additions & 0 deletions packages/provider-limrun/src/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,39 @@ test('visible text targets the provider label field, which is where the tree exp
expect(toIosSelector({ key: 'text', value: 'Sign In' })).toEqual({ AXLabel: 'Sign In' });
});

test('variant scalar attributes become the text every snapshot producer publishes', () => {
// The system photo picker's segmented tabs report the numbers 1/0 as AXValue, and the
// provider's JSON can carry a variant scalar in any text attribute.
const [selected, unselected, boolean, text, empty, missing, fallback, label, identifier] =
flattenIosTree([
{ type: 'RadioButton', AXLabel: 'Photos', AXValue: 1 },
{ type: 'RadioButton', AXLabel: 'Collections', AXValue: 0 },
{ type: 'Switch', value: true },
{ type: 'TextField', AXValue: 'qa@example.com' },
{ type: 'TextField', value: '' },
{ type: 'TextField', AXValue: null },
{ type: 'TextField', value: '', AXValue: 1 },
{ type: 'Other', label: 42, AXLabel: 'fallback' },
{ type: 'Other', identifier: true, AXUniqueId: 'fallback' },
]);
expect([selected?.value, unselected?.value, boolean?.value]).toEqual(['1', '0', 'true']);
expect(text?.value).toBe('qa@example.com');
expect([empty?.value, missing?.value]).toEqual([undefined, undefined]);
expect(fallback?.value).toBe('1');
expect([label?.label, identifier?.identifier]).toEqual(['42', 'true']);
});

test('the Selected trait publishes selection in the shape the Apple tree uses', () => {
const [selected, unselected, explicit] = flattenIosTree([
{ type: 'Image', traits: ['Scrollable', 'Selected', 'Image'] },
{ type: 'Image', traits: ['Image'] },
{ type: 'Image', selected: true },
]);
expect(selected).toHaveProperty('selected', true);
expect(unselected?.selected).toBeUndefined();
expect(explicit).toHaveProperty('selected', true);
});

test('Limrun raw mapping forwards observed Header traits and SDK role descriptions', () => {
const [heading, unknown] = flattenIosTree([
{ type: 'Heading', traits: ['Scrollable', 'Header'], role_description: 'heading' },
Expand Down
51 changes: 41 additions & 10 deletions packages/provider-limrun/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,23 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';

type LimrunSelector = { key: 'id' | 'label' | 'text' | 'value'; value: string };

/**
* One accessibility text attribute as the instance's JSON reports it. `AXValue` is a variant
* attribute — the picker's segmented tabs report the numbers `1`/`0` — and the other text
* attributes arrive over the same untyped channel, so every one of them is read through
* {@link scalarText} before it reaches a snapshot node.
*/
type IosTextAttribute = string | number | boolean | null;

export type IosTreeNode = {
elementType?: string;
type?: string;
label?: string;
AXLabel?: string | null;
identifier?: string;
AXUniqueId?: string | null;
value?: string;
AXValue?: string | null;
label?: IosTextAttribute;
AXLabel?: IosTextAttribute;
identifier?: IosTextAttribute;
AXUniqueId?: IosTextAttribute;
value?: IosTextAttribute;
AXValue?: IosTextAttribute;
frame?: { x?: number; y?: number; width?: number; height?: number };
rect?: { x?: number; y?: number; width?: number; height?: number };
enabled?: boolean;
Expand Down Expand Up @@ -53,7 +61,7 @@ function mapIosNode(
identifier: readIosNodeIdentifier(node),
rect: readIosNodeRect(node),
enabled: node.enabled,
selected: node.selected,
selected: readIosNodeSelected(node),
password: node.subrole === 'AXSecureTextField' ? true : undefined,
heading: node.traits?.includes('Header') === true ? true : undefined,
roleDescription: node.role_description,
Expand All @@ -67,20 +75,43 @@ function readIosNodeType(node: IosTreeNode): string | undefined {
return node.elementType ?? node.type;
}

/**
* Selection from an explicit field or the `Selected` trait the picker reports on a tapped
* photo. Published as `true` only when selected and omitted otherwise, the shape the Apple
* tree publishes, so a `selected:` selector cannot tell the producers apart.
*/
function readIosNodeSelected(node: IosTreeNode): true | undefined {
if (node.selected === true || node.traits?.includes('Selected') === true) return true;
return undefined;
}

function readIosNodeRole(node: IosTreeNode): string | undefined {
return node.role ?? readIosNodeType(node);
}

function readIosNodeLabel(node: IosTreeNode): string | undefined {
return node.label ?? node.AXLabel ?? undefined;
return scalarText(node.label) ?? scalarText(node.AXLabel);
}

function readIosNodeValue(node: IosTreeNode): string | undefined {
return node.value ?? node.AXValue ?? undefined;
return scalarText(node.value) ?? scalarText(node.AXValue);
}

/**
* One scalar text attribute of the snapshot-node contract. iOS `AXValue` is a variant
* attribute — the photo picker's segmented tabs report the numbers `1`/`0` — and every
* attribute arrives over the same untyped JSON channel, so each is read through here
* before it reaches a node. This mirrors what the Apple tree's scalar reader publishes;
* unlike that trusted-guest reader, an unexpected type here stringifies rather than
* throwing, because failing one picker screen is not worth failing the whole snapshot.
*/
function scalarText(value: string | number | boolean | null | undefined): string | undefined {
if (value === undefined || value === null || value === '') return undefined;
return typeof value === 'string' ? value : String(value);
}

function readIosNodeIdentifier(node: IosTreeNode): string | undefined {
return node.identifier ?? node.AXUniqueId ?? undefined;
return scalarText(node.identifier) ?? scalarText(node.AXUniqueId);
}

/** The node's screen-space rect, or undefined when the provider omitted a complete one. */
Expand Down
Loading