From 572f2ff5fb74f5a108ba2effbc2d860f1a996ac4 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:28:45 -0700 Subject: [PATCH] fix(provider-limrun): preserve Unicode in Limrun iOS fill After text-entry focus arrives, fill writes the whole string with one setElementValue call, including an empty string and characters that have no hardware key. A supplied delay remains on the signature and the replacement stays a single write. type still sends key events, with per-character timing when a delay is set. The Limrun iOS docs and the text-entry tests follow that path, including no write when focus never arrives and failure when the value write is rejected. Fixes #3819 --- .../src/ios-interactor-text-entry.test.ts | 107 +++++++++++++----- packages/provider-limrun/src/ios.ts | 15 +-- website/docs/docs/limrun.md | 2 +- 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/packages/provider-limrun/src/ios-interactor-text-entry.test.ts b/packages/provider-limrun/src/ios-interactor-text-entry.test.ts index 9986ffe273..233138c3f2 100644 --- a/packages/provider-limrun/src/ios-interactor-text-entry.test.ts +++ b/packages/provider-limrun/src/ios-interactor-text-entry.test.ts @@ -1,4 +1,4 @@ -import { expect, test, vi } from 'vitest'; +import { afterEach, expect, test, vi } from 'vitest'; import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts'; import type { IosTreeNode } from './snapshot.ts'; @@ -16,6 +16,10 @@ function emailField(editing: boolean): IosTreeNode { }; } +afterEach(() => { + vi.restoreAllMocks(); +}); + function sessionWithTrees(trees: IosTreeNode[]) { const calls: string[] = []; let reads = 0; @@ -35,6 +39,9 @@ function sessionWithTrees(trees: IosTreeNode[]) { typeText: vi.fn(async () => { calls.push('typeText'); }), + setElementValue: vi.fn(async () => { + calls.push('setElementValue'); + }), }; const session = { platform: 'ios', @@ -64,7 +71,7 @@ test('a delayed type sends one focused-blind key press per character', async () ]); }); -test('fill waits for text-entry focus to arrive and only then sends keys', async () => { +test('fill waits for text-entry focus to arrive and only then sets the value', async () => { const { interactor, client, calls } = sessionWithTrees([ emailField(false), emailField(false), @@ -73,40 +80,52 @@ test('fill waits for text-entry focus to arrive and only then sends keys', async const result = await interactor.fill(200, 170, 'qa@example.com'); - expect(calls).toEqual([ - 'elementTree', - 'tap', - 'elementTree', - 'elementTree', - 'pressKey:a+command', - 'typeText', - ]); + expect(calls).toEqual(['elementTree', 'tap', 'elementTree', 'elementTree', 'setElementValue']); expect(result).toEqual({ textEntryReadiness: 'focused-element' }); - expect(client.typeText).toHaveBeenCalledWith('qa@example.com', false, { requireFocus: false }); + expect(client.setElementValue.mock.calls).toEqual([['qa@example.com']]); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); }); -test('fill selects the field value first, so entry replaces it', async () => { - const { interactor, calls } = sessionWithTrees([emailField(false), emailField(true)]); +test('fill sets Chinese, mixed, and emoji text in one write after focus', async () => { + for (const text of ['你好', 'qa@例子.com', '👋']) { + const { interactor, client, calls } = sessionWithTrees([emailField(false), emailField(true)]); - await interactor.fill(200, 170, 'replacement'); + const result = await interactor.fill(200, 170, text); - expect(calls.indexOf('pressKey:a+command')).toBeLessThan(calls.indexOf('typeText')); + expect(result).toEqual({ textEntryReadiness: 'focused-element' }); + expect(calls.indexOf('tap')).toBeLessThan(calls.lastIndexOf('elementTree')); + expect(calls.lastIndexOf('elementTree')).toBeLessThan(calls.indexOf('setElementValue')); + expect(client.setElementValue.mock.calls).toEqual([[text]]); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); + } }); -test('the clear request empties the focused field instead of typing nothing', async () => { - const { interactor, client, calls } = sessionWithTrees([emailField(false), emailField(true)]); +test('fill replaces an ASCII value and clears with an empty string on the same path', async () => { + for (const text of ['replacement', '']) { + const { interactor, client } = sessionWithTrees([emailField(false), emailField(true)]); - const result = await interactor.fill(200, 170, ''); + const result = await interactor.fill(200, 170, text); - expect(calls).toEqual([ - 'elementTree', - 'tap', - 'elementTree', - 'pressKey:a+command', - 'pressKey:delete', - ]); - expect(client.typeText).not.toHaveBeenCalled(); - expect(result).toEqual({ textEntryReadiness: 'focused-element' }); + expect(result).toEqual({ textEntryReadiness: 'focused-element' }); + expect(client.setElementValue.mock.calls).toEqual([[text]]); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); + } +}); + +test('a delayed fill still sets the complete value once', async () => { + for (const text of ['你好a', '']) { + const { interactor, client } = sessionWithTrees([emailField(false), emailField(true)]); + + const result = await interactor.fill(200, 170, text, 1); + + expect(result).toEqual({ textEntryReadiness: 'focused-element' }); + expect(client.setElementValue.mock.calls).toEqual([[text]]); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); + } }); test('a session that cannot answer the focus read fails before the device is touched', async () => { @@ -121,5 +140,39 @@ test('a session that cannot answer the focus read fails before the device is tou ); expect(calls).toEqual(['elementTree']); expect(client.tap).not.toHaveBeenCalled(); + expect(client.setElementValue).not.toHaveBeenCalled(); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); +}); + +test('fill writes no value when text-entry focus never arrives', async () => { + const { interactor, client } = sessionWithTrees([emailField(false)]); + // The focus budget is wall-clock time. Step the clock by that budget on every + // read so the first miss expires it, and the refusal is not a three-second wait. + let now = 0; + vi.spyOn(Date, 'now').mockImplementation(() => { + const current = now; + now += 3_000; + return current; + }); + + await expect(interactor.fill(200, 170, '你好')).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'text_entry_focus_not_observed' }, + }); + expect(client.tap).toHaveBeenCalledTimes(1); + expect(client.setElementValue).not.toHaveBeenCalled(); + expect(client.pressKey).not.toHaveBeenCalled(); + expect(client.typeText).not.toHaveBeenCalled(); +}); + +test('a rejected value write fails the fill with that failure', async () => { + const { interactor, client } = sessionWithTrees([emailField(false), emailField(true)]); + const failure = new Error('element value refused'); + client.setElementValue.mockRejectedValueOnce(failure); + + await expect(interactor.fill(200, 170, '你好')).rejects.toBe(failure); + expect(client.setElementValue.mock.calls).toEqual([['你好']]); + expect(client.pressKey).not.toHaveBeenCalled(); expect(client.typeText).not.toHaveBeenCalled(); }); diff --git a/packages/provider-limrun/src/ios.ts b/packages/provider-limrun/src/ios.ts index 9c38d6ca4d..8040034c62 100644 --- a/packages/provider-limrun/src/ios.ts +++ b/packages/provider-limrun/src/ios.ts @@ -277,7 +277,7 @@ class LimrunIosInteractor implements Interactor { await this.enterText(text, delayMs); } - async fill(x: number, y: number, text: string, delayMs?: number): Promise { + async fill(x: number, y: number, text: string, _delayMs?: number): Promise { // Loaded on the fill path to keep this provider's declared import-time closure budget. const { awaitLimrunTextEntryFocus, readLimrunUnambiguousTapTargets, readLimrunTextEntryFocus } = await import('./text-entry-focus.ts'); @@ -295,14 +295,11 @@ class LimrunIosInteractor implements Interactor { x, y, }); - // Select first: iOS replaces a selection on the next key, so `fill` replaces and - // an empty fill clears. - await this.session.client.pressKey('a', ['command']); - if (text.length === 0) { - await this.session.client.pressKey('delete'); - return { textEntryReadiness }; - } - await this.enterText(text, delayMs); + // One value write on the focused element. The SDK omits the selector so the + // server resolves focus, and the whole string is the value, including an empty + // clear and characters that have no key code. A supplied delay stays on this + // signature for callers; per-character timing stays on `type`. + await this.session.client.setElementValue(text); return { textEntryReadiness }; } diff --git a/website/docs/docs/limrun.md b/website/docs/docs/limrun.md index 7691d738d4..ee5fecb28a 100644 --- a/website/docs/docs/limrun.md +++ b/website/docs/docs/limrun.md @@ -75,7 +75,7 @@ On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, ta On iOS, `settings` supports `appearance`, `permission`, `location`, and `clear-app-state`, with the same arguments as a local simulator. `clear-app-state` resets the app's data container through Limrun's soft reset. That reset relaunches the app once before agent-device stops it, so the app may run once after its data is cleared. `reset-keychain`, `wifi`, `airplane`, `mobile-data`, `faceid`, `touchid`, and `text-size` are unsupported, and you cannot read back any iOS setting. On Android, `settings mobile-data on|off` runs over the ADB tunnel like on a local device. -On iOS, `fill` taps the target, waits for that field to take text-entry focus, and only then types. Apps that never report a globally focused element, such as Flutter forms, still fill and report `textEntryReadiness: "focused-element"`. If the tap does not focus a field, `fill` fails with `text_entry_focus_not_observed` instead of typing into an unknown field. Use `type` to send text to whichever field already has focus. +On iOS, `fill` taps the target, waits for that field to take text-entry focus, and replaces the focused field's value in one write. The write is the complete text, including an empty string that clears the field and characters that have no hardware key. Apps that never report a globally focused element, such as Flutter forms, still fill and report `textEntryReadiness: "focused-element"`. If the tap does not focus a field, `fill` fails with `text_entry_focus_not_observed` and writes nothing. `type` sends key events to whichever field already has focus. `--delay-ms` on `type` enters one character at a time; `--delay-ms` on `fill` keeps that replacement as one write of the whole value. To drive Limrun only through MCP, see [Use a provider only through MCP](/docs/device-clouds#use-a-provider-only-through-mcp). Limrun returns no provider artifacts, so `artifacts` has nothing to fetch.