diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index ccc1318840..93f622f9f0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -504,6 +504,18 @@ - (void)agentDeviceTextEntryBecomesUnqueryable:(UITextField *)textField { // avoidance or a bottom sheet extending above the keyboard does, so the neighbouring field slides // into the point the focus tap hit. - (void)agentDeviceTextEntryDidBeginEditing:(UITextField *)textField { + [self moveTextEntryFieldUp]; +} + +// Moves the field once text arrives, the way an iOS 26 bottom search bar rises above the keyboard +// on the first character: the focus tap and the pre-entry read both find it where it started. +- (void)agentDeviceTextEntryMovesOnInput:(UITextField *)textField { + if (textField.text.length > 0) { + [self moveTextEntryFieldUp]; + } +} + +- (void)moveTextEntryFieldUp { self.textEntryFieldTop.constant = AgentDeviceTextEntryFieldTopInset - AgentDeviceTextEntryFieldHeight - AgentDeviceTextEntryNeighbourGap; [self.view layoutIfNeeded]; @@ -643,6 +655,11 @@ - (void)viewDidLoad { action:@selector(agentDeviceTextEntryBecomesUnqueryable:) forControlEvents:UIControlEventEditingDidBegin]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-moves-on-input"]) { + [textField addTarget:self + action:@selector(agentDeviceTextEntryMovesOnInput:) + forControlEvents:UIControlEventEditingChanged]; + } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-moves-on-focus"]) { textField.text = @"stale"; [textField addTarget:self diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 94b4d73b0c..58330f6813 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -441,7 +441,7 @@ extension RunnerTests { return node } - private func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool { + func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool { return focusBool(snapshot as! NSObject) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift index f54b22ef6e..b86d46f752 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift @@ -24,11 +24,21 @@ extension RunnerTests { struct TextEntryObservation: Equatable { let value: String let identity: TextEntryElementIdentity + let isFocused: Bool /// Same value on the same element; a frame change alone on an identified field is not a change. func isSettled(with other: TextEntryObservation) -> Bool { value == other.value && identity.isSameElement(as: other.identity) } + + /// An element without an identifier that kept focus but moved: a bottom search bar rises above + /// the keyboard and widens once the first characters arrive, so its frame before typing never + /// matches the read-back. + func isMovedWhileFocused(since baseline: TextEntryObservation) -> Bool { + identity.identifier == nil && baseline.identity.identifier == nil + && identity.elementType == baseline.identity.elementType + && isFocused && baseline.isFocused + } } /// Target-bound evidence that the entry changed the field while its value cannot confirm the text. @@ -86,12 +96,15 @@ extension RunnerTests { TextEntryResult(verified: verified, repaired: false, expectedText: requested, observedText: observed?.value) } guard let observed else { return result(nil) } - if let baseline, !baseline.identity.isSameElement(as: observed.identity) { - return result(false) - } let request = observed.identity.elementType == elementTypeNamesByRawValue[XCUIElement.ElementType.textView.rawValue] ? requested : textEntryRequestWithoutSubmitKeys(requested) - if observed.value == requested || observed.value == request { return result(true) } + let exact = observed.value == requested || observed.value == request + if let baseline, !baseline.identity.isSameElement(as: observed.identity) { + // Focus does not tell a moved field from an unidentified field that replaced it, so only an + // exact read-back passes. + return result(exact && observed.isMovedWhileFocused(since: baseline)) + } + if exact { return result(true) } // Any non-matching single-line submit result remains unverified, even a partial value. if request != requested { return result(nil) } guard !request.isEmpty, let baseline else { return result(false) } @@ -125,7 +138,8 @@ extension RunnerTests { identifier: snapshot.identifier.isEmpty ? nil : snapshot.identifier, elementType: elementTypeName(snapshot.elementType), frame: snapshot.frame - ) + ), + isFocused: snapshotHasFocus(snapshot) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift index 68a2646c34..57dbbac5e2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift @@ -13,7 +13,7 @@ extension RunnerTests { _ value: String, identity: TextEntryElementIdentity = otpFieldIdentity ) -> TextEntryObservation { - TextEntryObservation(value: value, identity: identity) + TextEntryObservation(value: value, identity: identity, isFocused: false) } func testReplacementExactAndLiteralLossReadBack() { @@ -177,6 +177,51 @@ extension RunnerTests { XCTAssertFalse(field.isSameElement(as: TextEntryElementIdentity(identifier: "otp-input", elementType: "TextField", frame: frame))) } + func testReplacementIntoAnUnidentifiedFieldThatMovedVerifiesOnlyAnExactReadBackWhileFocused() { + // A bottom search bar measured on an iPhone Air simulator, iOS 26.5: focused and docked before + // typing, then raised above the keyboard and widened once the text arrived. + let docked = TextEntryElementIdentity( + identifier: nil, elementType: "SearchField", frame: CGRect(x: 88, y: 836, width: 244, height: 48) + ) + let raised = TextEntryElementIdentity( + identifier: nil, elementType: "SearchField", frame: CGRect(x: 8, y: 548, width: 348, height: 48) + ) + let focusedBaseline = TextEntryObservation(value: "", identity: docked, isFocused: true) + let unfocusedBaseline = TextEntryObservation(value: "", identity: docked, isFocused: false) + let focusedExact = TextEntryObservation(value: "qxzqxzqxz", identity: raised, isFocused: true) + let cases: [(String, TextEntryObservation, TextEntryObservation, Bool)] = [ + ("exact, focused", focusedBaseline, focusedExact, true), + ("partial, focused", focusedBaseline, TextEntryObservation(value: "qxzqxz", identity: raised, isFocused: true), false), + ("exact, focus lost", focusedBaseline, TextEntryObservation(value: "qxzqxzqxz", identity: raised, isFocused: false), false), + ("exact, unfocused before typing", unfocusedBaseline, focusedExact, false), + ( + "exact, another type", + focusedBaseline, + TextEntryObservation( + value: "qxzqxzqxz", + identity: TextEntryElementIdentity(identifier: nil, elementType: "TextField", frame: raised.frame), + isFocused: true + ), + false + ), + ( + "exact, identified", + focusedBaseline, + TextEntryObservation( + value: "qxzqxzqxz", + identity: TextEntryElementIdentity(identifier: "search", elementType: "SearchField", frame: raised.frame), + isFocused: true + ), + false + ), + ] + for (name, baseline, observed, expected) in cases { + let result = Self.replacementTextEntryResult(requested: "qxzqxzqxz", baseline: baseline, observed: observed) + XCTAssertEqual(result.verified, expected, name) + XCTAssertNil(result.unconfirmed, name) + } + } + func testVerificationTargetEncodesMissingIdentityAsExplicitNulls() throws { let payload = TextEntryVerificationTargetPayload( resourceId: nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index e78280981f..22ffe9d0e0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -283,6 +283,30 @@ extension RunnerTests { XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "") } + // An unidentified field that moves once the first character arrives, the way an iOS 26 bottom + // search bar rises above the keyboard: the read-back finds it, still focused, at another frame + // than the pre-entry read, holding exactly the requested text. + @MainActor + func testFillVerifiesAnUnidentifiedFieldThatMovesWhileTyping() throws { + let textField = try launchRemovableInputFixture( + "--agent-device-text-entry-moves-on-input", + "--agent-device-text-entry-unnamed-input" + ) + let frameBefore = textField.frame + + let failureCountBefore = currentXCTestFailureCount() + let response = executeTypeCommand( + activeApp: app, + command: try fillCommandFixture(commandId: "fill-moves-on-input", text: "hello", at: textField) + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.message, "typed") + XCTAssertNotEqual(textField.frame, frameBefore) + XCTAssertEqual(editableTextValue(for: textField, treatingPlaceholderAsEmpty: true), "hello") + } + // Pins a known limit, not desired behavior: neither input has an identifier, and the successor // takes the code field's index in the query that bound it, so the remaining posts resolve to it // and land in the successor. Refusing point and focus re-resolution would not help, because the