diff --git a/pkg/driver/devicelab/keyboard.go b/pkg/driver/devicelab/keyboard.go index cf7d07f..152babc 100644 --- a/pkg/driver/devicelab/keyboard.go +++ b/pkg/driver/devicelab/keyboard.go @@ -5,6 +5,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/devicelab-dev/maestro-runner/pkg/core" "github.com/devicelab-dev/maestro-runner/pkg/flow" @@ -150,35 +151,81 @@ func (d *Driver) consumeInputFlag() bool { var errKeyboardOpen = fmt.Errorf("keyboard is open — add a `- hideKeyboard` step before this step") +// keyboardSettleWindow bounds how long checkKeyboardBlocking re-samples geometry before +// declaring the element covered. Windows with SOFT_INPUT_ADJUST_RESIZE (e.g. a plain +// AlertDialog whose body is a ScrollView) relayout a few frames after the IME appears or +// after typing: on the first frame the target still reports covered bounds, then the +// window shrinks and the target rises above the keyboard. A single-shot check reads that +// stale first frame and rejects a perfectly tappable element. Var (not const) so tests +// can shrink it. +var keyboardSettleWindow = 2 * time.Second + +// keyboardSettlePoll is the re-sample cadence while waiting for the geometry to settle. +const keyboardSettlePoll = 50 * time.Millisecond + +// keyboardStillCovering is the per-sample verdict: true only when the keyboard is +// visible AND a tap on the element's center would land on it. +func keyboardStillCovering(element core.Bounds, keyboard *core.Bounds) bool { + return keyboard != nil && tapWouldHitKeyboard(element, *keyboard) +} + // checkKeyboardBlocking checks if the keyboard overlaps the target element after an input step. // UIA2 finds elements via the accessibility tree even when the keyboard covers them, // but coordinate taps land on the keyboard overlay instead. This detects that case and -// fails fast with a helpful hint instead of silently tapping the keyboard. +// fails with a helpful hint instead of silently tapping the keyboard. // Returns nil if this check doesn't apply or element is not blocked — caller should proceed normally. func (d *Driver) checkKeyboardBlocking(wasInput bool, sel flow.Selector) *core.CommandResult { if !wasInput { return nil } - // Find element (UIA2 will find it even behind keyboard) - _, info, err := d.findElementOnce(sel) - if err != nil || info == nil { - // Element genuinely not found — let caller do the full-timeout find - return nil - } + return settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { + // Find element (UIA2 will find it even behind keyboard) + _, info, err := d.findElementOnce(sel) + if err != nil || info == nil { + // Element genuinely not found — let caller do the full-timeout find + return nil, false + } + return info, true + }, + d.getKeyboardBounds, + ) +} - // Element found — check if keyboard overlaps its bounds - kbBounds := d.getKeyboardBounds() - if kbBounds == nil { - return nil - } +// settleKeyboardBlocking re-samples element vs keyboard geometry until keyboardSettleWindow +// elapses: IME-aware windows (SOFT_INPUT_ADJUST_RESIZE) lift the target above the keyboard +// shortly after an input step, and rejecting on first-frame geometry would fail elements +// that are tappable an instant later. Only a PERSISTENT overlap — the true positive this +// guard exists for — returns the error. The samplers are injected so the settle behavior +// is testable without a live driver. +func settleKeyboardBlocking(findElement func() (*core.ElementInfo, bool), + keyboardBounds func() *core.Bounds) *core.CommandResult { + deadline := time.Now().Add(keyboardSettleWindow) + lastKbTop, lastCenterY := -1, -1 + for { + info, ok := findElement() + if !ok { + return nil + } + + kbBounds := keyboardBounds() + if !keyboardStillCovering(info.Bounds, kbBounds) { + // Keyboard dismissed, or the window resized/panned and the element + // now sits above it — nothing blocks the tap. + return nil + } - if tapWouldHitKeyboard(info.Bounds, *kbBounds) { _, cy := info.Bounds.Center() - return errorResult(errKeyboardOpen, - fmt.Sprintf("Element found but keyboard is covering it (keyboard top: %d, element center Y: %d) — add a `- hideKeyboard` step before this step", - kbBounds.Y, cy)) + lastKbTop, lastCenterY = kbBounds.Y, cy + + if !time.Now().Before(deadline) { + break + } + time.Sleep(keyboardSettlePoll) } - return nil + return errorResult(errKeyboardOpen, + fmt.Sprintf("Element found but keyboard is covering it (keyboard top: %d, element center Y: %d) — add a `- hideKeyboard` step before this step", + lastKbTop, lastCenterY)) } diff --git a/pkg/driver/devicelab/keyboard_test.go b/pkg/driver/devicelab/keyboard_test.go new file mode 100644 index 0000000..035fb9f --- /dev/null +++ b/pkg/driver/devicelab/keyboard_test.go @@ -0,0 +1,180 @@ +package devicelab + +import ( + "strings" + "testing" + "time" + + "github.com/devicelab-dev/maestro-runner/pkg/core" + "github.com/devicelab-dev/maestro-runner/pkg/flow" +) + +// Per-sample verdict for the keyboard-blocking settle loop. Pinned with the real-world +// geometry that motivated it: an AlertDialog positive button (android:id/button1) +// reported at centerY=1627 on the first frame after typing (keyboard top 1560), then +// lifted to ~1400 once the window's SOFT_INPUT_ADJUST_RESIZE relayout settled — the +// guard must treat the first sample as "still covering" and the second as clear, and a +// dismissed keyboard (nil bounds) as never covering. + +func elementCenteredAtY(cy int) core.Bounds { + // Height 100 → center = Y + 50. + return core.Bounds{X: 400, Y: cy - 50, Width: 200, Height: 100} +} + +func TestKeyboardStillCovering(t *testing.T) { + keyboard := &core.Bounds{X: 0, Y: 1560, Width: 1080, Height: 840} + + cases := []struct { + name string + element core.Bounds + keyboard *core.Bounds + want bool + }{ + {"covered on first frame after typing", elementCenteredAtY(1627), keyboard, true}, + {"clear after ADJUST_RESIZE relayout", elementCenteredAtY(1400), keyboard, false}, + {"keyboard dismissed mid-settle", elementCenteredAtY(1627), nil, false}, + // The 50px suggestion-strip margin: at keyboard.Y+margin is covered… + {"at margin boundary", elementCenteredAtY(1610), keyboard, true}, + // …one pixel above the margin is still tappable. + {"just above margin", elementCenteredAtY(1609), keyboard, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := keyboardStillCovering(tc.element, tc.keyboard) + if got != tc.want { + t.Errorf("keyboardStillCovering(%+v, %+v) = %v, want %v", + tc.element, tc.keyboard, got, tc.want) + } + }) + } +} + +// --- Settle loop ----------------------------------------------------------------------- + +// shrinkSettleWindow shortens the settle window so loop tests run in milliseconds, +// restoring the real value afterwards. +func shrinkSettleWindow(t *testing.T, d time.Duration) { + t.Helper() + orig := keyboardSettleWindow + keyboardSettleWindow = d + t.Cleanup(func() { keyboardSettleWindow = orig }) +} + +func TestSettleKeyboardBlocking_ElementNotFound(t *testing.T) { + shrinkSettleWindow(t, 100*time.Millisecond) + res := settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { return nil, false }, + func() *core.Bounds { + t.Fatal("keyboardBounds must not be sampled when the element is missing") + return nil + }, + ) + if res != nil { + t.Errorf("expected nil (caller does the full-timeout find), got %+v", res) + } +} + +func TestSettleKeyboardBlocking_ClearOnFirstSample(t *testing.T) { + shrinkSettleWindow(t, 100*time.Millisecond) + samples := 0 + res := settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { + samples++ + return &core.ElementInfo{Bounds: elementCenteredAtY(1400)}, true + }, + func() *core.Bounds { return &core.Bounds{X: 0, Y: 1560, Width: 1080, Height: 840} }, + ) + if res != nil { + t.Errorf("element above the keyboard must not block, got %+v", res) + } + if samples != 1 { + t.Errorf("clear-on-first-frame must return without re-sampling, sampled %d times", samples) + } +} + +func TestSettleKeyboardBlocking_RelayoutLiftsElementMidSettle(t *testing.T) { + // The real-world repro: first frame still covered (centerY 1627 vs keyboard top + // 1560), then the ADJUST_RESIZE relayout lifts the element clear (~1400). + shrinkSettleWindow(t, 2*time.Second) // generous — the early exit must beat it + samples := 0 + start := time.Now() + res := settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { + samples++ + if samples == 1 { + return &core.ElementInfo{Bounds: elementCenteredAtY(1627)}, true + } + return &core.ElementInfo{Bounds: elementCenteredAtY(1400)}, true + }, + func() *core.Bounds { return &core.Bounds{X: 0, Y: 1560, Width: 1080, Height: 840} }, + ) + if res != nil { + t.Errorf("element lifted mid-settle must not block, got %+v", res) + } + if samples != 2 { + t.Errorf("expected exactly 2 samples (covered, then clear), got %d", samples) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("early exit expected well before the settle window, took %v", elapsed) + } +} + +func TestSettleKeyboardBlocking_KeyboardDismissedMidSettle(t *testing.T) { + shrinkSettleWindow(t, 2*time.Second) + samples := 0 + res := settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { + samples++ + return &core.ElementInfo{Bounds: elementCenteredAtY(1627)}, true + }, + func() *core.Bounds { + if samples == 1 { + return &core.Bounds{X: 0, Y: 1560, Width: 1080, Height: 840} + } + return nil // keyboard gone + }, + ) + if res != nil { + t.Errorf("dismissed keyboard must not block, got %+v", res) + } +} + +func TestSettleKeyboardBlocking_PersistentOverlapFails(t *testing.T) { + shrinkSettleWindow(t, 120*time.Millisecond) // ~3 samples at the 50ms cadence + samples := 0 + res := settleKeyboardBlocking( + func() (*core.ElementInfo, bool) { + samples++ + return &core.ElementInfo{Bounds: elementCenteredAtY(1627)}, true + }, + func() *core.Bounds { return &core.Bounds{X: 0, Y: 1560, Width: 1080, Height: 840} }, + ) + if res == nil { + t.Fatal("persistent overlap through the whole settle window must fail") + } + if samples < 2 { + t.Errorf("expected re-sampling before failing, sampled %d times", samples) + } + if !strings.Contains(res.Message, "keyboard top: 1560") || + !strings.Contains(res.Message, "element center Y: 1627") { + t.Errorf("error must keep the actionable geometry hint, got %q", res.Message) + } +} + +func TestCheckKeyboardBlocking_SkipsWhenPreviousStepWasNotInput(t *testing.T) { + d := &Driver{client: &mockDeviceLabClient{}} + if res := d.checkKeyboardBlocking(false, flow.Selector{ID: "android:id/button1"}); res != nil { + t.Errorf("check must not apply after a non-input step, got %+v", res) + } +} + +func TestCheckKeyboardBlocking_ElementNotFoundProceeds(t *testing.T) { + // End-to-end through the driver: the mock client fails FindElement, so the + // settle loop must bail out and let the caller do the full-timeout find. + shrinkSettleWindow(t, 100*time.Millisecond) + d := New(&mockDeviceLabClient{}, &core.PlatformInfo{}, &mockShell{}) + if res := d.checkKeyboardBlocking(true, flow.Selector{ID: "android:id/button1"}); res != nil { + t.Errorf("element not found must proceed to the normal find, got %+v", res) + } +}