diff --git a/README.md b/README.md index 3660116c..18b126a4 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,20 @@ waitForIdleTimeout: 3000 # Device idle wait (ms), 0 to disable - assertVisible: "Welcome" ``` +## Visual Regression (`assertScreenshot`) + +`assertScreenshot` compares the current screen (optionally cropped with `cropOn`) against a reference PNG. + +- **First run:** if the reference file is missing, maestro-runner writes the captured screenshot as the new baseline and passes. +- **Re-baseline:** overwrite existing baselines with `--update-screenshots` (or `MAESTRO_UPDATE_SCREENSHOTS=true`). +- On mismatch, a `{name}_diff.png` overlay is written next to the reference. +- Pixel comparison is device-, resolution-, and OS-specific — pin your device config and set `thresholdPercentage` deliberately (default `95`). + +```bash +maestro-runner test flows/visual.yaml # seeds missing baselines +maestro-runner test --update-screenshots flows/visual.yaml +``` + ## Requirements - **Android testing:** `adb` (Android SDK Platform-Tools) diff --git a/pkg/cli/test.go b/pkg/cli/test.go index 4debfe3f..78a96006 100644 --- a/pkg/cli/test.go +++ b/pkg/cli/test.go @@ -169,6 +169,11 @@ Examples: Usage: "When to capture screenshots/hierarchy: on-failure (default), always, never", Value: "on-failure", }, + &cli.BoolFlag{ + Name: "update-screenshots", + Usage: "Overwrite existing assertScreenshot baselines with the current screen (missing baselines are always seeded on first run)", + EnvVars: []string{"MAESTRO_UPDATE_SCREENSHOTS"}, + }, // Emulator management flags (start-emulator, auto-start-emulator, // shutdown-after, boot-timeout) are global flags defined in cli.go. @@ -532,6 +537,9 @@ type RunConfig struct { // Artifacts Artifacts executor.ArtifactMode // When to capture screenshots/hierarchy + // UpdateScreenshots overwrites existing assertScreenshot baselines. + UpdateScreenshots bool + // Cloud provider (detected from AppiumURL, nil if not a cloud provider) CloudProvider cloud.Provider CloudMeta map[string]string @@ -715,6 +723,7 @@ func runTest(c *cli.Context) error { NoFlutterFallback: getBool("no-flutter-fallback"), AndroidTCPForward: getBool("android-tcp-forward"), Artifacts: parseArtifactMode(getString("artifacts")), + UpdateScreenshots: getBool("update-screenshots"), } // Apply waitForIdleTimeout with priority: @@ -1362,6 +1371,7 @@ func executeSingleDevice(cfg *RunConfig, flows []flow.Flow) (*executor.RunResult OutputDir: cfg.OutputDir, Parallelism: 0, Artifacts: cfg.Artifacts, + UpdateScreenshots: cfg.UpdateScreenshots, Device: deviceInfo, App: buildAppReport(driver), RunnerVersion: Version, @@ -1404,6 +1414,7 @@ func ExecuteFlowWithDriver(driver core.Driver, cfg *RunConfig, f flow.Flow) (*ex OutputDir: cfg.OutputDir, Parallelism: 0, Artifacts: cfg.Artifacts, + UpdateScreenshots: cfg.UpdateScreenshots, Device: deviceInfo, App: buildAppReport(driver), RunnerVersion: Version, @@ -1729,6 +1740,7 @@ func executeAppiumSingleSession(cfg *RunConfig, flows []flow.Flow) (*executor.Ru OutputDir: cfg.OutputDir, Parallelism: 0, Artifacts: cfg.Artifacts, + UpdateScreenshots: cfg.UpdateScreenshots, Device: deviceInfo, App: buildAppReport(driver), RunnerVersion: Version, @@ -2597,6 +2609,7 @@ func createParallelRunner(cfg *RunConfig, workers []executor.DeviceWorker, platf OutputDir: cfg.OutputDir, Parallelism: 0, Artifacts: cfg.Artifacts, + UpdateScreenshots: cfg.UpdateScreenshots, Device: deviceInfo, App: buildAppReport(firstDriver), RunnerVersion: Version, diff --git a/pkg/core/imagediff.go b/pkg/core/imagediff.go index a55e81d2..81662c8d 100644 --- a/pkg/core/imagediff.go +++ b/pkg/core/imagediff.go @@ -4,10 +4,18 @@ import ( "bytes" "fmt" "image" + "image/color" + "image/draw" _ "image/jpeg" // register decoders - _ "image/png" + "image/png" + "math" + "os" + "path/filepath" + "strings" ) +const maestroPixelTolerance = 0.1 + // ImageDifference returns the fraction of pixels that differ between two // encoded images (PNG or JPEG). Returns 1.0 when sizes differ or decoding // fails — that's "maximally different", which lets callers stop comparing and @@ -89,3 +97,315 @@ func CheckImageDifference(a, b []byte) (float64, error) { } return float64(differing) / float64(total), nil } + +// CheckImageMatchPercentage returns the percentage of pixels whose RGB color +// distance is within Maestro's per-pixel tolerance. This mirrors Maestro's +// assertScreenshot comparison rather than requiring exact RGB equality. +func CheckImageMatchPercentage(expectedData, actualData []byte) (float64, error) { + expected, _, err := image.Decode(bytes.NewReader(expectedData)) + if err != nil { + return 0, fmt.Errorf("decode expected image: %w", err) + } + actual, _, err := image.Decode(bytes.NewReader(actualData)) + if err != nil { + return 0, fmt.Errorf("decode actual image: %w", err) + } + + expectedBounds := expected.Bounds() + actualBounds := actual.Bounds() + if expectedBounds.Dx() != actualBounds.Dx() || expectedBounds.Dy() != actualBounds.Dy() { + return 0, fmt.Errorf( + "screenshot size mismatch: expected %dx%d, actual %dx%d", + expectedBounds.Dx(), + expectedBounds.Dy(), + actualBounds.Dx(), + actualBounds.Dy(), + ) + } + + total := expectedBounds.Dx() * expectedBounds.Dy() + if total == 0 { + return 100, nil + } + + maxColorDistance := math.Sqrt(255.0 * 255.0 * 3) + differenceLimit := math.Pow(maestroPixelTolerance*maxColorDistance, 2) + differing := 0 + for y := 0; y < expectedBounds.Dy(); y++ { + for x := 0; x < expectedBounds.Dx(); x++ { + er, eg, eb, _ := expected.At(expectedBounds.Min.X+x, expectedBounds.Min.Y+y).RGBA() + ar, ag, ab, _ := actual.At(actualBounds.Min.X+x, actualBounds.Min.Y+y).RGBA() + dr := float64(int(ar>>8) - int(er>>8)) + dg := float64(int(ag>>8) - int(eg>>8)) + db := float64(int(ab>>8) - int(eb>>8)) + if dr*dr+dg*dg+db*db > differenceLimit { + differing++ + } + } + } + + return 100 - float64(differing)/float64(total)*100, nil +} + +// DiffScreenshotPath returns the Maestro-style sidecar path for a screenshot +// comparison diff: "screen.png" → "screen_diff.png". +func DiffScreenshotPath(referencePath string) string { + ext := filepath.Ext(referencePath) + base := strings.TrimSuffix(referencePath, ext) + if ext == "" { + ext = ".png" + } + return base + "_diff" + ext +} + +// Diff overlay knobs aligned with Maestro's ScreenshotMatch / ImageComparison defaults. +const ( + diffRectangleLineWidth = 4 + diffMinimalRectSize = 40 + diffMergePad = 16 +) + +type diffRect struct { + minX, minY, maxX, maxY int +} + +// WriteScreenshotDiff writes a PNG of the actual screenshot with transparent +// red rectangles around regions that differ from the expected image (beyond +// Maestro's per-pixel color tolerance). Mirrors Maestro's assertScreenshot +// _diff.png artifact style (rectangle overlay rather than per-pixel paint). +func WriteScreenshotDiff(expectedData, actualData []byte, diffPath string) error { + expected, _, err := image.Decode(bytes.NewReader(expectedData)) + if err != nil { + return fmt.Errorf("decode expected image: %w", err) + } + actual, _, err := image.Decode(bytes.NewReader(actualData)) + if err != nil { + return fmt.Errorf("decode actual image: %w", err) + } + + expectedBounds := expected.Bounds() + actualBounds := actual.Bounds() + width, height := expectedBounds.Dx(), expectedBounds.Dy() + if width != actualBounds.Dx() || height != actualBounds.Dy() { + return fmt.Errorf( + "screenshot size mismatch: expected %dx%d, actual %dx%d", + width, + height, + actualBounds.Dx(), + actualBounds.Dy(), + ) + } + + mask := differingPixelMask(expected, actual, expectedBounds, actualBounds, width, height) + rects := mergeDiffRects(diffBoundingRects(mask, width, height), diffMergePad) + for i := range rects { + rects[i] = expandDiffRect(rects[i], width, height, diffMinimalRectSize) + } + + diff := image.NewRGBA(image.Rect(0, 0, width, height)) + draw.Draw(diff, diff.Bounds(), actual, actualBounds.Min, draw.Src) + for _, r := range rects { + drawDiffRectangle(diff, r, diffRectangleLineWidth) + } + + if err := os.MkdirAll(filepath.Dir(diffPath), 0o755); err != nil { + return fmt.Errorf("create diff directory: %w", err) + } + f, err := os.Create(diffPath) + if err != nil { + return fmt.Errorf("create diff file: %w", err) + } + defer f.Close() + if err := png.Encode(f, diff); err != nil { + return fmt.Errorf("encode diff PNG: %w", err) + } + return nil +} + +func differingPixelMask( + expected, actual image.Image, + expectedBounds, actualBounds image.Rectangle, + width, height int, +) []bool { + maxColorDistance := math.Sqrt(255.0 * 255.0 * 3) + differenceLimit := math.Pow(maestroPixelTolerance*maxColorDistance, 2) + mask := make([]bool, width*height) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + er, eg, eb, _ := expected.At(expectedBounds.Min.X+x, expectedBounds.Min.Y+y).RGBA() + ar, ag, ab, _ := actual.At(actualBounds.Min.X+x, actualBounds.Min.Y+y).RGBA() + dr := float64(int(ar>>8) - int(er>>8)) + dg := float64(int(ag>>8) - int(eg>>8)) + db := float64(int(ab>>8) - int(eb>>8)) + if dr*dr+dg*dg+db*db > differenceLimit { + mask[y*width+x] = true + } + } + } + return mask +} + +func diffBoundingRects(mask []bool, width, height int) []diffRect { + visited := make([]bool, len(mask)) + var rects []diffRect + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + idx := y*width + x + if !mask[idx] || visited[idx] { + continue + } + r := floodDiffRect(mask, visited, width, height, x, y) + rects = append(rects, r) + } + } + return rects +} + +func floodDiffRect(mask, visited []bool, width, height, startX, startY int) diffRect { + type point struct{ x, y int } + queue := []point{{startX, startY}} + visited[startY*width+startX] = true + r := diffRect{minX: startX, minY: startY, maxX: startX, maxY: startY} + + for len(queue) > 0 { + p := queue[0] + queue = queue[1:] + if p.x < r.minX { + r.minX = p.x + } + if p.y < r.minY { + r.minY = p.y + } + if p.x > r.maxX { + r.maxX = p.x + } + if p.y > r.maxY { + r.maxY = p.y + } + for _, n := range []point{ + {p.x - 1, p.y}, {p.x + 1, p.y}, {p.x, p.y - 1}, {p.x, p.y + 1}, + } { + if n.x < 0 || n.y < 0 || n.x >= width || n.y >= height { + continue + } + idx := n.y*width + n.x + if !mask[idx] || visited[idx] { + continue + } + visited[idx] = true + queue = append(queue, n) + } + } + return r +} + +func mergeDiffRects(rects []diffRect, pad int) []diffRect { + if len(rects) == 0 { + return nil + } + merged := append([]diffRect(nil), rects...) + changed := true + for changed { + changed = false + next := make([]diffRect, 0, len(merged)) + used := make([]bool, len(merged)) + for i := range merged { + if used[i] { + continue + } + cur := merged[i] + for j := i + 1; j < len(merged); j++ { + if used[j] { + continue + } + if diffRectsOverlapOrNear(cur, merged[j], pad) { + cur = unionDiffRect(cur, merged[j]) + used[j] = true + changed = true + } + } + used[i] = true + next = append(next, cur) + } + merged = next + } + return merged +} + +func diffRectsOverlapOrNear(a, b diffRect, pad int) bool { + return !(a.maxX+pad < b.minX || b.maxX+pad < a.minX || a.maxY+pad < b.minY || b.maxY+pad < a.minY) +} + +func unionDiffRect(a, b diffRect) diffRect { + return diffRect{ + minX: min(a.minX, b.minX), + minY: min(a.minY, b.minY), + maxX: max(a.maxX, b.maxX), + maxY: max(a.maxY, b.maxY), + } +} + +func expandDiffRect(r diffRect, width, height, minSize int) diffRect { + w := r.maxX - r.minX + 1 + h := r.maxY - r.minY + 1 + if w < minSize { + extra := minSize - w + r.minX -= extra / 2 + r.maxX += extra - extra/2 + } + if h < minSize { + extra := minSize - h + r.minY -= extra / 2 + r.maxY += extra - extra/2 + } + if r.minX < 0 { + r.minX = 0 + } + if r.minY < 0 { + r.minY = 0 + } + if r.maxX >= width { + r.maxX = width - 1 + } + if r.maxY >= height { + r.maxY = height - 1 + } + return r +} + +func drawDiffRectangle(img *image.RGBA, r diffRect, lineWidth int) { + if lineWidth < 1 { + lineWidth = 1 + } + fill := color.RGBA{R: 255, A: 48} // translucent red wash + border := color.RGBA{R: 255, A: 255} + + for y := r.minY; y <= r.maxY; y++ { + for x := r.minX; x <= r.maxX; x++ { + onBorder := x < r.minX+lineWidth || x > r.maxX-lineWidth || + y < r.minY+lineWidth || y > r.maxY-lineWidth + if onBorder { + img.SetRGBA(x, y, border) + continue + } + under := img.RGBAAt(x, y) + img.SetRGBA(x, y, blendRGBA(under, fill)) + } + } +} + +func blendRGBA(dst, src color.RGBA) color.RGBA { + if src.A == 0 { + return dst + } + if src.A == 255 { + return src + } + inv := 255 - uint16(src.A) + return color.RGBA{ + R: uint8((uint16(src.R)*uint16(src.A) + uint16(dst.R)*inv) / 255), + G: uint8((uint16(src.G)*uint16(src.A) + uint16(dst.G)*inv) / 255), + B: uint8((uint16(src.B)*uint16(src.A) + uint16(dst.B)*inv) / 255), + A: 255, + } +} diff --git a/pkg/core/imagediff_test.go b/pkg/core/imagediff_test.go index b30d3ca3..b2da98d3 100644 --- a/pkg/core/imagediff_test.go +++ b/pkg/core/imagediff_test.go @@ -5,6 +5,8 @@ import ( "image" "image/color" "image/png" + "os" + "path/filepath" "testing" ) @@ -87,3 +89,110 @@ func TestCheckImageDifference_ReturnsErrorOnDecodeFail(t *testing.T) { t.Error("expected decode error") } } + +func TestCheckImageMatchPercentage_UsesMaestroColorTolerance(t *testing.T) { + expected := encodePNG(t, 10, 10, color.RGBA{R: 100, G: 100, B: 100, A: 255}) + withinTolerance := encodePNG(t, 10, 10, color.RGBA{R: 120, G: 100, B: 100, A: 255}) + + match, err := CheckImageMatchPercentage(expected, withinTolerance) + if err != nil { + t.Fatalf("CheckImageMatchPercentage() error = %v", err) + } + if match != 100 { + t.Errorf("match = %f, want 100", match) + } +} + +func TestCheckImageMatchPercentage_CountsPixelsOutsideTolerance(t *testing.T) { + expected := encodePNG(t, 10, 10, color.RGBA{R: 100, G: 100, B: 100, A: 255}) + outsideTolerance := encodePNG(t, 10, 10, color.RGBA{R: 200, G: 100, B: 100, A: 255}) + + match, err := CheckImageMatchPercentage(expected, outsideTolerance) + if err != nil { + t.Fatalf("CheckImageMatchPercentage() error = %v", err) + } + if match != 0 { + t.Errorf("match = %f, want 0", match) + } +} + +func TestCheckImageMatchPercentage_ReportsSizeMismatch(t *testing.T) { + expected := encodePNG(t, 10, 10, color.Black) + actual := encodePNG(t, 20, 10, color.Black) + + _, err := CheckImageMatchPercentage(expected, actual) + if err == nil { + t.Fatal("expected size mismatch error") + } +} + +func TestDiffScreenshotPath(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"screen.png", "screen_diff.png"}, + {"/tmp/a/b.png", "/tmp/a/b_diff.png"}, + {"screen", "screen_diff.png"}, + {"screen.JPEG", "screen_diff.JPEG"}, + } + for _, tt := range tests { + if got := DiffScreenshotPath(tt.in); got != tt.want { + t.Errorf("DiffScreenshotPath(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestWriteScreenshotDiff_DrawsRectangleOverlay(t *testing.T) { + // 80×80 gray expected. Actual has a solid block of differing pixels in the center. + expected := encodePNG(t, 80, 80, color.RGBA{R: 100, G: 100, B: 100, A: 255}) + actualImg := image.NewRGBA(image.Rect(0, 0, 80, 80)) + for y := 0; y < 80; y++ { + for x := 0; x < 80; x++ { + if x >= 30 && x < 50 && y >= 30 && y < 50 { + actualImg.Set(x, y, color.RGBA{R: 200, G: 100, B: 100, A: 255}) + } else { + actualImg.Set(x, y, color.RGBA{R: 100, G: 100, B: 100, A: 255}) + } + } + } + var actualBuf bytes.Buffer + if err := png.Encode(&actualBuf, actualImg); err != nil { + t.Fatalf("encode actual: %v", err) + } + + diffPath := filepath.Join(t.TempDir(), "screen_diff.png") + if err := WriteScreenshotDiff(expected, actualBuf.Bytes(), diffPath); err != nil { + t.Fatalf("WriteScreenshotDiff() error = %v", err) + } + + data, err := os.ReadFile(diffPath) + if err != nil { + t.Fatalf("read diff: %v", err) + } + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + t.Fatalf("decode diff: %v", err) + } + + // Corner of the image should remain unchanged (outside any overlay). + r, g, b, _ := img.At(0, 0).RGBA() + if r>>8 != 100 || g>>8 != 100 || b>>8 != 100 { + t.Errorf("unaffected corner = rgb(%d,%d,%d), want gray", r>>8, g>>8, b>>8) + } + + // Border of the expanded/merged rectangle should be solid red. + foundBorder := false + for y := 20; y < 60 && !foundBorder; y++ { + for x := 20; x < 60; x++ { + rr, gg, bb, _ := img.At(x, y).RGBA() + if rr>>8 == 255 && gg>>8 == 0 && bb>>8 == 0 { + foundBorder = true + break + } + } + } + if !foundBorder { + t.Error("expected a red rectangle border around the differing region") + } +} diff --git a/pkg/device/android.go b/pkg/device/android.go index 1dc0025f..c4af0e78 100644 --- a/pkg/device/android.go +++ b/pkg/device/android.go @@ -216,6 +216,11 @@ func (d *AndroidDevice) Shell(cmd string) (string, error) { return d.adb("shell", cmd) } +// Screenshot captures the physical display through ADB. +func (d *AndroidDevice) Screenshot() ([]byte, error) { + return d.adbOutput("exec-out", "screencap", "-p") +} + // Install installs an APK on the device. func (d *AndroidDevice) Install(apkPath string) error { _, err := d.adb("install", "-r", "-g", apkPath) @@ -355,6 +360,11 @@ func (d *AndroidDevice) Info() (DeviceInfo, error) { // adb executes an ADB command. func (d *AndroidDevice) adb(args ...string) (string, error) { + out, err := d.adbOutput(args...) + return string(out), err +} + +func (d *AndroidDevice) adbOutput(args ...string) ([]byte, error) { cmdArgs := make([]string, 0, len(args)+2) if d.serial != "" { cmdArgs = append(cmdArgs, "-s", d.serial) @@ -371,10 +381,10 @@ func (d *AndroidDevice) adb(args ...string) (string, error) { if errMsg == "" { errMsg = stdout.String() } - return "", fmt.Errorf("adb %s: %w: %s", strings.Join(args, " "), err, errMsg) + return nil, fmt.Errorf("adb %s: %w: %s", strings.Join(args, " "), err, errMsg) } - return stdout.String(), nil + return stdout.Bytes(), nil } // waitForDevice waits for the device to be available. diff --git a/pkg/driver/appium/driver.go b/pkg/driver/appium/driver.go index f57c0b83..558df309 100644 --- a/pkg/driver/appium/driver.go +++ b/pkg/driver/appium/driver.go @@ -198,6 +198,8 @@ func (d *Driver) executeStep(step flow.Step) *core.CommandResult { return d.inputRandom(s) case *flow.TakeScreenshotStep: return d.takeScreenshot(s) + case *flow.AssertScreenshotStep: + return d.takeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) default: return errorResult(fmt.Errorf("unsupported step type: %T", step), "") } diff --git a/pkg/driver/browser/cdp/driver.go b/pkg/driver/browser/cdp/driver.go index 310e61fc..7001f3c9 100644 --- a/pkg/driver/browser/cdp/driver.go +++ b/pkg/driver/browser/cdp/driver.go @@ -530,6 +530,8 @@ func (d *Driver) Execute(step flow.Step) *core.CommandResult { // Media case *flow.TakeScreenshotStep: result = d.takeScreenshot(s) + case *flow.AssertScreenshotStep: + result = d.takeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) // Alert handling case *flow.AcceptAlertStep: diff --git a/pkg/driver/devicelab/driver.go b/pkg/driver/devicelab/driver.go index 506da525..ad4f2868 100644 --- a/pkg/driver/devicelab/driver.go +++ b/pkg/driver/devicelab/driver.go @@ -567,6 +567,8 @@ func (d *Driver) Execute(step flow.Step) *core.CommandResult { // Media case *flow.TakeScreenshotStep: result = d.takeScreenshot(s) + case *flow.AssertScreenshotStep: + result = d.takeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) case *flow.StartRecordingStep: result = d.startRecording(s) case *flow.StopRecordingStep: diff --git a/pkg/driver/devicelab_ios/commands.go b/pkg/driver/devicelab_ios/commands.go index 2ce94059..cce93731 100644 --- a/pkg/driver/devicelab_ios/commands.go +++ b/pkg/driver/devicelab_ios/commands.go @@ -46,6 +46,8 @@ func (d *Driver) executeStep(step flow.Step) *core.CommandResult { return d.handleAssertNotVisible(s) case *flow.TakeScreenshotStep: return d.handleTakeScreenshot(s) + case *flow.AssertScreenshotStep: + return d.handleTakeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) case *flow.PressKeyStep: return d.handlePressKey(s) case *flow.WaitForAnimationToEndStep: diff --git a/pkg/driver/uiautomator2/commands.go b/pkg/driver/uiautomator2/commands.go index 66eaeaa1..00607698 100644 --- a/pkg/driver/uiautomator2/commands.go +++ b/pkg/driver/uiautomator2/commands.go @@ -1489,7 +1489,7 @@ func (d *Driver) openLink(step *flow.OpenLinkStep) *core.CommandResult { // ============================================================================ func (d *Driver) takeScreenshot(step *flow.TakeScreenshotStep) *core.CommandResult { - data, err := d.client.Screenshot() + data, err := d.Screenshot() if err != nil { return errorResult(err, fmt.Sprintf("Failed to take screenshot: %v", err)) } diff --git a/pkg/driver/uiautomator2/driver.go b/pkg/driver/uiautomator2/driver.go index 742b18e6..37e6de2a 100644 --- a/pkg/driver/uiautomator2/driver.go +++ b/pkg/driver/uiautomator2/driver.go @@ -19,6 +19,10 @@ type ShellExecutor interface { Shell(cmd string) (string, error) } +type screenshotExecutor interface { + Screenshot() ([]byte, error) +} + // UIA2Client defines the interface for UIAutomator2 client operations. // Implemented by uiautomator2.Client. Allows mocking in tests. type UIA2Client interface { @@ -225,6 +229,8 @@ func (d *Driver) Execute(step flow.Step) *core.CommandResult { // Media case *flow.TakeScreenshotStep: result = d.takeScreenshot(s) + case *flow.AssertScreenshotStep: + result = d.takeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) case *flow.StartRecordingStep: result = d.startRecording(s) case *flow.StopRecordingStep: @@ -256,6 +262,11 @@ func (d *Driver) Execute(step flow.Step) *core.CommandResult { // Screenshot captures the current screen as PNG. func (d *Driver) Screenshot() ([]byte, error) { + if device, ok := d.device.(screenshotExecutor); ok { + if data, err := device.Screenshot(); err == nil && len(data) > 0 { + return data, nil + } + } return d.client.Screenshot() } diff --git a/pkg/driver/uiautomator2/driver_test.go b/pkg/driver/uiautomator2/driver_test.go index 10a600f4..4f05b932 100644 --- a/pkg/driver/uiautomator2/driver_test.go +++ b/pkg/driver/uiautomator2/driver_test.go @@ -197,9 +197,11 @@ func (m *MockUIA2Client) SetAppiumSettings(settings map[string]interface{}) erro // ============================================================================ type MockShellExecutor struct { - commands []string - response string - err error + commands []string + response string + err error + screenshotData []byte + screenshotErr error } func (m *MockShellExecutor) Shell(cmd string) (string, error) { @@ -207,6 +209,10 @@ func (m *MockShellExecutor) Shell(cmd string) (string, error) { return m.response, m.err } +func (m *MockShellExecutor) Screenshot() ([]byte, error) { + return m.screenshotData, m.screenshotErr +} + // ============================================================================ // Build Selectors Tests // ============================================================================ @@ -1100,6 +1106,38 @@ func TestScreenshotError(t *testing.T) { } } +func TestScreenshotPrefersDeviceCapture(t *testing.T) { + deviceData := []byte("adb screenshot") + client := &MockUIA2Client{ + screenshotData: []byte("uia2 screenshot"), + } + device := &MockShellExecutor{screenshotData: deviceData} + driver := New(client, nil, device) + + data, err := driver.Screenshot() + if err != nil { + t.Fatalf("Screenshot failed: %v", err) + } + if string(data) != string(deviceData) { + t.Errorf("Screenshot data = %q, want ADB capture %q", data, deviceData) + } +} + +func TestScreenshotFallsBackToUIA2(t *testing.T) { + clientData := []byte("uia2 screenshot") + client := &MockUIA2Client{screenshotData: clientData} + device := &MockShellExecutor{screenshotErr: errors.New("adb capture failed")} + driver := New(client, nil, device) + + data, err := driver.Screenshot() + if err != nil { + t.Fatalf("Screenshot failed: %v", err) + } + if string(data) != string(clientData) { + t.Errorf("Screenshot data = %q, want UIA2 fallback %q", data, clientData) + } +} + func TestHierarchy(t *testing.T) { client := &MockUIA2Client{ sourceData: "", diff --git a/pkg/driver/wda/driver.go b/pkg/driver/wda/driver.go index 571bd309..9d9e8e15 100644 --- a/pkg/driver/wda/driver.go +++ b/pkg/driver/wda/driver.go @@ -299,6 +299,8 @@ func (d *Driver) Execute(step flow.Step) *core.CommandResult { // Media case *flow.TakeScreenshotStep: result = d.takeScreenshot(s) + case *flow.AssertScreenshotStep: + result = d.takeScreenshot(&flow.TakeScreenshotStep{CropOn: s.CropOn}) // Airplane mode case *flow.SetAirplaneModeStep: diff --git a/pkg/executor/flow_runner.go b/pkg/executor/flow_runner.go index 377f99fb..73c2d468 100644 --- a/pkg/executor/flow_runner.go +++ b/pkg/executor/flow_runner.go @@ -3,6 +3,7 @@ package executor import ( "context" "fmt" + "os" "path/filepath" "strings" "time" @@ -472,19 +473,15 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin // TakeScreenshot - delegate to driver, then save the returned PNG data case *flow.TakeScreenshotStep: - result = fr.driver.Execute(step) - if result.Success { - if data, ok := result.Data.([]byte); ok && len(data) > 0 { - path, saveErr := fr.flowWriter.SaveNamedScreenshot(idx, s.Path, data) - if saveErr != nil { - logger.Warn("Failed to save screenshot: %v", saveErr) - } else { - artifacts.ScreenshotAfter = path - result.Message = fmt.Sprintf("Screenshot saved: %s", filepath.Base(path)) - } - } + var reportPath string + result, reportPath = fr.executeTakeScreenshot(s, idx) + if reportPath != "" { + artifacts.ScreenshotAfter = reportPath } + case *flow.AssertScreenshotStep: + result = fr.executeAssertScreenshot(s) + // PasteText - use in-memory copiedText first, clipboard as fallback case *flow.PasteTextStep: text := fr.script.GetCopiedText() @@ -574,6 +571,169 @@ func (fr *FlowRunner) executeStep(idx int, step flow.Step) (report.Status, strin return status, errorMsg, stepDuration } +func (fr *FlowRunner) executeTakeScreenshot( + step *flow.TakeScreenshotStep, + commandIndex int, +) (*core.CommandResult, string) { + result := fr.driver.Execute(step) + if !result.Success { + return result, "" + } + + data, ok := result.Data.([]byte) + if !ok || len(data) == 0 { + return result, "" + } + + requestedPath := "" + if step.Path != "" { + requestedPath = fr.script.ResolvePath(step.Path) + if filepath.Ext(requestedPath) == "" { + requestedPath += ".png" + } + if err := os.MkdirAll(filepath.Dir(requestedPath), 0o755); err != nil { + err = fmt.Errorf("create screenshot directory for %q: %w", requestedPath, err) + return &core.CommandResult{Success: false, Error: err, Message: err.Error()}, "" + } + if err := os.WriteFile(requestedPath, data, 0o644); err != nil { + err = fmt.Errorf("write screenshot %q: %w", requestedPath, err) + return &core.CommandResult{Success: false, Error: err, Message: err.Error()}, "" + } + } + + reportName := "" + if step.Path != "" { + reportName = filepath.Base(step.Path) + } + reportPath, reportErr := fr.flowWriter.SaveNamedScreenshot(commandIndex, reportName, data) + if reportErr != nil { + logger.Warn("Failed to save screenshot report artifact: %v", reportErr) + } + + if requestedPath != "" { + result.Message = fmt.Sprintf("Screenshot saved: %s", requestedPath) + } else if reportErr == nil { + result.Message = fmt.Sprintf("Screenshot saved: %s", filepath.Base(reportPath)) + } + return result, reportPath +} + +func (fr *FlowRunner) executeAssertScreenshot(step *flow.AssertScreenshotStep) *core.CommandResult { + result := fr.driver.Execute(step) + if !result.Success { + return result + } + + capturedData, ok := result.Data.([]byte) + if !ok || len(capturedData) == 0 { + err := fmt.Errorf("screenshot capture returned no image data") + return &core.CommandResult{ + Success: false, + Error: err, + Message: err.Error(), + } + } + + referencePath := fr.script.ResolvePath(step.Path) + if filepath.Ext(referencePath) == "" { + referencePath += ".png" + } + + referenceData, err := os.ReadFile(referencePath) + if err != nil { + if !os.IsNotExist(err) { + err = fmt.Errorf("read reference screenshot %q: %w", referencePath, err) + return &core.CommandResult{ + Success: false, + Error: err, + Message: err.Error(), + } + } + if writeErr := writeScreenshotBaseline(referencePath, capturedData); writeErr != nil { + return &core.CommandResult{ + Success: false, + Error: writeErr, + Message: writeErr.Error(), + } + } + return &core.CommandResult{ + Success: true, + Message: fmt.Sprintf("Baseline screenshot created: %s", referencePath), + Data: capturedData, + } + } + + if fr.config.UpdateScreenshots { + if writeErr := writeScreenshotBaseline(referencePath, capturedData); writeErr != nil { + return &core.CommandResult{ + Success: false, + Error: writeErr, + Message: writeErr.Error(), + } + } + return &core.CommandResult{ + Success: true, + Message: fmt.Sprintf("Baseline screenshot updated: %s", referencePath), + Data: capturedData, + } + } + + matchPercentage, err := core.CheckImageMatchPercentage(referenceData, capturedData) + if err != nil { + err = fmt.Errorf("compare screenshot with %q: %w", referencePath, err) + return &core.CommandResult{ + Success: false, + Error: err, + Message: err.Error(), + } + } + + if matchPercentage < step.ThresholdPercentage { + diffPath := core.DiffScreenshotPath(referencePath) + diffHint := "" + if writeErr := core.WriteScreenshotDiff(referenceData, capturedData, diffPath); writeErr != nil { + logger.Warn("Failed to write screenshot diff: %v", writeErr) + } else { + diffHint = fmt.Sprintf(". Check the diff image at %s", diffPath) + } + err = fmt.Errorf( + "screenshot match %.2f%% is below threshold %.2f%%", + matchPercentage, + step.ThresholdPercentage, + ) + return &core.CommandResult{ + Success: false, + Error: err, + Message: fmt.Sprintf( + "Screenshot mismatch: %.2f%% match (threshold: %.2f%%)%s", + matchPercentage, + step.ThresholdPercentage, + diffHint, + ), + } + } + + return &core.CommandResult{ + Success: true, + Message: fmt.Sprintf( + "Screenshot matches: %.2f%% (threshold: %.2f%%)", + matchPercentage, + step.ThresholdPercentage, + ), + Data: capturedData, + } +} + +func writeScreenshotBaseline(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create screenshot baseline directory for %q: %w", path, err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write screenshot baseline %q: %w", path, err) + } + return nil +} + // maxPrepareScanDepth bounds runFlow expansion during pre-session scanning so a // cyclic or deeply nested subflow graph can't loop forever. const maxPrepareScanDepth = 10 @@ -1060,18 +1220,10 @@ func (fr *FlowRunner) executeNestedStep(step flow.Step) *core.CommandResult { result = fr.driver.Execute(step) case *flow.TakeScreenshotStep: fr.script.ExpandStep(step) - result = fr.driver.Execute(step) - if result.Success { - if data, ok := result.Data.([]byte); ok && len(data) > 0 { - subIdx := len(fr.subCommands) - path, saveErr := fr.flowWriter.SaveNamedScreenshot(subIdx, s.Path, data) - if saveErr != nil { - logger.Warn("Failed to save nested screenshot: %v", saveErr) - } else { - result.Message = fmt.Sprintf("Screenshot saved: %s", filepath.Base(path)) - } - } - } + result, _ = fr.executeTakeScreenshot(s, len(fr.subCommands)) + case *flow.AssertScreenshotStep: + fr.script.ExpandStep(step) + result = fr.executeAssertScreenshot(s) case *flow.EvalBrowserScriptStep: fr.script.ExpandStep(step) result = fr.driver.Execute(step) diff --git a/pkg/executor/runner.go b/pkg/executor/runner.go index 7622adb6..b4f6da1a 100644 --- a/pkg/executor/runner.go +++ b/pkg/executor/runner.go @@ -31,6 +31,10 @@ type RunnerConfig struct { Retries int // Max retries per flow (0 = no retries) Artifacts ArtifactMode // When to capture artifacts + // UpdateScreenshots overwrites existing assertScreenshot baselines. + // Missing baselines are always seeded on first run regardless of this flag. + UpdateScreenshots bool + // Device/App info for reports Device report.Device App report.App diff --git a/pkg/executor/runner_test.go b/pkg/executor/runner_test.go index 64139b15..38f111bb 100644 --- a/pkg/executor/runner_test.go +++ b/pkg/executor/runner_test.go @@ -1,8 +1,12 @@ package executor import ( + "bytes" "context" "fmt" + "image" + imagecolor "image/color" + "image/png" "os" "path/filepath" "sync" @@ -1008,9 +1012,9 @@ func TestRunner_RunFlowStep_WhenTrue_RunsMainBranch(t *testing.T) { Config: flow.Config{Name: "RunFlow When True Test"}, Steps: []flow.Step{ &flow.RunFlowStep{ - BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, - When: &flow.Condition{Visible: &flow.Selector{Text: "Logout"}}, - Steps: []flow.Step{mainTap}, + BaseStep: flow.BaseStep{StepType: flow.StepRunFlow}, + When: &flow.Condition{Visible: &flow.Selector{Text: "Logout"}}, + Steps: []flow.Step{mainTap}, ElseSteps: []flow.Step{elseTap}, }, }, @@ -2315,7 +2319,7 @@ func TestRunner_TakeScreenshotStep_Success(t *testing.T) { flows := []flow.Flow{ { - SourcePath: "test.yaml", + SourcePath: filepath.Join(tmpDir, "test.yaml"), Config: flow.Config{Name: "Screenshot Test"}, Steps: []flow.Step{ &flow.TakeScreenshotStep{ @@ -2340,6 +2344,13 @@ func TestRunner_TakeScreenshotStep_Success(t *testing.T) { if _, err := os.Stat(screenshotPath); err != nil { t.Errorf("screenshot file not created at %s: %v", screenshotPath, err) } + + requestedPath := filepath.Join(tmpDir, "my-screenshot.png") + if data, err := os.ReadFile(requestedPath); err != nil { + t.Errorf("requested screenshot file not created at %s: %v", requestedPath, err) + } else if !bytes.Equal(data, screenshotData) { + t.Error("requested screenshot file does not contain captured data") + } } func TestRunner_TakeScreenshotStep_EmptyName(t *testing.T) { @@ -2536,3 +2547,212 @@ func TestRunner_TakeScreenshotStep_EmptyData(t *testing.T) { t.Errorf("Status = %v, want %v", result.Status, report.StatusPassed) } } + +func TestRunner_AssertScreenshotStep(t *testing.T) { + reference := encodeTestPNG(t, []imagecolor.RGBA{ + {R: 255, A: 255}, + {G: 255, A: 255}, + }) + withinTolerance := encodeTestPNG(t, []imagecolor.RGBA{ + {R: 240, A: 255}, + {G: 240, A: 255}, + }) + mismatch := encodeTestPNG(t, []imagecolor.RGBA{ + {B: 255, A: 255}, + {G: 255, A: 255}, + }) + + tests := []struct { + name string + captured []byte + expectedStatus report.Status + }{ + { + name: "image within Maestro color tolerance passes", + captured: withinTolerance, + expectedStatus: report.StatusPassed, + }, + { + name: "image below threshold fails", + captured: mismatch, + expectedStatus: report.StatusFailed, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "reference.png"), reference, 0o644); err != nil { + t.Fatalf("write reference image: %v", err) + } + + driver := &mockDriver{ + executeFunc: func(step flow.Step) *core.CommandResult { + if _, ok := step.(*flow.AssertScreenshotStep); ok { + return &core.CommandResult{Success: true, Data: tt.captured} + } + return &core.CommandResult{Success: true} + }, + } + runner := New(driver, RunnerConfig{ + OutputDir: filepath.Join(tmpDir, "output"), + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + App: report.App{ID: "com.test"}, + Parallelism: 0, + }) + flows := []flow.Flow{{ + SourcePath: filepath.Join(tmpDir, "test.yaml"), + Config: flow.Config{Name: "Assert Screenshot Test"}, + Steps: []flow.Step{&flow.AssertScreenshotStep{ + BaseStep: flow.BaseStep{StepType: flow.StepAssertScreenshot}, + Path: "reference", + ThresholdPercentage: 100, + }}, + }} + + result, err := runner.Run(context.Background(), flows) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if result.Status != tt.expectedStatus { + t.Errorf("Status = %v, want %v", result.Status, tt.expectedStatus) + } + + diffPath := filepath.Join(tmpDir, "reference_diff.png") + if tt.expectedStatus == report.StatusFailed { + if _, err := os.Stat(diffPath); err != nil { + t.Errorf("expected diff image at %s: %v", diffPath, err) + } + } else if _, err := os.Stat(diffPath); err == nil { + t.Errorf("unexpected diff image written at %s", diffPath) + } + }) + } +} + +func TestRunner_AssertScreenshotStep_SeedsMissingBaseline(t *testing.T) { + tmpDir := t.TempDir() + captured := encodeTestPNG(t, []imagecolor.RGBA{ + {R: 10, A: 255}, + {G: 20, A: 255}, + }) + + driver := &mockDriver{ + executeFunc: func(step flow.Step) *core.CommandResult { + if _, ok := step.(*flow.AssertScreenshotStep); ok { + return &core.CommandResult{Success: true, Data: captured} + } + return &core.CommandResult{Success: true} + }, + } + runner := New(driver, RunnerConfig{ + OutputDir: filepath.Join(tmpDir, "output"), + Artifacts: ArtifactNever, + Device: report.Device{ID: "test", Platform: "android"}, + App: report.App{ID: "com.test"}, + Parallelism: 0, + }) + flows := []flow.Flow{{ + SourcePath: filepath.Join(tmpDir, "test.yaml"), + Config: flow.Config{Name: "Seed Baseline Test"}, + Steps: []flow.Step{&flow.AssertScreenshotStep{ + BaseStep: flow.BaseStep{StepType: flow.StepAssertScreenshot}, + Path: "baselines/new-screen", + ThresholdPercentage: 100, + }}, + }} + + result, err := runner.Run(context.Background(), flows) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if result.Status != report.StatusPassed { + t.Fatalf("Status = %v, want %v", result.Status, report.StatusPassed) + } + + baselinePath := filepath.Join(tmpDir, "baselines", "new-screen.png") + data, err := os.ReadFile(baselinePath) + if err != nil { + t.Fatalf("expected baseline at %s: %v", baselinePath, err) + } + if !bytes.Equal(data, captured) { + t.Error("seeded baseline does not match captured screenshot") + } +} + +func TestRunner_AssertScreenshotStep_UpdateScreenshotsOverwritesBaseline(t *testing.T) { + tmpDir := t.TempDir() + oldBaseline := encodeTestPNG(t, []imagecolor.RGBA{ + {R: 255, A: 255}, + {G: 255, A: 255}, + }) + updated := encodeTestPNG(t, []imagecolor.RGBA{ + {B: 255, A: 255}, + {G: 255, A: 255}, + }) + baselinePath := filepath.Join(tmpDir, "reference.png") + if err := os.WriteFile(baselinePath, oldBaseline, 0o644); err != nil { + t.Fatalf("write baseline: %v", err) + } + + driver := &mockDriver{ + executeFunc: func(step flow.Step) *core.CommandResult { + if _, ok := step.(*flow.AssertScreenshotStep); ok { + return &core.CommandResult{Success: true, Data: updated} + } + return &core.CommandResult{Success: true} + }, + } + runner := New(driver, RunnerConfig{ + OutputDir: filepath.Join(tmpDir, "output"), + Artifacts: ArtifactNever, + UpdateScreenshots: true, + Device: report.Device{ID: "test", Platform: "android"}, + App: report.App{ID: "com.test"}, + Parallelism: 0, + }) + flows := []flow.Flow{{ + SourcePath: filepath.Join(tmpDir, "test.yaml"), + Config: flow.Config{Name: "Update Baseline Test"}, + Steps: []flow.Step{&flow.AssertScreenshotStep{ + BaseStep: flow.BaseStep{StepType: flow.StepAssertScreenshot}, + Path: "reference", + ThresholdPercentage: 100, + }}, + }} + + result, err := runner.Run(context.Background(), flows) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if result.Status != report.StatusPassed { + t.Fatalf("Status = %v, want %v", result.Status, report.StatusPassed) + } + + data, err := os.ReadFile(baselinePath) + if err != nil { + t.Fatalf("read updated baseline: %v", err) + } + if !bytes.Equal(data, updated) { + t.Error("baseline was not overwritten by --update-screenshots") + } + if _, err := os.Stat(filepath.Join(tmpDir, "reference_diff.png")); err == nil { + t.Error("unexpected diff image when updating screenshots") + } +} + +func encodeTestPNG(t *testing.T, pixels []imagecolor.RGBA) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, len(pixels), 1)) + for x, pixel := range pixels { + img.SetRGBA(x, 0, pixel) + } + + var data bytes.Buffer + if err := png.Encode(&data, img); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return data.Bytes() +} diff --git a/pkg/executor/scripting.go b/pkg/executor/scripting.go index 9f5f2c89..e114f095 100644 --- a/pkg/executor/scripting.go +++ b/pkg/executor/scripting.go @@ -700,6 +700,12 @@ func (se *ScriptEngine) ExpandStep(step flow.Step) { s.Link = se.ExpandVariables(s.Link) case *flow.PressKeyStep: s.Key = se.ExpandVariables(s.Key) + case *flow.TakeScreenshotStep: + s.Path = se.ExpandVariables(s.Path) + s.CropOn = se.expandSelector(s.CropOn) + case *flow.AssertScreenshotStep: + s.Path = se.ExpandVariables(s.Path) + s.CropOn = se.expandSelector(s.CropOn) case *flow.RunFlowStep: s.File = se.ExpandVariables(s.File) s.ElseFile = se.ExpandVariables(s.ElseFile) diff --git a/pkg/executor/scripting_test.go b/pkg/executor/scripting_test.go index 8757737d..d7f39668 100644 --- a/pkg/executor/scripting_test.go +++ b/pkg/executor/scripting_test.go @@ -1228,6 +1228,44 @@ func TestScriptEngine_ExpandStep_InputTextStep(t *testing.T) { } } +func TestScriptEngine_ExpandStep_ScreenshotSteps(t *testing.T) { + se := NewScriptEngine() + defer se.Close() + + se.SetVariable("ROOT", "/tmp/screenshots") + se.SetVariable("NAME", "alignment") + se.SetVariable("ELEMENT_ID", "enriched-text") + + take := &flow.TakeScreenshotStep{ + Path: "${ROOT}/${NAME}", + CropOn: &flow.Selector{ID: "${ELEMENT_ID}"}, + } + assert := &flow.AssertScreenshotStep{ + Path: "${ROOT}/${NAME}", + CropOn: &flow.Selector{ID: "${ELEMENT_ID}"}, + } + + se.ExpandStep(take) + se.ExpandStep(assert) + + for name, step := range map[string]struct { + path string + cropOn *flow.Selector + }{ + "takeScreenshot": {path: take.Path, cropOn: take.CropOn}, + "assertScreenshot": {path: assert.Path, cropOn: assert.CropOn}, + } { + t.Run(name, func(t *testing.T) { + if step.path != "/tmp/screenshots/alignment" { + t.Errorf("Path = %q, want %q", step.path, "/tmp/screenshots/alignment") + } + if step.cropOn == nil || step.cropOn.ID != "enriched-text" { + t.Errorf("CropOn = %#v, want id enriched-text", step.cropOn) + } + }) + } +} + func TestScriptEngine_ExpandStep_ScrollUntilVisibleDirection(t *testing.T) { se := NewScriptEngine() defer se.Close() diff --git a/pkg/flow/parser.go b/pkg/flow/parser.go index 4d3ba9c5..bdd9f1b1 100644 --- a/pkg/flow/parser.go +++ b/pkg/flow/parser.go @@ -252,7 +252,7 @@ func isStepType(key string) bool { StepInputText, StepInputRandom, StepInputRandomEmail, StepInputRandomNumber, StepInputRandomPersonName, StepInputRandomText, StepEraseText, StepCopyTextFrom, StepPasteText, StepSetClipboard, - StepAssertVisible, StepAssertNotVisible, StepAssertTrue, StepAssertCondition, + StepAssertVisible, StepAssertNotVisible, StepAssertTrue, StepAssertCondition, StepAssertScreenshot, StepAssertNoDefectsWithAI, StepAssertWithAI, StepExtractTextWithAI, StepWaitUntil, StepLaunchApp, StepStopApp, StepKillApp, StepClearState, StepClearKeychain, StepSetPermissions, StepSetLocation, StepSetOrientation, StepSetAirplaneMode, StepToggleAirplaneMode, @@ -871,6 +871,19 @@ func decodeStep(stepType StepType, valueNode *yaml.Node, sourcePath string) (Ste s.StepType = stepType return &s, nil + case StepAssertScreenshot: + var s AssertScreenshotStep + if valueNode.Kind == yaml.ScalarNode { + s.Path = valueNode.Value + } else if err := valueNode.Decode(&s); err != nil { + return nil, wrapParseError(sourcePath, valueNode.Line, err) + } + if s.ThresholdPercentage == 0 { + s.ThresholdPercentage = 95.0 + } + s.StepType = stepType + return &s, nil + case StepStartRecording: var s StartRecordingStep if valueNode.Kind == yaml.ScalarNode { diff --git a/pkg/flow/parser_test.go b/pkg/flow/parser_test.go index f4812eae..29c89562 100644 --- a/pkg/flow/parser_test.go +++ b/pkg/flow/parser_test.go @@ -182,6 +182,7 @@ func TestParse_AllStepTypes(t *testing.T) { {"waitForRequest mapping", `- waitForRequest: {url: "*/api/submit", method: POST, output: reqBody}`, StepWaitForRequest}, {"clearNetworkMocks", `- clearNetworkMocks:`, StepClearNetworkMocks}, {"takeScreenshot", `- takeScreenshot: "screen.png"`, StepTakeScreenshot}, + {"assertScreenshot", `- assertScreenshot: "screen.png"`, StepAssertScreenshot}, {"startRecording", `- startRecording: "video.mp4"`, StepStartRecording}, {"stopRecording", `- stopRecording: "video.mp4"`, StepStopRecording}, {"addMedia", `- addMedia: {files: ["img.png"]}`, StepAddMedia}, @@ -206,6 +207,50 @@ func TestParse_AllStepTypes(t *testing.T) { } } +func TestParse_AssertScreenshotStep(t *testing.T) { + yaml := ` +- assertScreenshot: + path: screenshots/banner.png + cropOn: + id: banner + thresholdPercentage: 98.5 +` + + parsed, err := Parse([]byte(yaml), "test.yaml") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + step, ok := parsed.Steps[0].(*AssertScreenshotStep) + if !ok { + t.Fatalf("expected AssertScreenshotStep, got %T", parsed.Steps[0]) + } + if step.Path != "screenshots/banner.png" { + t.Errorf("Path = %q, want %q", step.Path, "screenshots/banner.png") + } + if step.CropOn == nil || step.CropOn.ID != "banner" { + t.Errorf("CropOn = %#v, want selector with id banner", step.CropOn) + } + if step.ThresholdPercentage != 98.5 { + t.Errorf("ThresholdPercentage = %v, want 98.5", step.ThresholdPercentage) + } +} + +func TestParse_AssertScreenshotStep_DefaultThreshold(t *testing.T) { + parsed, err := Parse([]byte(`- assertScreenshot: splash.png`), "test.yaml") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + step, ok := parsed.Steps[0].(*AssertScreenshotStep) + if !ok { + t.Fatalf("expected AssertScreenshotStep, got %T", parsed.Steps[0]) + } + if step.ThresholdPercentage != 95 { + t.Errorf("ThresholdPercentage = %v, want 95", step.ThresholdPercentage) + } +} + func TestParse_RepeatStep(t *testing.T) { yaml := ` - repeat: diff --git a/pkg/flow/step.go b/pkg/flow/step.go index 4cf058c0..1b3c8e00 100644 --- a/pkg/flow/step.go +++ b/pkg/flow/step.go @@ -100,11 +100,12 @@ const ( StepClearNetworkMocks StepType = "clearNetworkMocks" // Media - StepTakeScreenshot StepType = "takeScreenshot" - StepStartRecording StepType = "startRecording" - StepStopRecording StepType = "stopRecording" - StepAddMedia StepType = "addMedia" - StepRemoveMedia StepType = "removeMedia" + StepTakeScreenshot StepType = "takeScreenshot" + StepAssertScreenshot StepType = "assertScreenshot" + StepStartRecording StepType = "startRecording" + StepStopRecording StepType = "stopRecording" + StepAddMedia StepType = "addMedia" + StepRemoveMedia StepType = "removeMedia" // Other StepPressKey StepType = "pressKey" @@ -785,6 +786,14 @@ type TakeScreenshotStep struct { CropOn *Selector `yaml:"cropOn,omitempty"` } +// AssertScreenshotStep compares a screenshot with a reference image. +type AssertScreenshotStep struct { + BaseStep `yaml:",inline"` + Path string `yaml:"path"` + CropOn *Selector `yaml:"cropOn,omitempty"` + ThresholdPercentage float64 `yaml:"thresholdPercentage"` +} + // StartRecordingStep starts recording. type StartRecordingStep struct { BaseStep `yaml:",inline"`