diff --git a/.changeset/screenshot-diff-width-normalization.md b/.changeset/screenshot-diff-width-normalization.md new file mode 100644 index 0000000000..c8d309826b --- /dev/null +++ b/.changeset/screenshot-diff-width-normalization.md @@ -0,0 +1,7 @@ +--- +'agent-device': patch +--- + +Compare screenshots at different resolutions without resizing them first. `diff screenshot` scales the current image to the baseline width, aligns both images at the top left, and counts extra or missing bottom rows as differences, even at `--threshold 1`. Your original files stay unchanged. + +For different-size inputs, the result replaces `dimensionMismatch` with `normalization`, which reports original, scaled, and compared dimensions plus coordinate scales. Update consumers that read `dimensionMismatch` to use `normalization`; different-size images can now match after scaling instead of always reporting 100% mismatch. Region and ref-match rectangles use comparison pixels. diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index 40340a4a81..c22d601593 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -234,14 +234,21 @@ export async function decodePngAsync(buffer: Buffer, label: string): Promise { - const result = await runPngJob({ kind: 'decode-image', image: bytes, label }, async () => { - // Read on demand so the JPEG decoder stays out of the import closure of every entry that only - // needs the worker's other jobs. - const { decodeScreenshotImage } = await import('./screenshot-image.ts'); - const image = decodeScreenshotImage(bytes, label); - return { kind: 'decode-image', width: image.width, height: image.height, data: image.data }; - }); +export async function decodeScreenshotImageAsync( + bytes: Buffer, + label: string, + maxPixels?: number, +): Promise { + const result = await runPngJob( + { kind: 'decode-image', image: bytes, label, maxPixels }, + async () => { + // Read on demand so the JPEG decoder stays out of the import closure of every entry that only + // needs the worker's other jobs. + const { decodeScreenshotImage } = await import('./screenshot-image.ts'); + const image = decodeScreenshotImage(bytes, label, maxPixels); + return { kind: 'decode-image', width: image.width, height: image.height, data: image.data }; + }, + ); return toDecodedPng(result); } diff --git a/packages/capture-kit/src/png-worker-contract.ts b/packages/capture-kit/src/png-worker-contract.ts index a24767da8f..0782195b8a 100644 --- a/packages/capture-kit/src/png-worker-contract.ts +++ b/packages/capture-kit/src/png-worker-contract.ts @@ -17,7 +17,7 @@ import type { PngRgbDifferenceResult } from './png-rgb-difference.ts'; export type PngWorkerJob = | { kind: 'decode'; png: Uint8Array; label: string } // A screenshot in whatever container it arrived in; answers decoded pixels, not PNG bytes. - | { kind: 'decode-image'; image: Uint8Array; label: string } + | { kind: 'decode-image'; image: Uint8Array; label: string; maxPixels?: number } | { kind: 'encode'; width: number; height: number; data: Uint8Array } | { kind: 'crop'; png: Uint8Array; label: string; box: Rect } | { kind: 'rgb-difference'; firstPng: Uint8Array; secondPng: Uint8Array; label: string } diff --git a/packages/capture-kit/src/png-worker.ts b/packages/capture-kit/src/png-worker.ts index a177f8f8a6..da551690c9 100644 --- a/packages/capture-kit/src/png-worker.ts +++ b/packages/capture-kit/src/png-worker.ts @@ -28,7 +28,11 @@ async function runJob(request: PngWorkerRequest): Promise { } case 'decode-image': { const { decodeScreenshotImage } = await import('./screenshot-image.ts'); - const image = decodeScreenshotImage(toBuffer(request.image), request.label); + const image = decodeScreenshotImage( + toBuffer(request.image), + request.label, + request.maxPixels, + ); return { kind: 'decode-image', width: image.width, height: image.height, data: image.data }; } case 'encode': { diff --git a/packages/capture-kit/src/screenshot-diff-overlay-matches.ts b/packages/capture-kit/src/screenshot-diff-overlay-matches.ts index afbc8ee5b9..41dfc4f10d 100644 --- a/packages/capture-kit/src/screenshot-diff-overlay-matches.ts +++ b/packages/capture-kit/src/screenshot-diff-overlay-matches.ts @@ -11,9 +11,19 @@ const MAX_MATCHES_PER_REGION = 3; export function attachCurrentOverlayMatches( regions: ScreenshotDiffRegion[], overlayRefs: ScreenshotOverlayRef[], + currentScale = { x: 1, y: 1 }, ): ScreenshotDiffRegion[] { + const scaledRefs = overlayRefs.map((ref) => ({ + ...ref, + overlayRect: { + x: ref.overlayRect.x * currentScale.x, + y: ref.overlayRect.y * currentScale.y, + width: ref.overlayRect.width * currentScale.x, + height: ref.overlayRect.height * currentScale.y, + }, + })); return regions.map((region) => { - const matches = findRegionOverlayMatches(region, overlayRefs); + const matches = findRegionOverlayMatches(region, scaledRefs); return matches.length > 0 ? { ...region, currentOverlayMatches: matches } : region; }); } diff --git a/packages/capture-kit/src/screenshot-diff-pixels.ts b/packages/capture-kit/src/screenshot-diff-pixels.ts index b6a28b67fa..9c92a1c12b 100644 --- a/packages/capture-kit/src/screenshot-diff-pixels.ts +++ b/packages/capture-kit/src/screenshot-diff-pixels.ts @@ -5,6 +5,7 @@ const DIFF_CHANGE_COLOR = { r: 220, g: 0, b: 0 } as const; export type ScreenshotDiffPixelsJob = { width: number; height: number; + overlapHeight?: number; baselineData: Uint8Array; currentData: Uint8Array; maxColorDistance: number; @@ -29,6 +30,7 @@ export function computeScreenshotDiffPixels( const diffData = Buffer.alloc(totalPixels * 4); const diffMask = new Uint8Array(totalPixels); let differentPixels = 0; + const overlapPixels = job.width * (job.overlapHeight ?? job.height); // PNG data is a flat RGBA buffer: [R, G, B, A, R, G, B, A, ...]. // We step by 4 to visit each pixel and compute its Euclidean distance @@ -39,7 +41,7 @@ export function computeScreenshotDiffPixels( const blueDelta = baselineData[index + 2]! - currentData[index + 2]!; const colorDistance = Math.sqrt(redDelta ** 2 + greenDelta ** 2 + blueDelta ** 2); - if (colorDistance > maxColorDistance) { + if (pixelIndex >= overlapPixels || colorDistance > maxColorDistance) { differentPixels += 1; diffMask[pixelIndex] = 1; const context = renderDiffContextChannel(currentData, index); diff --git a/packages/capture-kit/src/screenshot-diff.test.ts b/packages/capture-kit/src/screenshot-diff.test.ts index ba4e4792c0..008ae05291 100644 --- a/packages/capture-kit/src/screenshot-diff.test.ts +++ b/packages/capture-kit/src/screenshot-diff.test.ts @@ -1,4 +1,5 @@ import { test } from 'vitest'; +import fc from 'fast-check'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; @@ -83,7 +84,7 @@ test('identical images produce match: true with 0% mismatch', async () => { assert.equal(result.nonTextDeltas, undefined); assert.equal(Object.hasOwn(result, 'ocr'), false); assert.equal(Object.hasOwn(result, 'nonTextDeltas'), false); - assert.equal(result.dimensionMismatch, undefined); + assert.equal(Object.hasOwn(result, 'dimensionMismatch'), false); assert.equal(result.diffPath, undefined, 'diffPath should not be set when images match'); // No diff image should be written when images match assert.equal(fs.existsSync(diffOut), false); @@ -304,25 +305,36 @@ test('diff image marks changed pixels red over a light current-screen context', assert.equal(diffPng.data[6], 56); // B }); -test('dimension mismatch returns expected vs actual sizes', async () => { +test('different dimensions scale current to baseline width and count missing bottom rows', async () => { const dir = tmpDir(); const baseline = path.join(dir, 'baseline.png'); const current = path.join(dir, 'current.png'); const diffOut = path.join(dir, 'diff.png'); - writeSolidPng(baseline, 10, 20, { r: 0, g: 0, b: 0 }); writeSolidPng(current, 15, 25, { r: 0, g: 0, b: 0 }); + const original = fs.readFileSync(current); - const result = await compareScreenshots(baseline, current, { outputPath: diffOut }); + const result = await compareScreenshots(baseline, current, { outputPath: diffOut, threshold: 1 }); assert.equal(result.match, false); - assert.equal(result.mismatchPercentage, 100); - assert.equal(result.diffPath, undefined, 'diffPath should not be set for dimension mismatch'); - assert.equal(result.regions, undefined); - assert.deepEqual(result.dimensionMismatch, { - expected: { width: 10, height: 20 }, - actual: { width: 15, height: 25 }, + assert.equal(result.differentPixels, 30); + assert.equal(result.totalPixels, 200); + assert.equal(result.mismatchPercentage, 15); + assert.equal(result.diffPath, diffOut); + assert.equal(Object.hasOwn(result, 'dimensionMismatch'), false); + assert.deepEqual(result.normalization, { + baseline: { width: 10, height: 20 }, + current: { width: 15, height: 25 }, + scaledCurrent: { width: 10, height: 17 }, + compared: { width: 10, height: 20 }, + currentScale: { x: 10 / 15, y: 17 / 25 }, + alignment: 'top-left', + coordinateSpace: 'comparison-pixels', }); + assert.deepEqual(result.regions?.[0]?.rect, { x: 0, y: 17, width: 10, height: 3 }); + assert.deepEqual(fs.readFileSync(current), original); + const diff = PNG.sync.read(fs.readFileSync(diffOut)); + assert.deepEqual([diff.width, diff.height], [10, 20]); }); test('threshold controls sensitivity: small differences ignored at default threshold', async () => { @@ -472,21 +484,20 @@ test('a JPEG comparison writes the diff artifact as PNG', async () => { assert.equal(hasPngSignature(fs.readFileSync(diffPath)), true); }); -test('a JPEG input is measured at its decoded dimensions', async () => { +test('a JPEG input scales to baseline width and counts extra rows', async () => { const dir = tmpDir(); const baseline = path.join(dir, 'baseline.jpg'); const current = path.join(dir, 'current.png'); - await writeSolidJpeg(baseline, 8, 6, { r: 20, g: 200, b: 40 }); writeSolidPng(current, 5, 5, { r: 20, g: 200, b: 40 }); const result = await compareScreenshots(baseline, current); assert.equal(result.match, false); - assert.deepEqual(result.dimensionMismatch, { - expected: { width: 8, height: 6 }, - actual: { width: 5, height: 5 }, - }); + assert.equal(result.totalPixels, 64); + assert.equal(result.differentPixels, 16); + assert.deepEqual(result.normalization?.scaledCurrent, { width: 8, height: 8 }); + assert.deepEqual(result.normalization?.compared, { width: 8, height: 8 }); }); test('JPEG bytes that do not decode are refused with the JPEG decode error', async () => { @@ -543,3 +554,189 @@ test('mismatchPercentage is rounded to 2 decimal places', async () => { assert.equal(result.mismatchPercentage, 33.33); }); + +test('375x812 reference compares with a 1080x2340 capture and retains the extra row', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'reference.png'); + const current = path.join(dir, 'capture.png'); + writeSolidPng(baseline, 375, 812, { r: 30, g: 80, b: 120 }); + writeSolidPng(current, 1080, 2340, { r: 30, g: 80, b: 120 }); + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + assert.equal(result.differentPixels, 375); + assert.equal(result.totalPixels, 304875); + assert.equal(result.mismatchPercentage, 0.12); + assert.deepEqual(result.normalization?.scaledCurrent, { width: 375, height: 813 }); +}); + +test.each([0, 255])( + 'transparent and blank tails count in either direction (alpha %i)', + async (alpha) => { + const dir = tmpDir(); + const short = path.join(dir, 'short.png'); + const tall = path.join(dir, 'tall.png'); + for (const [file, height] of [ + [short, 2], + [tall, 4], + ] as const) { + const png = new PNG({ width: 4, height }); + for (let offset = 3; offset < png.data.length; offset += 4) png.data[offset] = alpha; + fs.writeFileSync(file, PNG.sync.write(png)); + } + for (const [baseline, current] of [ + [short, tall], + [tall, short], + ]) { + const result = await compareScreenshots(baseline!, current!, { threshold: 1 }); + assert.equal(result.differentPixels, 8); + assert.equal(result.totalPixels, 16); + assert.equal(result.mismatchPercentage, 50); + assert.deepEqual(result.regions?.[0]?.rect, { x: 0, y: 2, width: 4, height: 2 }); + assert.deepEqual(result.regions?.[0]?.normalizedRect, { + x: 0, + y: 50, + width: 100, + height: 50, + }); + } + }, +); + +test('scaled content stays aligned at top left and regions use comparison pixels', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + const source = new PNG({ width: 4, height: 4 }); + paintRect(source, { x: 0, y: 0, width: 2, height: 2 }, { r: 255, g: 255, b: 255 }); + fs.writeFileSync(current, PNG.sync.write(source)); + writeSolidPng(baseline, 2, 2, { r: 0, g: 0, b: 0 }); + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + assert.equal(result.differentPixels, 1); + assert.deepEqual(result.regions?.[0]?.rect, { x: 0, y: 0, width: 1, height: 1 }); + assert.deepEqual(result.regions?.[0]?.normalizedRect, { x: 0, y: 0, width: 50, height: 50 }); +}); + +const PROPERTY_RUNS = 40; +test('proportional captures match at narrower, wider and equal widths', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + await fc.assert( + fc.asyncProperty( + fc.record({ + width: fc.integer({ min: 1, max: 12 }), + height: fc.integer({ min: 1, max: 12 }), + baselineDensity: fc.integer({ min: 1, max: 4 }), + currentDensity: fc.integer({ min: 1, max: 4 }), + }), + async ({ width, height, baselineDensity, currentDensity }) => { + writeSolidPng(baseline, width * baselineDensity, height * baselineDensity, { + r: 32, + g: 80, + b: 120, + }); + writeSolidPng(current, width * currentDensity, height * currentDensity, { + r: 32, + g: 80, + b: 120, + }); + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + assert.equal(result.match, true); + assert.equal(result.totalPixels, width * height * baselineDensity ** 2); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); +}); + +test.each([ + [10, 10, 20, 20, 399], // original current exceeds budget even though downscaling fits + [20, 20, 10, 10, 399], // original baseline exceeds budget + [10, 1, 1, 100, 100], // both originals fit; normalized/union canvas exceeds budget +])( + 'pixel budget bounds original and scaled images (%i,%i vs %i,%i)', + async (bw, bh, cw, ch, maxPixels) => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + writeSolidPng(baseline, bw, bh, { r: 0, g: 0, b: 0 }); + writeSolidPng(current, cw, ch, { r: 0, g: 0, b: 0 }); + await assert.rejects(compareScreenshots(baseline, current, { maxPixels }), { + code: 'INVALID_ARGS', + }); + }, +); + +test('fractional downscaling keeps at least one row and accepts the exact union budget', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + writeSolidPng(baseline, 1, 1, { r: 0, g: 0, b: 0 }); + writeSolidPng(current, 100, 1, { r: 0, g: 0, b: 0 }); + const result = await compareScreenshots(baseline, current, { maxPixels: 100 }); + assert.equal(result.match, true); + assert.deepEqual(result.normalization?.scaledCurrent, { width: 1, height: 1 }); +}); + +test.each([ + [4, 2, 8, 4], + [8, 4, 4, 2], +])( + 'proportional patterned screenshots match at either density (%i x %i vs %i x %i)', + async (bw, bh, cw, ch) => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + for (const [file, width, height] of [ + [baseline, bw, bh], + [current, cw, ch], + ] as const) { + const png = new PNG({ width, height }); + paintRect(png, { x: 0, y: 0, width: width / 2, height: height / 2 }, { r: 255, g: 0, b: 0 }); + paintRect( + png, + { x: width / 2, y: height / 2, width: width / 2, height: height / 2 }, + { r: 0, g: 80, b: 255 }, + ); + fs.writeFileSync(file, PNG.sync.write(png)); + } + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + assert.equal(result.match, true); + assert.equal(result.totalPixels, bw * bh); + }, +); + +test('JPEG current screenshots use proportional normalization and the original pixel budget', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.jpg'); + writeSolidPng(baseline, 4, 3, { r: 20, g: 200, b: 40 }); + await writeSolidJpeg(current, 8, 6, { r: 20, g: 200, b: 40 }); + const result = await compareScreenshots(baseline, current, { maxPixels: 48 }); + assert.equal(result.match, true); + assert.equal(result.totalPixels, 12); + await assert.rejects(compareScreenshots(baseline, current, { maxPixels: 47 }), { + code: 'INVALID_ARGS', + }); +}); + +test('unequal-height screenshots keep shared content at the top left', async () => { + const dir = tmpDir(); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + for (const [file, width, height] of [ + [baseline, 2, 2], + [current, 4, 6], + ] as const) { + const image = new PNG({ width, height }); + paintRect( + image, + { x: 0, y: 0, width: width / 2, height: width / 2 }, + { r: 255, g: 255, b: 255 }, + ); + fs.writeFileSync(file, PNG.sync.write(image)); + } + const result = await compareScreenshots(baseline, current, { threshold: 0 }); + assert.equal(result.differentPixels, 2); + assert.equal(result.totalPixels, 6); + assert.deepEqual(result.regions?.[0]?.rect, { x: 0, y: 2, width: 2, height: 1 }); +}); diff --git a/packages/capture-kit/src/screenshot-diff.ts b/packages/capture-kit/src/screenshot-diff.ts index 500664f046..c2a40477bc 100644 --- a/packages/capture-kit/src/screenshot-diff.ts +++ b/packages/capture-kit/src/screenshot-diff.ts @@ -12,11 +12,6 @@ import { annotateDiffRegions } from './screenshot-diff-region-overlay.ts'; import { summarizeDiffRegions, type ScreenshotDiffRegion } from './screenshot-diff-regions.ts'; import type { ImageDimensions } from '@agent-device/kernel/screenshot-geometry'; -export type ScreenshotDimensionMismatch = { - expected: ImageDimensions; - actual: ImageDimensions; -}; - type ScreenshotOcrSummary = { provider: 'tesseract'; baselineBlocks: number; @@ -45,13 +40,24 @@ type ScreenshotNonTextDelta = { nearestText?: string; }; +export type ScreenshotDiffNormalization = { + baseline: ImageDimensions; + current: ImageDimensions; + scaledCurrent: ImageDimensions; + compared: ImageDimensions; + currentScale: { x: number; y: number }; + alignment: 'top-left'; + coordinateSpace: 'comparison-pixels'; +}; + export type ScreenshotDiffResult = { diffPath?: string; totalPixels: number; differentPixels: number; mismatchPercentage: number; match: boolean; - dimensionMismatch?: ScreenshotDimensionMismatch; + /** Present when input dimensions differ. Regions and their ref rectangles use comparison pixels. */ + normalization?: ScreenshotDiffNormalization; regions?: ScreenshotDiffRegion[]; currentOverlayPath?: string; currentOverlayRefCount?: number; @@ -96,39 +102,36 @@ export async function compareScreenshots( ]); const [baseline, current] = await Promise.all([ - decodeScreenshotImageAsync(baselineBuffer, 'baseline screenshot'), - decodeScreenshotImageAsync(currentBuffer, 'current screenshot'), + decodeScreenshotImageAsync(baselineBuffer, 'baseline screenshot', options.maxPixels), + decodeScreenshotImageAsync(currentBuffer, 'current screenshot', options.maxPixels), ]); - validateMaxPixels(baseline.width, baseline.height, 'baseline screenshot', options.maxPixels); - validateMaxPixels(current.width, current.height, 'current screenshot', options.maxPixels); - - const threshold = options.threshold ?? 0.1; - - // Handle dimension mismatch — no diff image can be generated for different-sized images - if (baseline.width !== current.width || baseline.height !== current.height) { - const totalPixels = baseline.width * baseline.height; - await removeStaleDiffOutput(options.outputPath); - return { - match: false, - mismatchPercentage: 100, - totalPixels, - differentPixels: totalPixels, - dimensionMismatch: { - expected: { width: baseline.width, height: baseline.height }, - actual: { width: current.width, height: current.height }, - }, - }; + const normalization = screenshotDiffNormalization(baseline, current); + const compared = normalization?.compared ?? baseline; + const { validateScreenshotPixels } = await import('./screenshot-image-size.ts'); + validateScreenshotPixels(compared.width, compared.height, 'comparison canvas', options.maxPixels); + let scaledCurrent = current; + if (normalization && current.width !== baseline.width) { + const { resizePngBox } = await import('./png-resize.ts'); + scaledCurrent = resizePngBox( + current, + normalization.scaledCurrent.width, + normalization.scaledCurrent.height, + ); } - - const totalPixels = baseline.width * baseline.height; + const alignedBaseline = padScreenshotHeight(baseline, compared.height); + const alignedCurrent = padScreenshotHeight(scaledCurrent, compared.height); + const overlapHeight = Math.min(baseline.height, scaledCurrent.height); + const threshold = options.threshold ?? 0.1; + const totalPixels = compared.width * compared.height; const maxColorDistance = threshold * COLOR_DISTANCE_SCALE; // Per-pixel comparison is CPU-heavy for full-resolution screenshots, so it // runs on the PNG worker thread (with an in-process synchronous fallback). const { diffData, diffMask, differentPixels } = await computeScreenshotDiffPixelsAsync({ - baselineData: baseline.data, - currentData: current.data, - width: baseline.width, - height: baseline.height, + baselineData: alignedBaseline.data, + currentData: alignedCurrent.data, + width: compared.width, + height: compared.height, + overlapHeight, maxColorDistance, }); @@ -136,15 +139,15 @@ export async function compareScreenshots( differentPixels > 0 ? summarizeDiffRegions({ diffMask, - baseline, - current, + baseline: alignedBaseline, + current: alignedCurrent, totalPixels, differentPixels, }) : []; if (differentPixels > 0 && diffOutputPath) { - const diff = new PNG({ width: baseline.width, height: baseline.height }); + const diff = new PNG({ width: compared.width, height: compared.height }); diff.data = diffData; annotateDiffRegions(diff, regions); await fs.mkdir(path.dirname(diffOutputPath), { recursive: true }); @@ -161,6 +164,7 @@ export async function compareScreenshots( return { ...(differentPixels > 0 && diffOutputPath ? { diffPath: diffOutputPath } : {}), ...(regions.length > 0 ? { regions } : {}), + ...(normalization ? { normalization } : {}), totalPixels, differentPixels, mismatchPercentage, @@ -176,19 +180,28 @@ async function validateFileExists(filePath: string, errorMessage: string): Promi } } -function validateMaxPixels( - width: number, - height: number, - label: string, - maxPixels: number | undefined, -): void { - if (maxPixels == null || maxPixels <= 0) return; - const totalPixels = width * height; - if (totalPixels <= maxPixels) return; - throw new AppError( - 'INVALID_ARGS', - `${label} is ${totalPixels} pixels, which exceeds the configured maxImagePixels limit of ${maxPixels}`, - ); +function screenshotDiffNormalization( + baseline: PNG, + current: PNG, +): ScreenshotDiffNormalization | undefined { + if (baseline.width === current.width && baseline.height === current.height) return undefined; + const scaledHeight = Math.max(1, Math.round(current.height * (baseline.width / current.width))); + return { + baseline: { width: baseline.width, height: baseline.height }, + current: { width: current.width, height: current.height }, + scaledCurrent: { width: baseline.width, height: scaledHeight }, + compared: { width: baseline.width, height: Math.max(baseline.height, scaledHeight) }, + currentScale: { x: baseline.width / current.width, y: scaledHeight / current.height }, + alignment: 'top-left', + coordinateSpace: 'comparison-pixels', + }; +} + +function padScreenshotHeight(image: PNG, height: number): PNG { + if (image.height === height) return image; + const padded = new PNG({ width: image.width, height }); + image.data.copy(padded.data); + return padded; } async function removeStaleDiffOutput(outputPath: string | undefined): Promise { diff --git a/packages/capture-kit/src/screenshot-image-size.ts b/packages/capture-kit/src/screenshot-image-size.ts new file mode 100644 index 0000000000..71edb24a75 --- /dev/null +++ b/packages/capture-kit/src/screenshot-image-size.ts @@ -0,0 +1,84 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { hasPngSignature } from './png.ts'; + +/** Checks the decoded allocation budget before either codec allocates pixel rows. */ +export function validateScreenshotImagePixels( + bytes: Buffer, + label: string, + maxPixels?: number, +): void { + if (maxPixels == null || maxPixels <= 0) return; + if (hasPngSignature(bytes)) { + validatePngPixels(bytes, label, maxPixels); + } else if (bytes[0] === 0xff && bytes[1] === 0xd8) { + validateJpegPixels(bytes, label, maxPixels); + } +} + +function validatePngPixels(bytes: Buffer, label: string, maxPixels: number): void { + let offset = 8; + while (offset + 16 <= bytes.length) { + const length = bytes.readUInt32BE(offset); + const type = bytes.toString('ascii', offset + 4, offset + 8); + if (type === 'IHDR' && length >= 8) { + validateScreenshotPixels( + bytes.readUInt32BE(offset + 8), + bytes.readUInt32BE(offset + 12), + label, + maxPixels, + ); + } + if (type === 'IEND') return; + offset += length + 12; + } +} + +function validateJpegPixels(bytes: Buffer, label: string, maxPixels: number): void { + for (const { marker, offset, length } of jpegSegments(bytes)) { + // These are the three frame types supported by the screenshot JPEG decoder. + if ([0xc0, 0xc1, 0xc2].includes(marker) && length >= 8) { + validateScreenshotPixels( + bytes.readUInt16BE(offset + 5), + bytes.readUInt16BE(offset + 3), + label, + maxPixels, + ); + } + } +} + +function* jpegSegments( + bytes: Buffer, +): Generator<{ marker: number; offset: number; length: number }> { + let offset = 2; + while (offset + 3 < bytes.length && bytes[offset] === 0xff) { + while (bytes[offset] === 0xff) offset += 1; + const marker = bytes[offset++]!; + if (marker === 0xda || marker === 0xd9) return; + if (offset + 2 > bytes.length) return; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) return; + yield { marker, offset, length }; + offset += length; + } +} + +/** Bounds original, resized and comparison canvases with the same pixel budget. */ +export function validateScreenshotPixels( + width: number, + height: number, + label: string, + maxPixels?: number, +): void { + const pixels = width * height; + if ( + !Number.isSafeInteger(pixels) || + pixels < 1 || + (maxPixels != null && maxPixels > 0 && pixels > maxPixels) + ) { + throw new AppError( + 'INVALID_ARGS', + `${label} is ${pixels} pixels, which exceeds the configured maxImagePixels limit of ${maxPixels}`, + ); + } +} diff --git a/packages/capture-kit/src/screenshot-image.test.ts b/packages/capture-kit/src/screenshot-image.test.ts index 172f56b363..0f8be1acc8 100644 --- a/packages/capture-kit/src/screenshot-image.test.ts +++ b/packages/capture-kit/src/screenshot-image.test.ts @@ -1,4 +1,5 @@ import { expect, test } from 'vitest'; +import fc from 'fast-check'; import { encode as encodeJpeg } from 'jpeg-js'; import { PNG } from './png.ts'; import { @@ -118,3 +119,68 @@ test('decoding refuses bytes in neither container and a JPEG body that cannot be details: { label: 'fixture', reason: expect.any(String) }, }); }); + +test('an oversized PNG is refused before decoding even an incomplete body', () => { + const png = PNG.sync.write(new PNG({ width: 10, height: 10 })); + expect(thrownBy(() => decodeScreenshotImage(png.subarray(0, 24), 'baseline', 99))).toMatchObject({ + code: 'INVALID_ARGS', + message: expect.stringContaining('maxImagePixels'), + }); +}); + +test('PNG headers cannot replace dimensions with an image above the pixel budget', () => { + const small = PNG.sync.write(new PNG({ width: 1, height: 1 })); + const large = PNG.sync.write(new PNG({ width: 4, height: 4 })); + const repeatedHeader = Buffer.concat([small.subarray(0, 33), large.subarray(8)]); + expect(decodeScreenshotImage(repeatedHeader, 'current', 16).width).toBe(4); + expect(thrownBy(() => decodeScreenshotImage(repeatedHeader, 'current', 1))).toMatchObject({ + code: 'INVALID_ARGS', + message: expect.stringContaining('maxImagePixels'), + }); +}); + +const PROPERTY_RUNS = 40; +test('JPEG frame dimensions enforce the original pixel budget before decoding', () => { + const jpeg = encodeJpeg( + { width: 2, height: 2, data: solidRgba(2, 2, [30, 80, 120, 255]) }, + 90, + ).data; + const frame = jpeg.indexOf(Buffer.from([0xff, 0xc0])); + expect(frame).toBeGreaterThan(0); + const header = Buffer.from(jpeg.subarray(0, frame + 2 + jpeg.readUInt16BE(frame + 2))); + fc.assert( + fc.property( + fc.integer({ min: 2, max: 65535 }), + fc.integer({ min: 2, max: 65535 }), + (width, height) => { + header.writeUInt16BE(height, frame + 5); + header.writeUInt16BE(width, frame + 7); + expect( + thrownBy(() => decodeScreenshotImage(header, 'current', width * height - 1)), + ).toMatchObject({ + code: 'INVALID_ARGS', + message: expect.stringContaining('maxImagePixels'), + }); + }, + ), + { numRuns: PROPERTY_RUNS }, + ); +}); + +test('the JPEG decoder enforces the budget through malformed APP marker recovery', () => { + const jpeg = encodeJpeg( + { width: 2, height: 2, data: solidRgba(2, 2, [30, 80, 120, 255]) }, + 90, + ).data; + const firstSegmentEnd = 4 + jpeg.readUInt16BE(4); + const malformed = Buffer.concat([ + jpeg.subarray(0, firstSegmentEnd), + Buffer.from([0, 0xe1, 0, 2]), + jpeg.subarray(firstSegmentEnd), + ]); + expect(decodeScreenshotImage(malformed, 'current', 4).width).toBe(2); + expect(thrownBy(() => decodeScreenshotImage(malformed, 'current', 3))).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: expect.stringContaining('maxResolutionInMP') }, + }); +}); diff --git a/packages/capture-kit/src/screenshot-image.ts b/packages/capture-kit/src/screenshot-image.ts index 687ac566cb..0a3cf9e3cd 100644 --- a/packages/capture-kit/src/screenshot-image.ts +++ b/packages/capture-kit/src/screenshot-image.ts @@ -1,6 +1,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { decode as decodeJpeg } from 'jpeg-js'; import { decodePng, hasPngSignature, PNG } from './png.ts'; +import { validateScreenshotImagePixels } from './screenshot-image-size.ts'; const JPEG_SIGNATURE = Buffer.from([0xff, 0xd8, 0xff]); @@ -21,10 +22,11 @@ export function detectScreenshotImageFormat(bytes: Buffer): ScreenshotImageForma * Synchronous and CPU-bound: daemon request paths reach it through the PNG worker * (`decodeScreenshotImageAsync` in `png-worker-client.ts`), like every other codec job. */ -export function decodeScreenshotImage(bytes: Buffer, label: string): PNG { +export function decodeScreenshotImage(bytes: Buffer, label: string, maxPixels?: number): PNG { + validateScreenshotImagePixels(bytes, label, maxPixels); const format = detectScreenshotImageFormat(bytes); if (format === 'png') return decodePng(bytes, label); - if (format === 'jpeg') return decodeJpegImage(bytes, label); + if (format === 'jpeg') return decodeJpegImage(bytes, label, maxPixels); throw unsupportedContainer(label, bytes); } @@ -45,9 +47,17 @@ export function transcodeScreenshotToPng(bytes: Buffer, label: string): Buffer { } /** A JPEG signature does not prove a decodable body; a failure keeps the label and the decoder's reason. */ -function decodeJpegImage(bytes: Buffer, label: string): PNG { +function decodeJpegImage(bytes: Buffer, label: string, maxPixels?: number): PNG { try { - const decoded = decodeJpeg(bytes, { useTArray: true, formatAsRGBA: true }); + const decoded = decodeJpeg(bytes, { + useTArray: true, + formatAsRGBA: true, + // The decoder also checks frames reached through its malformed-marker recovery. + // A half pixel keeps floating-point MP conversion away from integer budget boundaries. + ...(maxPixels != null && maxPixels > 0 + ? { maxResolutionInMP: (Math.floor(maxPixels) + 0.5) / 1_000_000 } + : {}), + }); const png = new PNG({ width: decoded.width, height: decoded.height }); png.data = Buffer.from(decoded.data.buffer, decoded.data.byteOffset, decoded.data.byteLength); return png; diff --git a/src/cli/commands/__tests__/screenshot.test.ts b/src/cli/commands/__tests__/screenshot.test.ts index e108e2945c..702f0d27d0 100644 --- a/src/cli/commands/__tests__/screenshot.test.ts +++ b/src/cli/commands/__tests__/screenshot.test.ts @@ -1,11 +1,14 @@ -import { createTestHostEnvironment } from '@agent-device/test-kit'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; import { test } from 'vitest'; import assert from 'node:assert/strict'; import { withCapturedStdoutAsync } from '../../../__tests__/test-utils/stream-capture.ts'; import { createAgentDeviceClient } from '../../../agent-device-client.ts'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { CliFlags } from '@agent-device/contracts/command'; -import { screenshotCommand } from '../screenshot.ts'; +import { screenshotCommand, diffCommand } from '../screenshot.ts'; +import fs from 'node:fs'; +import path from 'node:path'; +import { PNG } from '@agent-device/capture-kit/png'; function clientReturning( data: Record, @@ -96,3 +99,45 @@ test('screenshot --json at the default level still emits normalized screenshot m assert.equal(parsed.data.pixelDensity, 1); assert.ok(!('overlayCount' in parsed.data)); }); + +test.each([false, true])( + 'diff screenshot CLI normalizes saved and live current images (live=%s)', + async (live) => { + const dir = mkdtempForTestSync('cli-diff-width-'); + const baseline = path.join(dir, 'baseline.png'); + const current = path.join(dir, 'current.png'); + const diff = path.join(dir, 'diff.png'); + fs.writeFileSync(baseline, PNG.sync.write(new PNG({ width: 2, height: 2 }))); + fs.writeFileSync(current, PNG.sync.write(new PNG({ width: 4, height: 6 }))); + let captures = 0; + const client = createAgentDeviceClient( + { session: 'qa' }, + { + transport: async (req): Promise => { + assert.equal(req.command, 'screenshot'); + captures += 1; + const out = req.positionals?.[0]; + assert.equal(typeof out, 'string'); + fs.copyFileSync(current, out!); + return { ok: true, data: { path: out } }; + }, + }, + ); + const stdout = await withCapturedStdoutAsync(() => + diffCommand({ + host: createTestHostEnvironment(), + positionals: ['screenshot', ...(live ? [] : [current])], + flags: { json: true, baseline, out: diff, threshold: '1' } as CliFlags, + client, + }), + ); + const { data } = JSON.parse(stdout); + assert.equal(captures, live ? 1 : 0); + assert.equal(data.differentPixels, 2); + assert.equal(data.totalPixels, 6); + assert.equal(data.mismatchPercentage, 33.33); + assert.deepEqual(data.normalization.scaledCurrent, { width: 2, height: 3 }); + const image = PNG.sync.read(fs.readFileSync(diff)); + assert.deepEqual([image.width, image.height], [2, 3]); + }, +); diff --git a/src/commands/capture/diff.ts b/src/commands/capture/diff.ts index 3eee927c46..2a1a118edc 100644 --- a/src/commands/capture/diff.ts +++ b/src/commands/capture/diff.ts @@ -106,7 +106,7 @@ export const diffCommandFacet = defineCommandFacet({ text: { summary: 'Diff snapshot or screenshot', cliDetail: - 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; image dimensions must still match. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', + 'Screenshot --threshold is a per-pixel RGB tolerance: 0 requires exact colors and 1 ignores color differences; the current image scales proportionally to the baseline width and aligns top-left. Scaled height rounds to the nearest pixel (minimum 1). Extra or missing bottom rows always count as differences, even at threshold 1. The originals remain unchanged. When dimensions differ, normalization metadata reports original, scaled and compared sizes; region coordinates use comparison pixels. Both screenshot inputs are decoded from their bytes, so a baseline or current image may be PNG or JPEG, and the diff image is always PNG. JPEG is lossy, so keep --threshold above 0 whenever either input is JPEG. Live iOS simulator screenshot diffs normalize status-bar chrome by default; use screenshot --normalize-status-bar when capturing reusable baselines.', }, metadata: diffCommandMetadata, run: (client, input) => client.capture.diff(input), diff --git a/src/commands/capture/runtime/diff-screenshot.test.ts b/src/commands/capture/runtime/diff-screenshot.test.ts index 82bc00128b..27fd19809a 100644 --- a/src/commands/capture/runtime/diff-screenshot.test.ts +++ b/src/commands/capture/runtime/diff-screenshot.test.ts @@ -113,12 +113,12 @@ test('runtime diff screenshot rejects overlay refs with supplied current image', ); }); -test('runtime diff screenshot enforces max image pixels policy', async () => { +test('runtime diff screenshot enforces the normalized comparison pixel budget', async () => { const dir = mkdtempForTestSync('runtime-diff-screenshot-'); const baseline = path.join(dir, 'baseline.png'); const current = path.join(dir, 'current.png'); - fs.writeFileSync(baseline, solidPngBuffer(10, 10, { r: 0, g: 0, b: 0 })); - fs.writeFileSync(current, solidPngBuffer(10, 10, { r: 255, g: 255, b: 255 })); + fs.writeFileSync(baseline, solidPngBuffer(10, 1, { r: 0, g: 0, b: 0 })); + fs.writeFileSync(current, solidPngBuffer(1, 50, { r: 255, g: 255, b: 255 })); try { const device = createAgentDevice({ @@ -153,7 +153,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a try { const device = createAgentDevice({ backend: createScreenshotBackend((outPath, options) => { - fs.writeFileSync(outPath, solidPngBuffer(10, 10, { r: 255, g: 255, b: 255 })); + fs.writeFileSync(outPath, solidPngBuffer(20, 20, { r: 255, g: 255, b: 255 })); return { path: outPath, ...(options?.overlayRefs @@ -163,7 +163,7 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a ref: 'e1', label: 'Continue', rect: { x: 1, y: 2, width: 3, height: 4 }, - overlayRect: { x: 1, y: 2, width: 3, height: 4 }, + overlayRect: { x: 2, y: 4, width: 6, height: 8 }, center: { x: 3, y: 4 }, }, ], @@ -188,6 +188,16 @@ test('runtime diff screenshot attaches overlay refs to live mismatch regions', a assert.equal(result.currentOverlayRefCount, 1); assert.equal(fs.existsSync(overlayOut), true); assert.equal(result.regions?.[0]?.currentOverlayMatches?.[0]?.ref, 'e1'); + assert.deepEqual(result.regions?.[0]?.currentOverlayMatches?.[0]?.rect, { + x: 1, + y: 2, + width: 3, + height: 4, + }); + assert.equal(result.regions?.[0]?.currentOverlayMatches?.[0]?.regionCoveragePercentage, 12); + const overlay = PNG.sync.read(fs.readFileSync(overlayOut)); + assert.deepEqual([overlay.width, overlay.height], [20, 20]); + assert.deepEqual(result.normalization?.compared, { width: 10, height: 10 }); } finally { fs.rmSync(dir, { recursive: true, force: true }); } diff --git a/src/commands/capture/runtime/diff-screenshot.ts b/src/commands/capture/runtime/diff-screenshot.ts index 7abe0adbcb..656acd361f 100644 --- a/src/commands/capture/runtime/diff-screenshot.ts +++ b/src/commands/capture/runtime/diff-screenshot.ts @@ -156,7 +156,7 @@ async function maybeAttachCurrentOverlay( artifacts: ArtifactDescriptor[], ): Promise { if (!options.overlayRefs) return result; - if (result.match || result.dimensionMismatch) { + if (result.match) { if (diffOutputPath) await removeStaleCurrentOverlay(diffOutputPath); return result; } @@ -184,7 +184,11 @@ async function maybeAttachCurrentOverlay( : {}), ...(result.regions && overlayResult.overlayRefs ? { - regions: attachCurrentOverlayMatches(result.regions, overlayResult.overlayRefs), + regions: attachCurrentOverlayMatches( + result.regions, + overlayResult.overlayRefs, + result.normalization?.currentScale, + ), } : {}), }; diff --git a/src/commands/output/diff.test.ts b/src/commands/output/diff.test.ts index e2097fbc1a..0f6d622173 100644 --- a/src/commands/output/diff.test.ts +++ b/src/commands/output/diff.test.ts @@ -130,26 +130,30 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color' assert.equal(text.includes('\u001b['), false); }); -test('formatScreenshotDiffText renders dimension mismatch', () => { +test('formatScreenshotDiffText explains normalized comparison geometry', () => { const text = withNoColor(() => formatScreenshotDiffText( { match: false, - differentPixels: 100, - totalPixels: 100, - mismatchPercentage: 100, - dimensionMismatch: { - expected: { width: 1170, height: 2532 }, - actual: { width: 1080, height: 1920 }, + differentPixels: 375, + totalPixels: 304875, + mismatchPercentage: 0.12, + normalization: { + baseline: { width: 375, height: 812 }, + current: { width: 1080, height: 2340 }, + scaledCurrent: { width: 375, height: 813 }, + compared: { width: 375, height: 813 }, + currentScale: { x: 375 / 1080, y: 813 / 2340 }, + alignment: 'top-left', + coordinateSpace: 'comparison-pixels', }, }, { env: process.env, cwd: process.cwd() }, ), ); - assert.match(text, /✗ Screenshots have different dimensions/); - assert.match(text, /expected 1170x2532/); - assert.match(text, /got 1080x1920/); - assert.equal(text.includes('different /'), false); + assert.match(text, /current 1080x2340 → 375x813/); + assert.match(text, /Compared 375x813, aligned top-left; regions use comparison pixels/); + assert.match(text, /375 different \/ 304875 total pixels/); }); test('formatScreenshotDiffText renders diff path relative to cwd', () => { diff --git a/src/commands/output/diff.ts b/src/commands/output/diff.ts index 4553aa49af..ac79748cab 100644 --- a/src/commands/output/diff.ts +++ b/src/commands/output/diff.ts @@ -53,13 +53,21 @@ export function formatScreenshotDiffText( ): string { const useColor = supportsColor(host.env); const match = data.match === true; - const dimensionMismatch = data.dimensionMismatch; const lines: string[] = []; lines.push(...formatScreenshotDiffStatusLines(data, useColor)); + if (data.normalization) { + const { baseline, current, scaledCurrent, compared } = data.normalization; + lines.push( + ` Baseline ${baseline.width}x${baseline.height}; current ${current.width}x${current.height} → ${scaledCurrent.width}x${scaledCurrent.height}`, + ); + lines.push( + ` Compared ${compared.width}x${compared.height}, aligned top-left; regions use comparison pixels`, + ); + } lines.push(...formatScreenshotDiffArtifactLines(data, match, useColor, host.cwd)); - if (!match && !dimensionMismatch) { + if (!match) { lines.push(...formatScreenshotDiffPixelCountLines(data, useColor)); lines.push(...formatScreenshotDiffRegionLines(data, useColor)); } @@ -72,18 +80,7 @@ function formatScreenshotDiffStatusLines(data: ScreenshotDiffResult, useColor: b const indicator = useColor ? colorize('✓', 'green') : '✓'; return [`${indicator} Screenshots match.`]; } - const dimensionMismatch = data.dimensionMismatch; const indicator = useColor ? colorize('✗', 'red') : '✗'; - if (dimensionMismatch) { - const expected = dimensionMismatch.expected; - const actual = dimensionMismatch.actual; - return [ - `${indicator} Screenshots have different dimensions: ` + - `expected ${expected?.width}x${expected?.height}, ` + - `got ${actual?.width}x${actual?.height}`, - ]; - } - const differentPixels = toNumber(data.differentPixels); const mismatchPercentage = toNumber(data.mismatchPercentage); const pctLabel = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6fdba77b65..4a1ce180d1 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -1007,7 +1007,9 @@ agent-device record stop # Stop active recording **Screenshot diffs** - `diff screenshot` compares a live capture, or a saved `current.png` (no session needed), against `--baseline`. Inputs may be PNG or JPEG regardless of extension; HarmonyOS captures are JPEG. Output ranks changed regions with rectangles, changed-pixel counts, and share of the diff; `--out` writes a diff PNG with changed pixels tinted red. `ocr` and `nonTextDeltas` are never populated, so read the images yourself for qualitative review. -- `--threshold <0-1>` sets the per-pixel RGB tolerance (default `0.1`). Image dimensions must match. Keep it above `0` when either input is JPEG. +- `--threshold <0-1>` sets the per-pixel RGB tolerance (default `0.1`). Keep it above `0` when either input is JPEG. +- The current image scales proportionally to the baseline width and aligns at the top left. Its scaled height rounds to the nearest pixel, with a minimum of one row. The comparison includes the full height of both images: extra or missing bottom rows count as differences, including blank or transparent rows, even at `--threshold 1`. Your original files stay unchanged. This does not crop system bars or move content into alignment. +- When dimensions differ, JSON `normalization` reports `baseline`, `current`, `scaledCurrent`, and `compared` dimensions. `currentScale.x` and `.y` map original current pixels to comparison pixels; the height scale includes pixel rounding. Region rectangles and ref-match rectangles use comparison pixels, and normalized region rectangles use percentages of the compared canvas. The separate current-screen overlay keeps its original capture size. - `--overlay-refs` also writes a separate current-screen ref overlay and lists refs under changed regions. It needs a live capture, so it is unavailable with a saved `current.png`. - Live iOS simulator diffs normalize status-bar chrome by default.