diff --git a/README.md b/README.md index 1994c22..43d759b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Built with **FastMCP** leveraging the latest MCP spec (2025-11-25) features. ## ✨ What It Does -``` +```text YOU: "Watch the modal animation on this page" ↓ ANIMAWATCH: @@ -41,6 +41,12 @@ RESULT: "Jank detected at 1.2s - fade-in stutters for 180ms" | `analyze_video` | πŸŽ₯ Analyze an existing video file | | `record` | ⏺️ Just record without analysis | | `check_accessibility` | β™Ώ Visual accessibility analysis | +| `list_devices` | πŸ“± List available device profiles for emulation | +| `watch_with_device` | πŸ“² Watch with mobile/tablet device emulation | +| `compare_screenshots` | πŸ” Visual diff comparison between two URLs | +| `analyze_fps` | 🎯 FPS consistency and jank detection | +| `get_performance_metrics` | πŸ“Š Core Web Vitals (LCP, FCP, CLS, TTFB) | +| `analyze_with_consensus_tool` | 🀝 Multi-model consensus analysis | ### Resources @@ -84,6 +90,7 @@ cp .env.example .env ### 4. Add to Your MCP Client **Claude Desktop** (`~/Library/Application Support/Claude/claude_desktop_config.json`): + ```json { "mcpServers": { @@ -99,6 +106,7 @@ cp .env.example .env ``` **Augment Code** (settings): + ```json { "mcpServers": { @@ -116,30 +124,71 @@ cp .env.example .env ## πŸ“– Usage Examples ### Watch Animation Issues -``` + +```text "Watch the modal animation on https://example.com for any jank" ``` ### Perform Actions Then Watch -``` + +```text "Click the hamburger menu on https://example.com and watch the slide-in animation" ``` ### Focus on Specific Area -``` + +```text "Watch https://example.com with focus on scroll behavior" ``` ### Accessibility Check -``` + +```text "Check accessibility on https://example.com" ``` ### Access Previous Results -``` + +```text "Show me the analysis from animawatch://analyses/abc123" ``` +### Test on Mobile Device + +```text +"Watch https://example.com on an iPhone 15 Pro and check for animation issues" +``` + +### Compare Before/After + +```text +"Compare screenshots of https://staging.example.com and https://example.com for visual differences" +``` + +### Check Performance Metrics + +```text +"Get Core Web Vitals for https://example.com" +``` + +### Analyze FPS + +```text +"Analyze the FPS of this video recording for frame drops" +``` + +### Multi-Model Consensus + +```text +"Analyze https://example.com using both Gemini and Ollama for higher accuracy" +``` + +### List Available Devices + +```text +"What mobile devices can I test with?" +``` + ## πŸ”§ Configuration | Environment Variable | Default | Description | @@ -180,25 +229,31 @@ export OLLAMA_MODEL=qwen2.5-vl:7b ## πŸ—οΈ Architecture -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ AnimaWatch β”‚ -β”‚ (FastMCP Server) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Lifespan Context (AppContext) β”‚ -β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ -β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ -β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ -β”‚ └── analyses: dict[id, str] β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Tools β”‚ Resources β”‚ Prompts β”‚ -β”‚ ───────── β”‚ ───────────────────── β”‚ ────────────── β”‚ -β”‚ watch β”‚ animawatch://recordingsβ”‚ animation_diagnosisβ”‚ -β”‚ screenshot β”‚ animawatch://analyses β”‚ page_analysis β”‚ -β”‚ record β”‚ animawatch://config β”‚ accessibility_checkβ”‚ -β”‚ analyze_videoβ”‚ β”‚ β”‚ -β”‚ check_accessβ”‚ β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +```text +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ AnimaWatch β”‚ +β”‚ (FastMCP Server) β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Lifespan Context (AppContext) β”‚ +β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ +β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ +β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ +β”‚ └── analyses: dict[id, str] β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Core Tools β”‚ Device & Performance β”‚ Comparison & Accuracy β”‚ +β”‚ ────────────────── β”‚ ──────────────────── β”‚ ──────────────────── β”‚ +β”‚ watch β”‚ list_devices β”‚ compare_screenshots β”‚ +β”‚ screenshot β”‚ watch_with_device β”‚ analyze_with_consensus β”‚ +β”‚ record β”‚ analyze_fps β”‚ β”‚ +β”‚ analyze_video β”‚ get_performance_metricsβ”‚ β”‚ +β”‚ check_accessibility β”‚ β”‚ β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Resources β”‚ Prompts β”‚ +β”‚ ─────────────────────────── β”‚ ─────────────────────────────────────── β”‚ +β”‚ animawatch://recordings/{id} β”‚ animation_diagnosis β”‚ +β”‚ animawatch://analyses/{id} β”‚ page_analysis β”‚ +β”‚ animawatch://config β”‚ accessibility_check β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` ## πŸ“„ License diff --git a/pyproject.toml b/pyproject.toml index a244e9c..913b59e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,14 +8,37 @@ requires-python = ">=3.11" authors = [ { name = "Rishi Tank", email = "3771081+rishitank@users.noreply.github.com" } ] -keywords = ["mcp", "animation", "visual-testing", "playwright", "gemini", "ai-vision", "ui-testing", "fastmcp"] +keywords = [ + "mcp", + "model-context-protocol", + "animation", + "visual-testing", + "playwright", + "gemini", + "ai-vision", + "ui-testing", + "fastmcp", + "web-vitals", + "performance", + "accessibility", + "browser-automation", + "claude", + "llm", +] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Internet :: WWW/HTTP :: Browsers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Typing :: Typed", ] dependencies = [ diff --git a/src/animawatch/diff.py b/src/animawatch/diff.py index 3fcd135..e0cd6ef 100644 --- a/src/animawatch/diff.py +++ b/src/animawatch/diff.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path -from PIL import Image, ImageChops, ImageDraw +from PIL import Image, ImageChops @dataclass @@ -74,7 +74,7 @@ def compare_images( diff_sum = 0 # Get pixel data as a flat list for grayscale image - pixel_data = list(diff_gray.getdata()) + pixel_data = diff_gray.tobytes() for pixel_value in pixel_data: # For grayscale ("L" mode), pixels are single int values pixel_diff = int(pixel_value) @@ -129,7 +129,7 @@ def _find_diff_regions( # Calculate region difference score region = diff_gray.crop(bbox) - pixels = list(region.getdata()) + pixels = region.tobytes() avg_diff = sum(pixels) / len(pixels) if pixels else 0 difference_score = (avg_diff / 255) * 100 @@ -152,22 +152,16 @@ def _generate_diff_image( highlight_color: tuple[int, int, int, int], ) -> Path: """Generate a diff image with highlighted changes.""" - # Create a copy of the "after" image - result = after.copy() - - # Create an overlay for highlighting - overlay = Image.new("RGBA", result.size, (0, 0, 0, 0)) - draw = ImageDraw.Draw(overlay) - - # Highlight differing pixels using getpixel for type safety - for x in range(diff_gray.width): - for y in range(diff_gray.height): - pixel_value = diff_gray.getpixel((x, y)) - if isinstance(pixel_value, int) and pixel_value > threshold: - draw.point((x, y), fill=highlight_color) - - # Composite the overlay onto the result - result = Image.alpha_composite(result.convert("RGBA"), overlay) + # Mask of differing pixels: 255 where the difference exceeds the threshold. + # Image.point runs in C, unlike a per-pixel getpixel loop in Python. + mask = diff_gray.point(lambda p: 255 if p > threshold else 0) + + # Transparent overlay with the highlight colour painted only under the mask + overlay = Image.new("RGBA", after.size, (0, 0, 0, 0)) + overlay.paste(highlight_color, (0, 0, *after.size), mask) + + # Composite the overlay onto the "after" image + result = Image.alpha_composite(after.convert("RGBA"), overlay) # Save to temp file fd, tmp_path = tempfile.mkstemp(suffix="_diff.png") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..76105d7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,60 @@ +"""Shared pytest fixtures for AnimaWatch tests.""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from animawatch.server import AppContext + + +@pytest.fixture +def mock_page() -> MagicMock: + """A Playwright ``Page`` stand-in with awaitable navigation and evaluation.""" + page = MagicMock() + page.goto = AsyncMock() + page.evaluate = AsyncMock(return_value={}) + return page + + +@pytest.fixture +def mock_app_context(mock_page: MagicMock, tmp_path: Path) -> AppContext: + """An AppContext whose browser and vision provider are mocks. + + ``browser.pooled_context`` behaves like the real async context manager: calling + it returns an object usable in ``async with`` that yields ``(context, page)``. + It is wrapped in a ``MagicMock`` so tests can assert how it was called. + """ + mock_browser = AsyncMock() + # Paths live under tmp_path so tools that unlink them can't touch real files. + mock_browser.record_interaction = AsyncMock(return_value=tmp_path / "video.webm") + mock_browser.take_screenshot = AsyncMock(return_value=tmp_path / "screenshot.png") + + @asynccontextmanager + async def pooled_context(*_args: Any, **_kwargs: Any) -> AsyncIterator[tuple[Any, Any]]: + """Yield a ``(context, page)`` pair like ``BrowserRecorder.pooled_context``.""" + yield (MagicMock(), mock_page) + + mock_browser.pooled_context = MagicMock(side_effect=pooled_context) + + mock_vision = AsyncMock() + mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") + mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") + + return AppContext( + browser=mock_browser, + vision=mock_vision, + recordings={}, + analyses={}, + ) + + +@pytest.fixture +def mock_ctx(mock_app_context: AppContext) -> MagicMock: + """A tool ``Context`` whose lifespan context is ``mock_app_context``.""" + ctx = MagicMock() + ctx.request_context.lifespan_context = mock_app_context + return ctx diff --git a/tests/test_diff.py b/tests/test_diff.py index cb415a2..44a51dd 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from PIL import Image +from PIL import Image, ImageChops from animawatch.diff import ( DiffRegion, @@ -113,6 +113,46 @@ def test_diff_image_generation(self, different_images: tuple[Path, Path]) -> Non assert result.diff_image_path.exists() result.diff_image_path.unlink(missing_ok=True) + @pytest.mark.parametrize( + "highlight_color", [(0, 0, 255, 255), (255, 0, 0, 128)], ids=["opaque", "translucent"] + ) + def test_diff_image_highlights_exactly_pixels_above_threshold( + self, tmp_path: Path, highlight_color: tuple[int, int, int, int] + ) -> None: + """The diff image is the "after" image with only above-threshold pixels overlaid.""" + threshold = 5 + before = Image.new("RGB", (50, 50), "white") + after = before.copy() + after.paste((255, 0, 0), (10, 10, 20, 20)) # strong difference + after.paste((250, 250, 250), (30, 30, 40, 40)) # grey diff of exactly `threshold` + before_path, after_path = tmp_path / "before.png", tmp_path / "after.png" + before.save(before_path) + after.save(after_path) + + result = compare_images( + before_path, after_path, threshold=threshold, highlight_color=highlight_color + ) + assert result.diff_image_path is not None + try: + actual = Image.open(result.diff_image_path).convert("RGBA") + + # Reference: overlay the highlight pixel by pixel wherever the grayscale + # difference is strictly greater than the threshold. + diff_gray = ImageChops.difference(before, after).convert("L") + overlay = Image.new("RGBA", after.size, (0, 0, 0, 0)) + for x in range(after.width): + for y in range(after.height): + value = diff_gray.getpixel((x, y)) + if isinstance(value, int) and value > threshold: + overlay.putpixel((x, y), highlight_color) + expected = Image.alpha_composite(after.convert("RGBA"), overlay) + + assert actual.tobytes() == expected.tobytes() + assert actual.getpixel((35, 35)) == (250, 250, 250, 255) # at threshold: untouched + assert actual.getpixel((0, 0)) == (255, 255, 255, 255) # unchanged: untouched + finally: + result.diff_image_path.unlink(missing_ok=True) + def test_result_paths(self, identical_images: tuple[Path, Path]) -> None: """Test that result contains correct paths.""" before, after = identical_images diff --git a/tests/test_server.py b/tests/test_server.py index 5e96343..47a1dc6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,7 +5,12 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from PIL import Image +from animawatch.consensus import ConsensusResult +from animawatch.fps import FPSAnalysisResult, JankEvent +from animawatch.metrics import CoreWebVitals, PerformanceMetrics +from animawatch.models import Finding, IssueCategory, Severity from animawatch.server import ( ANIMATION_PROMPT, AppContext, @@ -169,32 +174,7 @@ def test_get_config_returns_settings(self) -> None: class TestTools: - """Tests for MCP tools.""" - - @pytest.fixture - def mock_app_context(self) -> AppContext: - """Create a mock AppContext for testing tools.""" - mock_browser = AsyncMock() - mock_browser.record_interaction = AsyncMock(return_value=Path("/tmp/video.webm")) - mock_browser.take_screenshot = AsyncMock(return_value=Path("/tmp/screenshot.png")) - - mock_vision = AsyncMock() - mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") - mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") - - return AppContext( - browser=mock_browser, - vision=mock_vision, - recordings={}, - analyses={}, - ) - - @pytest.fixture - def mock_ctx(self, mock_app_context: AppContext) -> MagicMock: - """Create a mock Context for testing tools.""" - mock_ctx = MagicMock() - mock_ctx.request_context.lifespan_context = mock_app_context - return mock_ctx + """Tests for MCP tools (fixtures ``mock_app_context``/``mock_ctx`` live in conftest).""" @pytest.mark.asyncio async def test_watch_records_and_analyzes( @@ -303,3 +283,321 @@ async def test_check_accessibility_tool( assert "Accessibility Analysis" in result cast(AsyncMock, mock_app_context.browser.take_screenshot).assert_called_once() cast(AsyncMock, mock_app_context.vision.analyze_image).assert_called_once() + + +class TestNewTools: + """Tests for new MCP tools (devices, diff, fps, metrics, consensus).""" + + @pytest.mark.asyncio + async def test_list_devices_all(self) -> None: + """Test list_devices returns all devices.""" + from animawatch.server import list_devices + + result = await list_devices() + + assert "Available Device Profiles" in result + assert "iphone" in result.lower() or "pixel" in result.lower() + + @pytest.mark.asyncio + async def test_list_devices_by_category(self) -> None: + """Test list_devices filters by category.""" + from animawatch.server import list_devices + + result = await list_devices(category="mobile") + + assert "Available Device Profiles" in result + assert "**Category**: mobile" in result + # Mobile profiles are listed; tablet and desktop profiles are filtered out. + assert "**iphone_15_pro**" in result + assert "**pixel_8**" in result + assert "**ipad_pro_12**" not in result + assert "**desktop_1080p**" not in result + + @pytest.mark.asyncio + async def test_list_devices_invalid_category(self) -> None: + """Test list_devices with invalid category.""" + from animawatch.server import list_devices + + result = await list_devices(category="invalid") + + assert "Invalid category" in result + + @pytest.mark.asyncio + async def test_watch_with_device_valid( + self, mock_ctx: MagicMock, mock_app_context: AppContext + ) -> None: + """Test watch_with_device with valid device.""" + from animawatch.server import watch_with_device + + with patch("animawatch.server.uuid.uuid4") as mock_uuid: + mock_uuid.return_value = MagicMock() + mock_uuid.return_value.__str__ = lambda self: "abc12345-6789" + + result = await watch_with_device( + url="https://example.com", + device="iphone_15_pro", + ctx=mock_ctx, + ) + + assert "Device Animation Analysis" in result + assert "iPhone 15 Pro" in result + # The selected device is forwarded to the recorder. + record = cast(AsyncMock, mock_app_context.browser.record_interaction) + record.assert_called_once() + assert record.call_args.kwargs["device"] == "iphone_15_pro" + assert record.call_args.kwargs["url"] == "https://example.com" + + @pytest.mark.asyncio + async def test_watch_with_device_invalid( + self, mock_ctx: MagicMock, mock_app_context: AppContext + ) -> None: + """Test watch_with_device with invalid device.""" + from animawatch.server import watch_with_device + + result = await watch_with_device( + url="https://example.com", + device="invalid_device", + ctx=mock_ctx, + ) + + assert "not found" in result + assert "Available devices" in result + + @pytest.mark.asyncio + async def test_watch_with_device_without_context(self) -> None: + """Test watch_with_device raises without context.""" + from animawatch.server import watch_with_device + + with pytest.raises(RuntimeError, match="Context is required"): + await watch_with_device( + url="https://example.com", + device="iphone_15_pro", + ctx=None, + ) + + @pytest.mark.asyncio + async def test_analyze_fps_file_not_found(self) -> None: + """Test analyze_fps with non-existent file.""" + from animawatch.server import analyze_fps + + result = await analyze_fps(video_path="/nonexistent/video.mp4") + + assert "Video not found" in result + + @pytest.mark.asyncio + async def test_get_performance_metrics_without_context(self) -> None: + """Test get_performance_metrics raises without context.""" + from animawatch.server import get_performance_metrics + + with pytest.raises(RuntimeError, match="Context is required"): + await get_performance_metrics(url="https://example.com", ctx=None) + + @pytest.mark.asyncio + async def test_analyze_with_consensus_without_context(self) -> None: + """Test analyze_with_consensus_tool raises without context.""" + from animawatch.server import analyze_with_consensus_tool + + with pytest.raises(RuntimeError, match="Context is required"): + await analyze_with_consensus_tool(url="https://example.com", ctx=None) + + @pytest.mark.asyncio + async def test_compare_screenshots_without_context(self) -> None: + """Test compare_screenshots raises without context.""" + from animawatch.server import compare_screenshots + + with pytest.raises(RuntimeError, match="Context is required"): + await compare_screenshots( + url1="https://example.com", + url2="https://example.org", + ctx=None, + ) + + # ------------------------------------------------------------------ + # Happy paths for the new tools + # ------------------------------------------------------------------ + + @pytest.mark.asyncio + async def test_analyze_fps_reports_results(self, tmp_path: Path) -> None: + """analyze_fps passes its arguments through and renders the FPS report.""" + from animawatch.server import analyze_fps + + video = tmp_path / "recording.webm" + video.write_bytes(b"fake video") + fps_result = FPSAnalysisResult( + average_fps=58.5, + target_fps=30.0, + min_fps=40.0, + max_fps=60.0, + total_frames=117, + jank_events=[ + JankEvent( + frame_number=42, + timestamp_ms=700.0, + expected_delta_ms=33.3, + actual_delta_ms=50.0, + severity="minor", + dropped_frames=1, + ) + ], + jank_percentage=0.85, + frame_time_consistency=92.0, + duration_ms=2000.0, + ) + + with patch( + "animawatch.server.analyze_video_fps", AsyncMock(return_value=fps_result) + ) as mock_analyze: + result = await analyze_fps( + video_path=str(video), target_fps=30.0, jank_threshold_ms=8.0 + ) + + mock_analyze.assert_awaited_once_with(video, 30.0, 8.0) + assert result.startswith("## 🎯 FPS Analysis") + assert "**Average FPS**: 58.5 (target: 30.0)" in result + assert "**Jank Events**: 1" in result + assert "Frame 42 @ 700ms" in result + + @pytest.mark.asyncio + async def test_get_performance_metrics_reports_and_stores( + self, mock_ctx: MagicMock, mock_app_context: AppContext, mock_page: MagicMock + ) -> None: + """get_performance_metrics navigates a pooled page and stores the report.""" + from animawatch.server import get_performance_metrics + + metrics = PerformanceMetrics( + url="https://example.com", + core_web_vitals=CoreWebVitals(lcp_ms=1800.0, fcp_ms=900.0, cls_score=0.02), + load_time_ms=2100.0, + dom_content_loaded_ms=1200.0, + resource_count=12, + total_transfer_size_kb=340.0, + ) + + with patch( + "animawatch.server.extract_performance_metrics", AsyncMock(return_value=metrics) + ) as mock_extract: + result = await get_performance_metrics(url="https://example.com", ctx=mock_ctx) + + cast(MagicMock, mock_app_context.browser.pooled_context).assert_called_once_with() + mock_page.goto.assert_awaited_once_with("https://example.com", wait_until="networkidle") + mock_extract.assert_awaited_once_with(mock_page, "https://example.com") + assert result.startswith("**Analysis ID**: `") + assert "Performance Metrics Report" in result + assert "LCP" in result + assert len(mock_app_context.analyses) == 1 + (stored,) = mock_app_context.analyses.values() + assert result.endswith(stored) + + @pytest.mark.asyncio + async def test_analyze_with_consensus_reports_findings( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """analyze_with_consensus_tool renders agreed and single-model findings.""" + from animawatch.server import analyze_with_consensus_tool + + screenshot = tmp_path / "consensus.png" + screenshot.write_bytes(b"fake png") + mock_app_context.browser.take_screenshot = AsyncMock(return_value=screenshot) + + agreed = Finding( + id="f1", + category=IssueCategory.ANIMATION, + severity=Severity.MAJOR, + confidence=90, + element="hero banner", + description="Banner stutters while sliding in", + suggestion="Animate transform instead of left", + ) + gemini_only = Finding( + id="f2", + category=IssueCategory.LAYOUT, + severity=Severity.MINOR, + confidence=60, + element="footer", + description="Footer shifts after fonts load", + suggestion="Reserve space for the footer", + ) + consensus = ConsensusResult( + merged_findings=[agreed, gemini_only], + gemini_only=[gemini_only], + ollama_only=[], + agreed_findings=[agreed], + gemini_result=None, + ollama_result=None, + consensus_score=50.0, + ) + + with patch( + "animawatch.server.run_consensus_analysis", AsyncMock(return_value=consensus) + ) as mock_consensus: + result = await analyze_with_consensus_tool( + url="https://example.com", focus="timing", ctx=mock_ctx + ) + + mock_consensus.assert_awaited_once() + assert mock_consensus.await_args is not None + assert mock_consensus.await_args.args[0] == screenshot + assert "**Consensus Score**: 50%" in result + assert "**major** [animation]: Banner stutters while sliding in" in result + assert "Animate transform instead of left" in result + assert "Gemini-Only Findings" in result + assert "Ollama-Only Findings" not in result + assert not screenshot.exists() # temporary screenshot is cleaned up + assert list(mock_app_context.analyses.values()) == [result] + + @staticmethod + def _write_png(path: Path, square: bool) -> Path: + """Write a 40x40 white PNG, optionally with a 20x20 black square at (10, 10).""" + image = Image.new("RGB", (40, 40), "white") + if square: + image.paste((0, 0, 0), (10, 10, 30, 30)) + image.save(path) + return path + + @pytest.mark.asyncio + async def test_compare_screenshots_detects_differences( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """compare_screenshots reports the differing region and cleans up screenshots.""" + from animawatch.server import compare_screenshots + + before = self._write_png(tmp_path / "before.png", square=False) + after = self._write_png(tmp_path / "after.png", square=True) + mock_app_context.browser.take_screenshot = AsyncMock(side_effect=[before, after]) + + result = await compare_screenshots( + url1="https://example.com", url2="https://example.org", ctx=mock_ctx + ) + + # Locate the generated diff image first so it is removed even if an assertion fails. + marker = "Diff image saved: `" + diff_image = Path(result.rsplit(marker, 1)[1].rstrip("`")) if marker in result else None + try: + assert "Differences detected!" in result + assert "**Diff Percentage**: 25.00%" in result # 400 of 1600 pixels + assert "**Region 1**: (10, 10) 20x20" in result + assert not before.exists() + assert not after.exists() + assert diff_image is not None + assert diff_image.exists() + finally: + if diff_image is not None: + diff_image.unlink(missing_ok=True) + + @pytest.mark.asyncio + async def test_compare_screenshots_identical( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """compare_screenshots reports no differences for identical pages.""" + from animawatch.server import compare_screenshots + + before = self._write_png(tmp_path / "before.png", square=True) + after = self._write_png(tmp_path / "after.png", square=True) + mock_app_context.browser.take_screenshot = AsyncMock(side_effect=[before, after]) + + result = await compare_screenshots( + url1="https://example.com", url2="https://example.com", ctx=mock_ctx + ) + + assert "No visual differences detected!" in result + assert "Similarity: 100.0%" in result