From 3eecdc58490601a1a8dc62d0a299e0ba4ff16b75 Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Sat, 7 Feb 2026 02:36:27 +0000 Subject: [PATCH 1/7] feat: improvements for PyPI publishing and testing - Fix Pillow deprecation warnings (getdata -> tobytes) - Update README with new tool examples and architecture diagram - Add 10 integration tests for new MCP tools - Enhance pyproject.toml with more classifiers and keywords - Update development status to Beta --- README.md | 78 ++++++++++++++----- pyproject.toml | 27 ++++++- src/animawatch/diff.py | 4 +- tests/test_server.py | 166 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 253 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1994c22..3b74754 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ RESULT: "Jank detected at 1.2s - fade-in stutters for 180ms" | `analyze_video` | πŸŽ₯ Analyze an existing video file | | `record` | ⏺️ Just record without analysis | | `check_accessibility` | β™Ώ Visual accessibility analysis | +| `list_devices` | πŸ“± List available device profiles for emulation | +| `watch_with_device` | πŸ“² Watch with mobile/tablet device emulation | +| `compare_screenshots` | πŸ” Visual diff comparison between two URLs | +| `analyze_fps` | 🎯 FPS consistency and jank detection | +| `get_performance_metrics` | πŸ“Š Core Web Vitals (LCP, FCP, CLS, TTFB) | +| `analyze_with_consensus_tool` | 🀝 Multi-model consensus analysis | ### Resources @@ -140,6 +146,36 @@ cp .env.example .env "Show me the analysis from animawatch://analyses/abc123" ``` +### Test on Mobile Device +``` +"Watch https://example.com on an iPhone 15 Pro and check for animation issues" +``` + +### Compare Before/After +``` +"Compare screenshots of https://staging.example.com and https://example.com for visual differences" +``` + +### Check Performance Metrics +``` +"Get Core Web Vitals for https://example.com" +``` + +### Analyze FPS +``` +"Analyze the FPS of this video recording for frame drops" +``` + +### Multi-Model Consensus +``` +"Analyze https://example.com using both Gemini and Ollama for higher accuracy" +``` + +### List Available Devices +``` +"What mobile devices can I test with?" +``` + ## πŸ”§ Configuration | Environment Variable | Default | Description | @@ -181,24 +217,30 @@ export OLLAMA_MODEL=qwen2.5-vl:7b ## πŸ—οΈ Architecture ``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ AnimaWatch β”‚ -β”‚ (FastMCP Server) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Lifespan Context (AppContext) β”‚ -β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ -β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ -β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ -β”‚ └── analyses: dict[id, str] β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Tools β”‚ Resources β”‚ Prompts β”‚ -β”‚ ───────── β”‚ ───────────────────── β”‚ ────────────── β”‚ -β”‚ watch β”‚ animawatch://recordingsβ”‚ animation_diagnosisβ”‚ -β”‚ screenshot β”‚ animawatch://analyses β”‚ page_analysis β”‚ -β”‚ record β”‚ animawatch://config β”‚ accessibility_checkβ”‚ -β”‚ analyze_videoβ”‚ β”‚ β”‚ -β”‚ check_accessβ”‚ β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ AnimaWatch β”‚ +β”‚ (FastMCP Server) β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Lifespan Context (AppContext) β”‚ +β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ +β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ +β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ +β”‚ └── analyses: dict[id, str] β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Core Tools β”‚ Device & Performance β”‚ Comparison & Accuracy β”‚ +β”‚ ────────────────── β”‚ ──────────────────── β”‚ ──────────────────── β”‚ +β”‚ watch β”‚ list_devices β”‚ compare_screenshots β”‚ +β”‚ screenshot β”‚ watch_with_device β”‚ analyze_with_consensus β”‚ +β”‚ record β”‚ analyze_fps β”‚ β”‚ +β”‚ analyze_video β”‚ get_performance_metricsβ”‚ β”‚ +β”‚ check_accessibility β”‚ β”‚ β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Resources β”‚ Prompts β”‚ +β”‚ ─────────────────────────── β”‚ ─────────────────────────────────────── β”‚ +β”‚ animawatch://recordings/{id} β”‚ animation_diagnosis β”‚ +β”‚ animawatch://analyses/{id} β”‚ page_analysis β”‚ +β”‚ animawatch://config β”‚ accessibility_check β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` ## πŸ“„ License diff --git a/pyproject.toml b/pyproject.toml index a244e9c..913b59e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,14 +8,37 @@ requires-python = ">=3.11" authors = [ { name = "Rishi Tank", email = "3771081+rishitank@users.noreply.github.com" } ] -keywords = ["mcp", "animation", "visual-testing", "playwright", "gemini", "ai-vision", "ui-testing", "fastmcp"] +keywords = [ + "mcp", + "model-context-protocol", + "animation", + "visual-testing", + "playwright", + "gemini", + "ai-vision", + "ui-testing", + "fastmcp", + "web-vitals", + "performance", + "accessibility", + "browser-automation", + "claude", + "llm", +] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Internet :: WWW/HTTP :: Browsers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Typing :: Typed", ] dependencies = [ diff --git a/src/animawatch/diff.py b/src/animawatch/diff.py index 3fcd135..1fa2f7e 100644 --- a/src/animawatch/diff.py +++ b/src/animawatch/diff.py @@ -74,7 +74,7 @@ def compare_images( diff_sum = 0 # Get pixel data as a flat list for grayscale image - pixel_data = list(diff_gray.getdata()) + pixel_data = diff_gray.tobytes() for pixel_value in pixel_data: # For grayscale ("L" mode), pixels are single int values pixel_diff = int(pixel_value) @@ -129,7 +129,7 @@ def _find_diff_regions( # Calculate region difference score region = diff_gray.crop(bbox) - pixels = list(region.getdata()) + pixels = region.tobytes() avg_diff = sum(pixels) / len(pixels) if pixels else 0 difference_score = (avg_diff / 255) * 100 diff --git a/tests/test_server.py b/tests/test_server.py index 5e96343..fe52668 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -303,3 +303,169 @@ async def test_check_accessibility_tool( assert "Accessibility Analysis" in result cast(AsyncMock, mock_app_context.browser.take_screenshot).assert_called_once() cast(AsyncMock, mock_app_context.vision.analyze_image).assert_called_once() + + +class TestNewTools: + """Tests for new MCP tools (devices, diff, fps, metrics, consensus).""" + + @pytest.mark.asyncio + async def test_list_devices_all(self) -> None: + """Test list_devices returns all devices.""" + from animawatch.server import list_devices + + result = await list_devices() + + assert "Available Device Profiles" in result + assert "iphone" in result.lower() or "pixel" in result.lower() + + @pytest.mark.asyncio + async def test_list_devices_by_category(self) -> None: + """Test list_devices filters by category.""" + from animawatch.server import list_devices + + result = await list_devices(category="mobile") + + assert "Available Device Profiles" in result + assert "Category" in result + assert "mobile" in result.lower() + + @pytest.mark.asyncio + async def test_list_devices_invalid_category(self) -> None: + """Test list_devices with invalid category.""" + from animawatch.server import list_devices + + result = await list_devices(category="invalid") + + assert "Invalid category" in result + + @pytest.fixture + def mock_app_context(self) -> AppContext: + """Create a mock AppContext for testing.""" + mock_browser = MagicMock() + mock_browser.record_interaction = AsyncMock(return_value=Path("/tmp/video.webm")) + mock_browser.take_screenshot = AsyncMock(return_value=Path("/tmp/screenshot.png")) + mock_browser.pooled_context = MagicMock() + + # Mock pooled_context as async context manager + mock_context = MagicMock() + mock_page = MagicMock() + mock_page.goto = AsyncMock() + mock_page.evaluate = AsyncMock(return_value={}) + + async def mock_pooled_context_cm(*args, **kwargs): + class AsyncCM: + async def __aenter__(self): + return (mock_context, mock_page) + + async def __aexit__(self, *args): + pass + + return AsyncCM() + + def pooled_ctx(*args, **kwargs): + return mock_pooled_context_cm(*args, **kwargs).__aenter__() + + mock_browser.pooled_context = pooled_ctx + + mock_vision = AsyncMock() + mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") + mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") + + return AppContext( + browser=mock_browser, + vision=mock_vision, + recordings={}, + analyses={}, + ) + + @pytest.fixture + def mock_ctx(self, mock_app_context: AppContext) -> MagicMock: + """Create a mock Context for testing tools.""" + mock_ctx = MagicMock() + mock_ctx.request_context.lifespan_context = mock_app_context + return mock_ctx + + @pytest.mark.asyncio + async def test_watch_with_device_valid( + self, mock_ctx: MagicMock, mock_app_context: AppContext + ) -> None: + """Test watch_with_device with valid device.""" + from animawatch.server import watch_with_device + + with patch("animawatch.server.uuid.uuid4") as mock_uuid: + mock_uuid.return_value = MagicMock() + mock_uuid.return_value.__str__ = lambda self: "abc12345-6789" + + result = await watch_with_device( + url="https://example.com", + device="iphone_15_pro", + ctx=mock_ctx, + ) + + assert "Device Animation Analysis" in result + assert "iPhone 15 Pro" in result + + @pytest.mark.asyncio + async def test_watch_with_device_invalid( + self, mock_ctx: MagicMock, mock_app_context: AppContext + ) -> None: + """Test watch_with_device with invalid device.""" + from animawatch.server import watch_with_device + + result = await watch_with_device( + url="https://example.com", + device="invalid_device", + ctx=mock_ctx, + ) + + assert "not found" in result + assert "Available devices" in result + + @pytest.mark.asyncio + async def test_watch_with_device_without_context(self) -> None: + """Test watch_with_device raises without context.""" + from animawatch.server import watch_with_device + + with pytest.raises(RuntimeError, match="Context is required"): + await watch_with_device( + url="https://example.com", + device="iphone_15_pro", + ctx=None, + ) + + @pytest.mark.asyncio + async def test_analyze_fps_file_not_found(self) -> None: + """Test analyze_fps with non-existent file.""" + from animawatch.server import analyze_fps + + result = await analyze_fps(video_path="/nonexistent/video.mp4") + + assert "Video not found" in result + + @pytest.mark.asyncio + async def test_get_performance_metrics_without_context(self) -> None: + """Test get_performance_metrics raises without context.""" + from animawatch.server import get_performance_metrics + + with pytest.raises(RuntimeError, match="Context is required"): + await get_performance_metrics(url="https://example.com", ctx=None) + + @pytest.mark.asyncio + async def test_analyze_with_consensus_without_context(self) -> None: + """Test analyze_with_consensus_tool raises without context.""" + from animawatch.server import analyze_with_consensus_tool + + with pytest.raises(RuntimeError, match="Context is required"): + await analyze_with_consensus_tool(url="https://example.com", ctx=None) + + @pytest.mark.asyncio + async def test_compare_screenshots_without_context(self) -> None: + """Test compare_screenshots raises without context.""" + from animawatch.server import compare_screenshots + + with pytest.raises(RuntimeError, match="Context is required"): + await compare_screenshots( + url1="https://example.com", + url2="https://example.org", + ctx=None, + ) From 4a23177c094c559a9ebc288c2d89f5d00868b3db Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 15:12:27 +0100 Subject: [PATCH 2/7] docs: fix markdownlint MD022/MD031/MD040 and align architecture box - Add a blank line between each heading and its fenced block, and tag every bare fence as `text` (usage examples, "What It Does", the architecture diagram); blank line before the two JSON config blocks. - Architecture box: nine rows were 77 columns wide against a 76-column border; trim the extra padding so every row's right edge lines up. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- README.md | 63 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 3b74754..43d759b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Built with **FastMCP** leveraging the latest MCP spec (2025-11-25) features. ## ✨ What It Does -``` +```text YOU: "Watch the modal animation on this page" ↓ ANIMAWATCH: @@ -90,6 +90,7 @@ cp .env.example .env ### 4. Add to Your MCP Client **Claude Desktop** (`~/Library/Application Support/Claude/claude_desktop_config.json`): + ```json { "mcpServers": { @@ -105,6 +106,7 @@ cp .env.example .env ``` **Augment Code** (settings): + ```json { "mcpServers": { @@ -122,57 +124,68 @@ cp .env.example .env ## πŸ“– Usage Examples ### Watch Animation Issues -``` + +```text "Watch the modal animation on https://example.com for any jank" ``` ### Perform Actions Then Watch -``` + +```text "Click the hamburger menu on https://example.com and watch the slide-in animation" ``` ### Focus on Specific Area -``` + +```text "Watch https://example.com with focus on scroll behavior" ``` ### Accessibility Check -``` + +```text "Check accessibility on https://example.com" ``` ### Access Previous Results -``` + +```text "Show me the analysis from animawatch://analyses/abc123" ``` ### Test on Mobile Device -``` + +```text "Watch https://example.com on an iPhone 15 Pro and check for animation issues" ``` ### Compare Before/After -``` + +```text "Compare screenshots of https://staging.example.com and https://example.com for visual differences" ``` ### Check Performance Metrics -``` + +```text "Get Core Web Vitals for https://example.com" ``` ### Analyze FPS -``` + +```text "Analyze the FPS of this video recording for frame drops" ``` ### Multi-Model Consensus -``` + +```text "Analyze https://example.com using both Gemini and Ollama for higher accuracy" ``` ### List Available Devices -``` + +```text "What mobile devices can I test with?" ``` @@ -216,16 +229,16 @@ export OLLAMA_MODEL=qwen2.5-vl:7b ## πŸ—οΈ Architecture -``` +```text β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ AnimaWatch β”‚ -β”‚ (FastMCP Server) β”‚ +β”‚ AnimaWatch β”‚ +β”‚ (FastMCP Server) β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Lifespan Context (AppContext) β”‚ -β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ -β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ -β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ -β”‚ └── analyses: dict[id, str] β”‚ +β”‚ Lifespan Context (AppContext) β”‚ +β”‚ β”œβ”€β”€ BrowserRecorder (Playwright) β”‚ +β”‚ β”œβ”€β”€ VisionProvider (Gemini/Ollama) β”‚ +β”‚ β”œβ”€β”€ recordings: dict[id, Path] β”‚ +β”‚ └── analyses: dict[id, str] β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ β”‚ Core Tools β”‚ Device & Performance β”‚ Comparison & Accuracy β”‚ β”‚ ────────────────── β”‚ ──────────────────── β”‚ ──────────────────── β”‚ @@ -235,11 +248,11 @@ export OLLAMA_MODEL=qwen2.5-vl:7b β”‚ analyze_video β”‚ get_performance_metricsβ”‚ β”‚ β”‚ check_accessibility β”‚ β”‚ β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ Resources β”‚ Prompts β”‚ -β”‚ ─────────────────────────── β”‚ ─────────────────────────────────────── β”‚ -β”‚ animawatch://recordings/{id} β”‚ animation_diagnosis β”‚ -β”‚ animawatch://analyses/{id} β”‚ page_analysis β”‚ -β”‚ animawatch://config β”‚ accessibility_check β”‚ +β”‚ Resources β”‚ Prompts β”‚ +β”‚ ─────────────────────────── β”‚ ─────────────────────────────────────── β”‚ +β”‚ animawatch://recordings/{id} β”‚ animation_diagnosis β”‚ +β”‚ animawatch://analyses/{id} β”‚ page_analysis β”‚ +β”‚ animawatch://config β”‚ accessibility_check β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` From 53c7dc7ae3381eb7128fb0bb3ff20839953340ef Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 15:17:22 +0100 Subject: [PATCH 3/7] test: share tool fixtures via conftest, fix pooled_context mock, add happy paths - Move the duplicated mock_app_context/mock_ctx fixtures from TestTools and TestNewTools into tests/conftest.py (plus a mock_page fixture). - Fix the pooled_context mock: the old factory called __aenter__ on a coroutine (AttributeError) and returned a coroutine, not an async context manager. It is now an asynccontextmanager wrapped in a MagicMock, so `async with browser.pooled_context() as (_, page)` works and calls can be asserted. - Mocked recording/screenshot paths now live under tmp_path, so tools that unlink them can never touch real files in /tmp. - Add happy-path tests for analyze_fps, get_performance_metrics, analyze_with_consensus_tool and compare_screenshots (differences and identical pages), asserting arguments, rendered output, stored analyses and temp-file cleanup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- tests/conftest.py | 59 ++++++++++ tests/test_server.py | 266 +++++++++++++++++++++++++++++++------------ 2 files changed, 252 insertions(+), 73 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..462d22a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,59 @@ +"""Shared pytest fixtures for AnimaWatch tests.""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from animawatch.server import AppContext + + +@pytest.fixture +def mock_page() -> MagicMock: + """A Playwright ``Page`` stand-in with awaitable navigation and evaluation.""" + page = MagicMock() + page.goto = AsyncMock() + page.evaluate = AsyncMock(return_value={}) + return page + + +@pytest.fixture +def mock_app_context(mock_page: MagicMock, tmp_path: Path) -> AppContext: + """An AppContext whose browser and vision provider are mocks. + + ``browser.pooled_context`` behaves like the real async context manager: calling + it returns an object usable in ``async with`` that yields ``(context, page)``. + It is wrapped in a ``MagicMock`` so tests can assert how it was called. + """ + mock_browser = AsyncMock() + # Paths live under tmp_path so tools that unlink them can't touch real files. + mock_browser.record_interaction = AsyncMock(return_value=tmp_path / "video.webm") + mock_browser.take_screenshot = AsyncMock(return_value=tmp_path / "screenshot.png") + + @asynccontextmanager + async def pooled_context(*_args: Any, **_kwargs: Any) -> AsyncIterator[tuple[Any, Any]]: + yield (MagicMock(), mock_page) + + mock_browser.pooled_context = MagicMock(side_effect=pooled_context) + + mock_vision = AsyncMock() + mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") + mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") + + return AppContext( + browser=mock_browser, + vision=mock_vision, + recordings={}, + analyses={}, + ) + + +@pytest.fixture +def mock_ctx(mock_app_context: AppContext) -> MagicMock: + """A tool ``Context`` whose lifespan context is ``mock_app_context``.""" + ctx = MagicMock() + ctx.request_context.lifespan_context = mock_app_context + return ctx diff --git a/tests/test_server.py b/tests/test_server.py index fe52668..8c9b9c9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,7 +5,12 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from PIL import Image +from animawatch.consensus import ConsensusResult +from animawatch.fps import FPSAnalysisResult, JankEvent +from animawatch.metrics import CoreWebVitals, PerformanceMetrics +from animawatch.models import Finding, IssueCategory, Severity from animawatch.server import ( ANIMATION_PROMPT, AppContext, @@ -169,32 +174,7 @@ def test_get_config_returns_settings(self) -> None: class TestTools: - """Tests for MCP tools.""" - - @pytest.fixture - def mock_app_context(self) -> AppContext: - """Create a mock AppContext for testing tools.""" - mock_browser = AsyncMock() - mock_browser.record_interaction = AsyncMock(return_value=Path("/tmp/video.webm")) - mock_browser.take_screenshot = AsyncMock(return_value=Path("/tmp/screenshot.png")) - - mock_vision = AsyncMock() - mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") - mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") - - return AppContext( - browser=mock_browser, - vision=mock_vision, - recordings={}, - analyses={}, - ) - - @pytest.fixture - def mock_ctx(self, mock_app_context: AppContext) -> MagicMock: - """Create a mock Context for testing tools.""" - mock_ctx = MagicMock() - mock_ctx.request_context.lifespan_context = mock_app_context - return mock_ctx + """Tests for MCP tools (fixtures ``mock_app_context``/``mock_ctx`` live in conftest).""" @pytest.mark.asyncio async def test_watch_records_and_analyzes( @@ -338,53 +318,6 @@ async def test_list_devices_invalid_category(self) -> None: assert "Invalid category" in result - @pytest.fixture - def mock_app_context(self) -> AppContext: - """Create a mock AppContext for testing.""" - mock_browser = MagicMock() - mock_browser.record_interaction = AsyncMock(return_value=Path("/tmp/video.webm")) - mock_browser.take_screenshot = AsyncMock(return_value=Path("/tmp/screenshot.png")) - mock_browser.pooled_context = MagicMock() - - # Mock pooled_context as async context manager - mock_context = MagicMock() - mock_page = MagicMock() - mock_page.goto = AsyncMock() - mock_page.evaluate = AsyncMock(return_value={}) - - async def mock_pooled_context_cm(*args, **kwargs): - class AsyncCM: - async def __aenter__(self): - return (mock_context, mock_page) - - async def __aexit__(self, *args): - pass - - return AsyncCM() - - def pooled_ctx(*args, **kwargs): - return mock_pooled_context_cm(*args, **kwargs).__aenter__() - - mock_browser.pooled_context = pooled_ctx - - mock_vision = AsyncMock() - mock_vision.analyze_video = AsyncMock(return_value="Video analysis result") - mock_vision.analyze_image = AsyncMock(return_value="Image analysis result") - - return AppContext( - browser=mock_browser, - vision=mock_vision, - recordings={}, - analyses={}, - ) - - @pytest.fixture - def mock_ctx(self, mock_app_context: AppContext) -> MagicMock: - """Create a mock Context for testing tools.""" - mock_ctx = MagicMock() - mock_ctx.request_context.lifespan_context = mock_app_context - return mock_ctx - @pytest.mark.asyncio async def test_watch_with_device_valid( self, mock_ctx: MagicMock, mock_app_context: AppContext @@ -469,3 +402,190 @@ async def test_compare_screenshots_without_context(self) -> None: url2="https://example.org", ctx=None, ) + + # ------------------------------------------------------------------ + # Happy paths for the new tools + # ------------------------------------------------------------------ + + @pytest.mark.asyncio + async def test_analyze_fps_reports_results(self, tmp_path: Path) -> None: + """analyze_fps passes its arguments through and renders the FPS report.""" + from animawatch.server import analyze_fps + + video = tmp_path / "recording.webm" + video.write_bytes(b"fake video") + fps_result = FPSAnalysisResult( + average_fps=58.5, + target_fps=30.0, + min_fps=40.0, + max_fps=60.0, + total_frames=117, + jank_events=[ + JankEvent( + frame_number=42, + timestamp_ms=700.0, + expected_delta_ms=33.3, + actual_delta_ms=50.0, + severity="minor", + dropped_frames=1, + ) + ], + jank_percentage=0.85, + frame_time_consistency=92.0, + duration_ms=2000.0, + ) + + with patch( + "animawatch.server.analyze_video_fps", AsyncMock(return_value=fps_result) + ) as mock_analyze: + result = await analyze_fps( + video_path=str(video), target_fps=30.0, jank_threshold_ms=8.0 + ) + + mock_analyze.assert_awaited_once_with(video, 30.0, 8.0) + assert result.startswith("## 🎯 FPS Analysis") + assert "**Average FPS**: 58.5 (target: 30.0)" in result + assert "**Jank Events**: 1" in result + assert "Frame 42 @ 700ms" in result + + @pytest.mark.asyncio + async def test_get_performance_metrics_reports_and_stores( + self, mock_ctx: MagicMock, mock_app_context: AppContext, mock_page: MagicMock + ) -> None: + """get_performance_metrics navigates a pooled page and stores the report.""" + from animawatch.server import get_performance_metrics + + metrics = PerformanceMetrics( + url="https://example.com", + core_web_vitals=CoreWebVitals(lcp_ms=1800.0, fcp_ms=900.0, cls_score=0.02), + load_time_ms=2100.0, + dom_content_loaded_ms=1200.0, + resource_count=12, + total_transfer_size_kb=340.0, + ) + + with patch( + "animawatch.server.extract_performance_metrics", AsyncMock(return_value=metrics) + ) as mock_extract: + result = await get_performance_metrics(url="https://example.com", ctx=mock_ctx) + + cast(MagicMock, mock_app_context.browser.pooled_context).assert_called_once_with() + mock_page.goto.assert_awaited_once_with("https://example.com", wait_until="networkidle") + mock_extract.assert_awaited_once_with(mock_page, "https://example.com") + assert result.startswith("**Analysis ID**: `") + assert "Performance Metrics Report" in result + assert "LCP" in result + assert len(mock_app_context.analyses) == 1 + (stored,) = mock_app_context.analyses.values() + assert result.endswith(stored) + + @pytest.mark.asyncio + async def test_analyze_with_consensus_reports_findings( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """analyze_with_consensus_tool renders agreed and single-model findings.""" + from animawatch.server import analyze_with_consensus_tool + + screenshot = tmp_path / "consensus.png" + screenshot.write_bytes(b"fake png") + mock_app_context.browser.take_screenshot = AsyncMock(return_value=screenshot) + + agreed = Finding( + id="f1", + category=IssueCategory.ANIMATION, + severity=Severity.MAJOR, + confidence=90, + element="hero banner", + description="Banner stutters while sliding in", + suggestion="Animate transform instead of left", + ) + gemini_only = Finding( + id="f2", + category=IssueCategory.LAYOUT, + severity=Severity.MINOR, + confidence=60, + element="footer", + description="Footer shifts after fonts load", + suggestion="Reserve space for the footer", + ) + consensus = ConsensusResult( + merged_findings=[agreed, gemini_only], + gemini_only=[gemini_only], + ollama_only=[], + agreed_findings=[agreed], + gemini_result=None, + ollama_result=None, + consensus_score=50.0, + ) + + with patch( + "animawatch.server.run_consensus_analysis", AsyncMock(return_value=consensus) + ) as mock_consensus: + result = await analyze_with_consensus_tool( + url="https://example.com", focus="timing", ctx=mock_ctx + ) + + mock_consensus.assert_awaited_once() + assert mock_consensus.await_args is not None + assert mock_consensus.await_args.args[0] == screenshot + assert "**Consensus Score**: 50%" in result + assert "**major** [animation]: Banner stutters while sliding in" in result + assert "Animate transform instead of left" in result + assert "Gemini-Only Findings" in result + assert "Ollama-Only Findings" not in result + assert not screenshot.exists() # temporary screenshot is cleaned up + assert list(mock_app_context.analyses.values()) == [result] + + @staticmethod + def _write_png(path: Path, square: bool) -> Path: + """Write a 40x40 white PNG, optionally with a 20x20 black square at (10, 10).""" + image = Image.new("RGB", (40, 40), "white") + if square: + image.paste((0, 0, 0), (10, 10, 30, 30)) + image.save(path) + return path + + @pytest.mark.asyncio + async def test_compare_screenshots_detects_differences( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """compare_screenshots reports the differing region and cleans up screenshots.""" + from animawatch.server import compare_screenshots + + before = self._write_png(tmp_path / "before.png", square=False) + after = self._write_png(tmp_path / "after.png", square=True) + mock_app_context.browser.take_screenshot = AsyncMock(side_effect=[before, after]) + + result = await compare_screenshots( + url1="https://example.com", url2="https://example.org", ctx=mock_ctx + ) + + assert "Differences detected!" in result + assert "**Diff Percentage**: 25.00%" in result # 400 of 1600 pixels + assert "**Region 1**: (10, 10) 20x20" in result + assert not before.exists() + assert not after.exists() + + diff_image = Path(result.rsplit("Diff image saved: `", 1)[1].rstrip("`")) + try: + assert diff_image.exists() + finally: + diff_image.unlink(missing_ok=True) + + @pytest.mark.asyncio + async def test_compare_screenshots_identical( + self, mock_ctx: MagicMock, mock_app_context: AppContext, tmp_path: Path + ) -> None: + """compare_screenshots reports no differences for identical pages.""" + from animawatch.server import compare_screenshots + + before = self._write_png(tmp_path / "before.png", square=True) + after = self._write_png(tmp_path / "after.png", square=True) + mock_app_context.browser.take_screenshot = AsyncMock(side_effect=[before, after]) + + result = await compare_screenshots( + url1="https://example.com", url2="https://example.com", ctx=mock_ctx + ) + + assert "No visual differences detected!" in result + assert "Similarity: 100.0%" in result From c370ecc6c8cfb0680f29a2e4f42dab8bc305a6ed Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 15:18:24 +0100 Subject: [PATCH 4/7] perf(diff): build the diff highlight overlay with a mask instead of getpixel _generate_diff_image looped over every pixel in Python, calling getpixel and draw.point. Build a threshold mask with Image.point and paste the highlight colour through it instead; both run in C. Output is byte-identical (1280x720: ~0.55s -> ~0.05s including the PNG save). Regression test compares the generated image byte-for-byte against a pixel-by-pixel reference, for opaque and translucent highlights, and checks that a pixel exactly at the threshold is left untouched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- src/animawatch/diff.py | 28 +++++++++++----------------- tests/test_diff.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/animawatch/diff.py b/src/animawatch/diff.py index 1fa2f7e..e0cd6ef 100644 --- a/src/animawatch/diff.py +++ b/src/animawatch/diff.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path -from PIL import Image, ImageChops, ImageDraw +from PIL import Image, ImageChops @dataclass @@ -152,22 +152,16 @@ def _generate_diff_image( highlight_color: tuple[int, int, int, int], ) -> Path: """Generate a diff image with highlighted changes.""" - # Create a copy of the "after" image - result = after.copy() - - # Create an overlay for highlighting - overlay = Image.new("RGBA", result.size, (0, 0, 0, 0)) - draw = ImageDraw.Draw(overlay) - - # Highlight differing pixels using getpixel for type safety - for x in range(diff_gray.width): - for y in range(diff_gray.height): - pixel_value = diff_gray.getpixel((x, y)) - if isinstance(pixel_value, int) and pixel_value > threshold: - draw.point((x, y), fill=highlight_color) - - # Composite the overlay onto the result - result = Image.alpha_composite(result.convert("RGBA"), overlay) + # Mask of differing pixels: 255 where the difference exceeds the threshold. + # Image.point runs in C, unlike a per-pixel getpixel loop in Python. + mask = diff_gray.point(lambda p: 255 if p > threshold else 0) + + # Transparent overlay with the highlight colour painted only under the mask + overlay = Image.new("RGBA", after.size, (0, 0, 0, 0)) + overlay.paste(highlight_color, (0, 0, *after.size), mask) + + # Composite the overlay onto the "after" image + result = Image.alpha_composite(after.convert("RGBA"), overlay) # Save to temp file fd, tmp_path = tempfile.mkstemp(suffix="_diff.png") diff --git a/tests/test_diff.py b/tests/test_diff.py index cb415a2..44a51dd 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from PIL import Image +from PIL import Image, ImageChops from animawatch.diff import ( DiffRegion, @@ -113,6 +113,46 @@ def test_diff_image_generation(self, different_images: tuple[Path, Path]) -> Non assert result.diff_image_path.exists() result.diff_image_path.unlink(missing_ok=True) + @pytest.mark.parametrize( + "highlight_color", [(0, 0, 255, 255), (255, 0, 0, 128)], ids=["opaque", "translucent"] + ) + def test_diff_image_highlights_exactly_pixels_above_threshold( + self, tmp_path: Path, highlight_color: tuple[int, int, int, int] + ) -> None: + """The diff image is the "after" image with only above-threshold pixels overlaid.""" + threshold = 5 + before = Image.new("RGB", (50, 50), "white") + after = before.copy() + after.paste((255, 0, 0), (10, 10, 20, 20)) # strong difference + after.paste((250, 250, 250), (30, 30, 40, 40)) # grey diff of exactly `threshold` + before_path, after_path = tmp_path / "before.png", tmp_path / "after.png" + before.save(before_path) + after.save(after_path) + + result = compare_images( + before_path, after_path, threshold=threshold, highlight_color=highlight_color + ) + assert result.diff_image_path is not None + try: + actual = Image.open(result.diff_image_path).convert("RGBA") + + # Reference: overlay the highlight pixel by pixel wherever the grayscale + # difference is strictly greater than the threshold. + diff_gray = ImageChops.difference(before, after).convert("L") + overlay = Image.new("RGBA", after.size, (0, 0, 0, 0)) + for x in range(after.width): + for y in range(after.height): + value = diff_gray.getpixel((x, y)) + if isinstance(value, int) and value > threshold: + overlay.putpixel((x, y), highlight_color) + expected = Image.alpha_composite(after.convert("RGBA"), overlay) + + assert actual.tobytes() == expected.tobytes() + assert actual.getpixel((35, 35)) == (250, 250, 250, 255) # at threshold: untouched + assert actual.getpixel((0, 0)) == (255, 255, 255, 255) # unchanged: untouched + finally: + result.diff_image_path.unlink(missing_ok=True) + def test_result_paths(self, identical_images: tuple[Path, Path]) -> None: """Test that result contains correct paths.""" before, after = identical_images From 71efe3a77a1b3c5c82290d6831f63ee93a0811d5 Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 15:24:12 +0100 Subject: [PATCH 5/7] test: document the pooled_context stand-in in conftest Addresses the pre-merge Docstring Coverage warning (78.95% < 80%): the only undocumented function this PR touches is the nested helper. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 462d22a..76105d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,7 @@ def mock_app_context(mock_page: MagicMock, tmp_path: Path) -> AppContext: @asynccontextmanager async def pooled_context(*_args: Any, **_kwargs: Any) -> AsyncIterator[tuple[Any, Any]]: + """Yield a ``(context, page)`` pair like ``BrowserRecorder.pooled_context``.""" yield (MagicMock(), mock_page) mock_browser.pooled_context = MagicMock(side_effect=pooled_context) From e13577c2ca68c7c6d88b2535a06dcb6d77098c5e Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 17:18:06 +0100 Subject: [PATCH 6/7] test: clean up the diff image even when an assertion fails Locate the generated diff image straight after compare_screenshots and run every result assertion inside the try block, so the finally clause always removes the temp file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- tests/test_server.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index 8c9b9c9..f75207a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -560,17 +560,20 @@ async def test_compare_screenshots_detects_differences( url1="https://example.com", url2="https://example.org", ctx=mock_ctx ) - assert "Differences detected!" in result - assert "**Diff Percentage**: 25.00%" in result # 400 of 1600 pixels - assert "**Region 1**: (10, 10) 20x20" in result - assert not before.exists() - assert not after.exists() - - diff_image = Path(result.rsplit("Diff image saved: `", 1)[1].rstrip("`")) + # Locate the generated diff image first so it is removed even if an assertion fails. + marker = "Diff image saved: `" + diff_image = Path(result.rsplit(marker, 1)[1].rstrip("`")) if marker in result else None try: + assert "Differences detected!" in result + assert "**Diff Percentage**: 25.00%" in result # 400 of 1600 pixels + assert "**Region 1**: (10, 10) 20x20" in result + assert not before.exists() + assert not after.exists() + assert diff_image is not None assert diff_image.exists() finally: - diff_image.unlink(missing_ok=True) + if diff_image is not None: + diff_image.unlink(missing_ok=True) @pytest.mark.asyncio async def test_compare_screenshots_identical( From 964484e17d37703d4ac32b913cd1c492e8460a0d Mon Sep 17 00:00:00 2001 From: Rishi Tank Date: Fri, 25 Sep 2026 22:50:37 +0100 Subject: [PATCH 7/7] test: assert device filtering and device forwarding explicitly Review of d7b2d4a: - test_list_devices_by_category: the old check would pass for any output that mentions "mobile". It now requires the "**Category**: mobile" header, two mobile profiles (iphone_15_pro, pixel_8), and the absence of a tablet profile (ipad_pro_12) and a desktop profile (desktop_1080p). - test_watch_with_device_valid: now also asserts that browser.record_interaction was called once with device="iphone_15_pro" (and the URL), so dropping the device argument would fail the test. 166 tests pass. Ruff check, ruff format --check and mypy src/ are clean. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0171XyyqTUH64gAi1AcerwjN --- tests/test_server.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index f75207a..47a1dc6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -306,8 +306,12 @@ async def test_list_devices_by_category(self) -> None: result = await list_devices(category="mobile") assert "Available Device Profiles" in result - assert "Category" in result - assert "mobile" in result.lower() + assert "**Category**: mobile" in result + # Mobile profiles are listed; tablet and desktop profiles are filtered out. + assert "**iphone_15_pro**" in result + assert "**pixel_8**" in result + assert "**ipad_pro_12**" not in result + assert "**desktop_1080p**" not in result @pytest.mark.asyncio async def test_list_devices_invalid_category(self) -> None: @@ -337,6 +341,11 @@ async def test_watch_with_device_valid( assert "Device Animation Analysis" in result assert "iPhone 15 Pro" in result + # The selected device is forwarded to the recorder. + record = cast(AsyncMock, mock_app_context.browser.record_interaction) + record.assert_called_once() + assert record.call_args.kwargs["device"] == "iphone_15_pro" + assert record.call_args.kwargs["url"] == "https://example.com" @pytest.mark.asyncio async def test_watch_with_device_invalid(