From 4ad74eccf2861dc893079b30d768f929b8a396dc Mon Sep 17 00:00:00 2001 From: Keenan Zhou Date: Wed, 17 Jun 2026 11:16:09 -0700 Subject: [PATCH 01/35] Add deps and config for alt-text-quality rule --- .gitignore | 1 + package-lock.json | 519 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 4 +- tsconfig.json | 2 +- 4 files changed, 524 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2eb939f..697a621 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ coverage/ .vitest/ *.log .DS_Store +.env diff --git a/package-lock.json b/package-lock.json index 171add8..a13e022 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "eslint-plugin-check-file": "^3.3.1", "playwright": "^1.61.1", "prettier": "^3.8.4", + "tsx": "^4.22.4", "typescript": "^6.0.3", "typescript-eslint": "^8.62.0", "vitest": "^4.1.9" @@ -58,6 +59,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -1126,6 +1569,48 @@ "dev": true, "license": "MIT" }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -2355,6 +2840,40 @@ "license": "0BSD", "optional": true }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 146e1d5..5e9350a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "typecheck": "tsc --noEmit", "lint": "eslint .", "format": "prettier --write .", - "format:check": "prettier --check ." + "format:check": "prettier --check .", + "probe": "tsx --env-file=.env scripts/probe-alt-quality.ts" }, "prettier": "@github/prettier-config", "engines": { @@ -25,6 +26,7 @@ "eslint-plugin-check-file": "^3.3.1", "playwright": "^1.61.1", "prettier": "^3.8.4", + "tsx": "^4.22.4", "typescript": "^6.0.3", "typescript-eslint": "^8.62.0", "vitest": "^4.1.9" diff --git a/tsconfig.json b/tsconfig.json index fa0e524..bad082c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,5 +15,5 @@ "isolatedModules": true, "noEmit": true }, - "include": ["index.ts", "src/**/*.ts", "tests/**/*.ts"] + "include": ["index.ts", "src/**/*.ts", "tests/**/*.ts", "scripts/**/*.ts"] } From 7681fd9430445c546c2a6dacf26ffa11190f8b47 Mon Sep 17 00:00:00 2001 From: Keenan Zhou Date: Wed, 17 Jun 2026 11:16:30 -0700 Subject: [PATCH 02/35] Add alt-text judge provider seam (Copilot + Azure-augmented) --- src/judges/azure-augmented-judge.ts | 111 +++++++++++++++++++ src/judges/azure-vision-api-client.ts | 121 +++++++++++++++++++++ src/judges/copilot-judge.ts | 100 +++++++++++++++++ src/judges/index.ts | 58 ++++++++++ src/judges/prompt.ts | 150 ++++++++++++++++++++++++++ src/judges/types.ts | 33 ++++++ src/utils/load-image-data-url.ts | 47 ++++++++ 7 files changed, 620 insertions(+) create mode 100644 src/judges/azure-augmented-judge.ts create mode 100644 src/judges/azure-vision-api-client.ts create mode 100644 src/judges/copilot-judge.ts create mode 100644 src/judges/index.ts create mode 100644 src/judges/prompt.ts create mode 100644 src/judges/types.ts create mode 100644 src/utils/load-image-data-url.ts diff --git a/src/judges/azure-augmented-judge.ts b/src/judges/azure-augmented-judge.ts new file mode 100644 index 0000000..2cb5139 --- /dev/null +++ b/src/judges/azure-augmented-judge.ts @@ -0,0 +1,111 @@ +// AzureAugmentedJudge — decorator over a JudgeAltText (typically CopilotJudge) +// that runs an Azure Computer Vision pre-pass and folds the result into the +// context handed to the inner judge. +// +// The decorator class, the AzureVisionClient interface, and the context +// composition logic are all production code today. The only piece that is +// not implemented is a real AzureVisionClient — see NotImplementedAzureVisionClient +// at the bottom of this file. To enable azure-augmented mode in production: +// +// 1. npm install @azure/ai-vision-image-analysis (or your preferred SDK) +// 2. Create src/judges/azure-vision-api-client.ts implementing AzureVisionClient +// against the SDK; shape the SDK response into AzureVisionAnalysis. +// 3. Wire it into the factory: +// createJudge({mode: 'azure-augmented', visionClient: new AzureVisionApiClient(creds)}) +// +// No other file in the codebase has to change. + +import type {JudgeAltText, JudgeInput, JudgeVerdict} from './types.js' + +// Narrowed shape of an Azure AI Vision 4.0 Image Analysis response — only the +// fields we actually feed into the Copilot judge. +export type AzureVisionAnalysis = { + caption?: {text: string; confidence: number} + denseCaptions?: Array<{text: string; confidence: number}> + // OCR result, concatenated to a single string. Useful for images-of-text + // (logos, screenshots, charts with embedded labels) where GPT-4o is shakier. + readText?: string + tags?: Array<{name: string; confidence: number}> +} + +// The capability the decorator depends on. Abstracted so the SDK choice +// (Azure SDK v4 vs REST) is swappable, and so tests can mock it cheaply. +export interface AzureVisionClient { + analyze(imageDataUrl: string): Promise +} + +export type AzureAugmentedJudgeConfig = { + // The judge that does the actual reasoning. Typically a CopilotJudge. + inner: JudgeAltText + // The Azure CV client. Today this is NotImplementedAzureVisionClient by + // default; replace with a real client to enable the mode. + vision: AzureVisionClient + // Tags below this confidence are dropped before being passed to the model. + tagConfidenceThreshold?: number + // Maximum number of tags forwarded to the inner judge. + maxTags?: number +} + +const DEFAULT_TAG_CONFIDENCE = 0.7 +const DEFAULT_MAX_TAGS = 8 + +export class AzureAugmentedJudge implements JudgeAltText { + private readonly inner: JudgeAltText + private readonly vision: AzureVisionClient + private readonly tagConfidenceThreshold: number + private readonly maxTags: number + + constructor(config: AzureAugmentedJudgeConfig) { + this.inner = config.inner + this.vision = config.vision + this.tagConfidenceThreshold = config.tagConfidenceThreshold ?? DEFAULT_TAG_CONFIDENCE + this.maxTags = config.maxTags ?? DEFAULT_MAX_TAGS + } + + async judge(input: JudgeInput): Promise { + let analysis: AzureVisionAnalysis | null = null + try { + analysis = await this.vision.analyze(input.imageDataUrl) + } catch (err) { + // Azure CV is auxiliary grounding, not a hard dependency. If it fails + // (image too small, transient 5xx, rate-limit, etc.), degrade gracefully + // to Copilot-only for this image rather than failing the whole judgment. + console.warn( + `[alt-text-quality] Azure pre-pass failed; falling back to Copilot-only for this image. ${err instanceof Error ? err.message : String(err)}`, + ) + } + const enriched = analysis ? this.composeContext(input.context, analysis) : input.context + return this.inner.judge({...input, context: enriched}) + } + + private composeContext(original: string, a: AzureVisionAnalysis): string { + const parts: string[] = [] + if (a.caption) parts.push(`Azure CV caption: ${a.caption.text}`) + if (a.denseCaptions?.length) { + parts.push(`Azure CV regions: ${a.denseCaptions.map(c => c.text).join('; ')}`) + } + if (a.readText) parts.push(`Azure CV OCR: ${a.readText}`) + if (a.tags?.length) { + const top = a.tags + .filter(t => t.confidence >= this.tagConfidenceThreshold) + .slice(0, this.maxTags) + .map(t => t.name) + if (top.length) parts.push(`Azure CV tags: ${top.join(', ')}`) + } + if (parts.length === 0) return original + return `${original}\n\nAzure Computer Vision pre-analysis (use as supplementary signals; the page's own context above is authoritative):\n${parts.join('\n')}` + } +} + +// Today's placeholder AzureVisionClient. Throws on use with a clear message +// pointing at the file-level setup instructions. Replace with a real SDK-backed +// client when adopting Azure. +export class NotImplementedAzureVisionClient implements AzureVisionClient { + async analyze(_imageDataUrl: string): Promise { + throw new Error( + 'AzureAugmentedJudge was invoked but no real AzureVisionClient is wired. ' + + 'See src/judges/azure-augmented-judge.ts for the integration steps, then pass ' + + 'a real client via createJudge({mode: "azure-augmented", visionClient}).', + ) + } +} diff --git a/src/judges/azure-vision-api-client.ts b/src/judges/azure-vision-api-client.ts new file mode 100644 index 0000000..781c225 --- /dev/null +++ b/src/judges/azure-vision-api-client.ts @@ -0,0 +1,121 @@ +// Real Azure AI Vision (Image Analysis 4.0) client. Implements the +// AzureVisionClient interface that AzureAugmentedJudge depends on. +// +// Auth is via subscription key (Ocp-Apim-Subscription-Key header). For +// production deployments, prefer Azure managed identity; this client is +// designed for local development and CI where a key is the simplest path. +// +// Docs: https://learn.microsoft.com/azure/ai-services/computer-vision/concept-describe-images-40 +// https://learn.microsoft.com/rest/api/computervision/image-analysis/analyze-image + +import {Buffer} from 'node:buffer' +import type {AzureVisionAnalysis, AzureVisionClient} from './azure-augmented-judge.js' + +export type AzureVisionApiClientConfig = { + // Resource endpoint, e.g. "https://alt-text-scanner-plugin.cognitiveservices.azure.com/". + // Trailing slash is optional. Defaults to AZURE_VISION_ENDPOINT. + endpoint?: string + // Subscription key from the resource's "Keys and endpoint" page. + // Defaults to AZURE_VISION_KEY. + key?: string + // Image Analysis API version. Defaults to the GA version. + apiVersion?: string + // Comma-separated feature list. The decorator's composeContext can use any + // subset of: caption, denseCaptions, read, tags. Note: caption and + // denseCaptions are restricted to specific regions (East US, West US, + // West Europe, etc.) — West US 2 does NOT support them. Default is the + // always-available subset; override per-region if your resource supports more. + features?: string +} + +const DEFAULT_API_VERSION = '2024-02-01' +// Read (OCR) + tags are available in every region. Caption / denseCaptions +// require East US, West US, West Europe, North Europe, Southeast Asia, +// East Asia, France Central, Korea Central, West Central US, Switzerland +// North, or Australia East. Override via AZURE_VISION_FEATURES if your +// resource is in one of those regions. +const DEFAULT_FEATURES = 'read,tags' + +// Narrowed shape of the Image Analysis 4.0 response. +type AzureRawResponse = { + captionResult?: {text: string; confidence: number} + denseCaptionsResult?: {values: Array<{text: string; confidence: number}>} + readResult?: { + blocks?: Array<{ + lines?: Array<{text: string}> + }> + } + tagsResult?: {values: Array<{name: string; confidence: number}>} +} + +export class AzureVisionApiClient implements AzureVisionClient { + private readonly endpoint: string + private readonly key: string + private readonly apiVersion: string + private readonly features: string + + constructor(config: AzureVisionApiClientConfig = {}) { + const endpoint = config.endpoint ?? process.env['AZURE_VISION_ENDPOINT'] + const key = config.key ?? process.env['AZURE_VISION_KEY'] + if (!endpoint) { + throw new Error( + 'AzureVisionApiClient requires an endpoint. Set AZURE_VISION_ENDPOINT or pass {endpoint} to the constructor.', + ) + } + if (!key) { + throw new Error( + 'AzureVisionApiClient requires a key. Set AZURE_VISION_KEY or pass {key} to the constructor.', + ) + } + this.endpoint = endpoint.replace(/\/$/, '') + this.key = key + this.apiVersion = config.apiVersion ?? process.env['AZURE_VISION_API_VERSION'] ?? DEFAULT_API_VERSION + this.features = config.features ?? process.env['AZURE_VISION_FEATURES'] ?? DEFAULT_FEATURES + } + + async analyze(imageDataUrl: string): Promise { + const bytes = decodeDataUrl(imageDataUrl) + + const url = new URL(`${this.endpoint}/computervision/imageanalysis:analyze`) + url.searchParams.set('api-version', this.apiVersion) + url.searchParams.set('features', this.features) + + const res = await fetch(url, { + method: 'POST', + headers: { + 'Ocp-Apim-Subscription-Key': this.key, + 'Content-Type': 'application/octet-stream', + }, + body: new Blob([Uint8Array.from(bytes)]), + }) + + if (!res.ok) { + const errText = await res.text() + throw new Error(`Azure Vision request failed: ${res.status} ${res.statusText}\n${errText}`) + } + + const raw = (await res.json()) as AzureRawResponse + return shape(raw) + } +} + +function decodeDataUrl(dataUrl: string): Buffer { + const match = /^data:[^;]+;base64,(.+)$/.exec(dataUrl) + if (!match) throw new Error('AzureVisionApiClient.analyze expects a base64 data URL.') + return Buffer.from(match[1]!, 'base64') +} + +function shape(raw: AzureRawResponse): AzureVisionAnalysis { + const out: AzureVisionAnalysis = {} + if (raw.captionResult) out.caption = raw.captionResult + if (raw.denseCaptionsResult?.values?.length) out.denseCaptions = raw.denseCaptionsResult.values + if (raw.readResult?.blocks?.length) { + const lines: string[] = [] + for (const block of raw.readResult.blocks) { + for (const line of block.lines ?? []) lines.push(line.text) + } + if (lines.length) out.readText = lines.join('\n') + } + if (raw.tagsResult?.values?.length) out.tags = raw.tagsResult.values + return out +} diff --git a/src/judges/copilot-judge.ts b/src/judges/copilot-judge.ts new file mode 100644 index 0000000..242c394 --- /dev/null +++ b/src/judges/copilot-judge.ts @@ -0,0 +1,100 @@ +// CopilotJudge — calls a vision-capable model on GitHub Models with the +// shared SYSTEM_PROMPT and VERDICT_SCHEMA, returns a JudgeVerdict. +// +// This is the default JudgeAltText implementation and the one the offline +// probe exercises end-to-end. The AzureAugmentedJudge wraps an instance of +// this class and feeds it an enriched context. + +import {SYSTEM_PROMPT, VERDICT_SCHEMA} from './prompt.js' +import type {JudgeAltText, JudgeInput, JudgeVerdict} from './types.js' + +export type CopilotJudgeConfig = { + // PAT with the `models:read` scope. Defaults to GITHUB_MODELS_TOKEN + // or GITHUB_TOKEN from the environment. + token?: string + // Model id, e.g. "openai/gpt-4o". Defaults to PROBE_MODEL or "openai/gpt-4o". + model?: string + // Override for tests or self-hosted gateways. + endpoint?: string + apiVersion?: string + // Sampling temperature. Defaults to 0 for deterministic verdicts. + temperature?: number +} + +const DEFAULT_ENDPOINT = 'https://models.github.ai/inference/chat/completions' +const DEFAULT_API_VERSION = '2026-03-10' +const DEFAULT_MODEL = 'openai/gpt-4o' + +type ChatCompletionResponse = { + choices?: Array<{message?: {content?: string}}> +} + +export class CopilotJudge implements JudgeAltText { + private readonly token: string + private readonly model: string + private readonly endpoint: string + private readonly apiVersion: string + private readonly temperature: number + + constructor(config: CopilotJudgeConfig = {}) { + const token = config.token ?? process.env['GITHUB_MODELS_TOKEN'] ?? process.env['GITHUB_TOKEN'] + if (!token) { + throw new Error( + 'CopilotJudge requires a token. Set GITHUB_MODELS_TOKEN (or GITHUB_TOKEN) ' + + 'to a PAT with the `models:read` scope, or pass {token} to the constructor.', + ) + } + this.token = token + this.model = config.model ?? process.env['PROBE_MODEL'] ?? DEFAULT_MODEL + this.endpoint = config.endpoint ?? DEFAULT_ENDPOINT + this.apiVersion = config.apiVersion ?? DEFAULT_API_VERSION + this.temperature = config.temperature ?? 0 + } + + async judge(input: JudgeInput): Promise { + const userText = + `Surrounding context: ${input.context || '(none provided)'}\n` + + `Current alt text: ${JSON.stringify(input.alt)}\n\n` + + `Evaluate the alt text against the image and respond with the required JSON object.` + + const body = { + model: this.model, + messages: [ + {role: 'system', content: SYSTEM_PROMPT}, + { + role: 'user', + content: [ + {type: 'text', text: userText}, + {type: 'image_url', image_url: {url: input.imageDataUrl, detail: 'high'}}, + ], + }, + ], + response_format: {type: 'json_schema', json_schema: VERDICT_SCHEMA}, + temperature: this.temperature, + } + + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${this.token}`, + 'X-GitHub-Api-Version': this.apiVersion, + }, + body: JSON.stringify(body), + }) + + if (!res.ok) { + const errText = await res.text() + throw new Error(`inference failed: ${res.status} ${res.statusText}\n${errText}`) + } + + const json = (await res.json()) as ChatCompletionResponse + const raw = json.choices?.[0]?.message?.content ?? '' + try { + return JSON.parse(raw) as JudgeVerdict + } catch { + throw new Error(`failed to parse model output as JSON:\n${raw}`) + } + } +} diff --git a/src/judges/index.ts b/src/judges/index.ts new file mode 100644 index 0000000..f4b81ef --- /dev/null +++ b/src/judges/index.ts @@ -0,0 +1,58 @@ +// Public entry-point for the judge layer. +// +// Consumers (the rule, the probe, tests) call createJudge() and never reference +// concrete implementations directly. Mode selection is driven by the +// ALT_TEXT_JUDGE_MODE env var unless overridden via the options argument. + +import {AzureAugmentedJudge, NotImplementedAzureVisionClient} from './azure-augmented-judge.js' +import type {AzureVisionClient} from './azure-augmented-judge.js' +import {AzureVisionApiClient} from './azure-vision-api-client.js' +import {CopilotJudge} from './copilot-judge.js' +import type {CopilotJudgeConfig} from './copilot-judge.js' +import type {JudgeAltText, JudgeMode} from './types.js' + +export type {JudgeAltText, JudgeInput, JudgeVerdict, JudgeMode, Verdict} from './types.js' +export {CopilotJudge} from './copilot-judge.js' +export type {CopilotJudgeConfig} from './copilot-judge.js' +export {AzureAugmentedJudge, NotImplementedAzureVisionClient} from './azure-augmented-judge.js' +export type {AzureVisionClient, AzureVisionAnalysis, AzureAugmentedJudgeConfig} from './azure-augmented-judge.js' +export {AzureVisionApiClient} from './azure-vision-api-client.js' +export type {AzureVisionApiClientConfig} from './azure-vision-api-client.js' +export {SYSTEM_PROMPT, VERDICT_SCHEMA} from './prompt.js' + +export type CreateJudgeOptions = { + // Overrides ALT_TEXT_JUDGE_MODE. Defaults to 'copilot' when neither is set. + mode?: JudgeMode + // Forwarded to the inner CopilotJudge. + copilot?: CopilotJudgeConfig + // Required for azure-augmented mode. When omitted, the factory auto-constructs + // an AzureVisionApiClient if AZURE_VISION_ENDPOINT and AZURE_VISION_KEY are set, + // and falls back to NotImplementedAzureVisionClient (which throws on use) + // otherwise — so misconfiguration surfaces with a clear error on first call. + visionClient?: AzureVisionClient +} + +function resolveMode(opts: CreateJudgeOptions): JudgeMode { + if (opts.mode) return opts.mode + const env = process.env['ALT_TEXT_JUDGE_MODE'] + if (env === 'copilot' || env === 'azure-augmented') return env + return 'copilot' +} + +function resolveVisionClient(opts: CreateJudgeOptions): AzureVisionClient { + if (opts.visionClient) return opts.visionClient + if (process.env['AZURE_VISION_ENDPOINT'] && process.env['AZURE_VISION_KEY']) { + return new AzureVisionApiClient() + } + return new NotImplementedAzureVisionClient() +} + +export function createJudge(opts: CreateJudgeOptions = {}): JudgeAltText { + const mode = resolveMode(opts) + const copilot = new CopilotJudge(opts.copilot) + if (mode === 'copilot') return copilot + return new AzureAugmentedJudge({ + inner: copilot, + vision: resolveVisionClient(opts), + }) +} diff --git a/src/judges/prompt.ts b/src/judges/prompt.ts new file mode 100644 index 0000000..61c275d --- /dev/null +++ b/src/judges/prompt.ts @@ -0,0 +1,150 @@ +// System prompt and structured-output schema for the alt-text-quality judge. +// +// Both the production rule and the offline probe import these constants, which +// guarantees the two paths reason against the exact same instructions. + +export const SYSTEM_PROMPT = `# Identity + +You are an accessibility reviewer evaluating whether the alt text on a single HTML element is appropriate, given the image and the surrounding page context. + +You are conservative and trust the page author's framing. You do not impose your own preferred level of detail. You do not flag alt text just because you would have written something different. + +# Decision procedure + +Walk these steps in order. Stop at the first step that matches and emit its verdict. + +## Step 1 — Purely decorative +The image is purely visual styling: separators, borders, spacers, ornamental flourishes, or generic mood/stock photography that contributes no information specific to this page. + - If current alt is "" → verdict = "ok" + - If current alt is non-empty → verdict = "decorative" + +## Step 2 — Redundant with adjacent label/caption +The image's full informational content is already conveyed by **label-style or caption-style** adjacent text — specifically a figure caption directly above/below the image, or a short labeling string next to the image within the same component (e.g., a name plate under a portrait, an HTML figcaption element). + +This step does NOT apply when surrounding **body prose** merely *mentions* the image's subject. Body prose mentioning the subject still leaves the image with informational content (the subject's appearance, the visual evidence of a property, etc.) that an alt should convey — use Step 4 for that case. + +A reliable signal: if the provided context calls the adjacent text a "caption", "label", or "figcaption", Step 2 may apply. If the context calls it "body text", "paragraph", or "surrounding text", Step 2 does NOT apply — go to Step 4. + - If current alt is "" → verdict = "ok" + - If current alt is non-empty → verdict = "decorative" + +## Step 3 — Functional (image inside a link or button) +The image is the only content of a link or button. The alt becomes the link's accessible name. Apply criteria F–H below. The alt is "ok" if and only if ALL hold: + + F. NAMES-THE-TARGET — The alt names what the user will reach (the entity, page, or action). Naming the entity (e.g., "Astronaut Ellen Ochoa" for a link to her bio) qualifies; the alt does NOT need to use the literal word "link" or "page". + G. NOT-MEDIUM-ANNOUNCING — The alt does not announce the destination's medium or technology ("Wikipedia entry for …", "PDF of …", "Click here to read about …", "Link to …"). Screen readers already announce the link role. + H. NOT-GENERIC — The alt is not empty and not a placeholder like "Read More", "Image", "Click here". + +If any criterion fails → verdict = "needs-fix". Otherwise → verdict = "ok". + +## Step 4 — Informative (default) +The image conveys information that contributes meaning to the page. Apply criteria A–E below. The alt is "ok" if and only if ALL hold: + + A. ACCURATE — The alt describes what the image actually shows in this context. + B. CONTEXTUALLY-COMPLETE — The alt captures the content the page relies on the image to convey, given the surrounding text. If the surrounding text already covers some of that content, the alt may be brief. + C. NOT-DUPLICATIVE — The alt does not repeat sentences from nearby text verbatim. + D. NOT-INTRUSIVE — The alt does not introduce new claims absent from the page (no extra trivia, no marketing copy). + E. APPROPRIATELY-FRAMED — Vocabulary, register, and detail level match the audience and genre indicated by the context. + +If any criterion fails → verdict = "needs-fix". Otherwise → verdict = "ok". + +# Operational rules + +R1. **Trust the framing.** The provided context describes the page's audience and genre. If the context warrants longer description (educational pages applying "general-to-specific", art-history pages, textbook questions referring to specific image elements), longer alt is appropriate. If the context targets young children or is a labeled caption, short alt is appropriate. Do not penalize length when the context warrants it. + +R1a. **Surrounding analysis is not a detail requirement.** When the surrounding text *analyzes or discusses properties of the image* (e.g., an art-history paragraph dissecting a painting's composition, light, color, perspective), this does NOT mean the alt must reproduce that analysis. The alt names what the image is; the surrounding text supplies the analysis. A short, accurate alt is correct even when the surrounding paragraph is long and analytical — in fact, a longer alt would duplicate the surrounding analysis (failing C). + +R2. **Distinguish redundant prefixes from semantic prefixes.** + - REDUNDANT (penalize): "Image of …", "Picture of …", "Graphic of …" — these add no information. + - SEMANTIC (do NOT penalize): "Photograph of …", "Painting of …", "Drawing of …", "Diagram of …", "Map of …", "Chart of …", "Screenshot of …", "Icon of …" — these communicate the medium or genre, which is information. + +R3. **Labeled diagrams.** When the image is a diagram whose content is the labels it carries (anatomical labels, mitosis stages, taxonomic charts, flow-chart steps), naming the labels IS the correct alt content. Do not demand visual descriptions of how each labeled region looks. + +R4. **An empty alt ("") is the correct answer when (and only when) the image matches Step 1 or Step 2.** When the alt is already empty in such a case, return verdict = "ok" — no action is needed. Use "decorative" only when the current alt is non-empty and should be removed. + +R5. **Do not overthink.** Walk the decision procedure once, in order. Do not invent context that wasn't provided. Do not bring in general knowledge about what "good" alt text usually looks like beyond the rules above. Pick the first matching step and stop. + +# Output + +Return a single JSON object with EXACTLY these fields, in this order: step, reasoning, verdict, issue, confidence. + +# Examples + + +The image is a horizontal separator graphic placed between two sections. The structural separation is conveyed by the page's headings and layout. + +{"step":1,"reasoning":"Step 1 matches: the image is a purely decorative separator. The current alt is empty, which is correct, so no action is needed.","verdict":"ok","issue":"","confidence":0.95} + + + +Adjacent caption immediately above the image: "Ellen Ochoa, Astronaut". Body text below names her achievements. + +{"step":2,"reasoning":"Step 2 matches: the image's content is fully conveyed by the adjacent caption. The current alt is empty, which is correct per R4, so no action is needed.","verdict":"ok","issue":"","confidence":0.95} + + + +Book about national parks. Photograph of the Grand Canyon during monsoon season, surrounding text describes a violent storm. +A photograph of a blue sky peeking through grey storm clouds over the Grand Canyon in early September. +{"step":4,"reasoning":"Step 4 matches and all of A–E hold. Per R2, 'A photograph of' is a semantic prefix communicating medium and is not penalized.","verdict":"ok","issue":"","confidence":0.9} + + + +Biology textbook chapter-summary question: which of the following is not a main stage of mitosis? The image is a hand-drawn diagram with each stage labeled. +Mitosis stages: Prophase, Prometaphase, Metaphase, Anaphase, and Telophase +{"step":4,"reasoning":"Step 4 matches and all of A–E hold. Per R3, naming the diagram's labels is the correct alt content for a labeled diagram tied to a textbook question about those labels.","verdict":"ok","issue":"","confidence":0.9} + + + +Geology page about acid erosion of rock formations; bird droppings are cited as a contributor. +A close-up of a puffin with bright orange feet and a colorful beak. +{"step":4,"reasoning":"Step 4 matches but criterion B fails: the alt omits the contextually relevant content (the bird defecating, which is what makes the image relevant to acid erosion).","verdict":"needs-fix","issue":"missing-context-relevant-content","confidence":0.9} + + + +Image is the only content inside a link to an Ellen Ochoa Wikipedia page. +Read More +{"step":3,"reasoning":"Step 3 matches: the image is the only content of a link, and 'Read More' does not describe the destination.","verdict":"needs-fix","issue":"missing-link-name","confidence":0.95} + + + +Body text near the image: "As the first Hispanic woman to go to space, Ellen Ochoa is widely regarded as a role model." The image is not inside a link. The body text names her but does not state she is an astronaut; the uniform in the image conveys that. +Image of Ellen Ochoa, Astronaut +{"step":4,"reasoning":"Step 2 does not apply because the surrounding text is body prose, not a caption or label. Step 4 applies and criterion E fails: 'Image of' is a redundant prefix per R2 and should be removed; the rest of the alt is otherwise fine.","verdict":"needs-fix","issue":"redundant-prefix","confidence":0.9} + + + +The image is the only content inside a link pointing to https://en.wikipedia.org/wiki/Ellen_Ochoa. The link's accessible name comes entirely from the image's alt text. +Astronaut Ellen Ochoa +{"step":3,"reasoning":"Step 3 matches and all of F–H hold. The alt names the entity at the destination ('Astronaut Ellen Ochoa'), does not announce the medium, and is not generic. Per F, naming the entity is sufficient — the alt does not need to say 'link' or 'page'.","verdict":"ok","issue":"","confidence":0.9} + + + +The image is the only content inside a link pointing to https://en.wikipedia.org/wiki/Ellen_Ochoa. The link's accessible name comes entirely from the image's alt text. +Wikipedia entry for Ellen Ochoa, Astronaut +{"step":3,"reasoning":"Step 3 matches but criterion G fails: 'Wikipedia entry for' announces the destination's medium, which screen readers already convey through the link role. Drop the medium announcement.","verdict":"needs-fix","issue":"medium-announcing","confidence":0.9} + + + +An art-history textbook chapter on artistic technique. The surrounding paragraph analyzes the painting's use of light, color, form, perspective, proportion, and motion to depict the iconic crossing. +Painting of George Washington crossing the Delaware River +{"step":4,"reasoning":"Step 4 matches and all of A–E hold. Per R1a, the surrounding paragraph's analysis of light/color/form is not a requirement that the alt reproduce that analysis; the alt names what the image is, and the surrounding text supplies the artistic analysis. Per R2, 'Painting of' is a semantic prefix.","verdict":"ok","issue":"","confidence":0.9} +` + +// JSON Schema for Structured Outputs strict mode. Field order in `properties` +// is the generation order; reasoning is generated before verdict to force +// chain-of-thought before classification. +export const VERDICT_SCHEMA = { + name: 'alt_text_verdict', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + properties: { + step: {type: 'integer', enum: [1, 2, 3, 4]}, + reasoning: {type: 'string'}, + verdict: {type: 'string', enum: ['ok', 'needs-fix', 'decorative']}, + issue: {type: 'string'}, + confidence: {type: 'number', minimum: 0, maximum: 1}, + }, + required: ['step', 'reasoning', 'verdict', 'issue', 'confidence'], + }, +} as const diff --git a/src/judges/types.ts b/src/judges/types.ts new file mode 100644 index 0000000..8b28318 --- /dev/null +++ b/src/judges/types.ts @@ -0,0 +1,33 @@ +// Shared contract for any backend that judges alt-text quality. +// +// A "judge" takes one image plus its alt text and surrounding context and +// returns a structured verdict. Implementations may call any combination of +// vision models, OCR services, or rule engines, but they must all return the +// same JudgeVerdict shape so the rule layer never branches on backend. + +export type Verdict = 'ok' | 'needs-fix' | 'decorative' + +// Output the model is forced to produce (matches VERDICT_SCHEMA in prompt.ts). +// Field order is the generation order; reasoning precedes verdict so the model +// thinks before it classifies. +export type JudgeVerdict = { + step: 1 | 2 | 3 | 4 + reasoning: string + verdict: Verdict + issue: string + confidence: number +} + +// What the rule (or probe) hands to a judge. The image is pre-loaded as a +// data URL so the judge does not have to know about file paths or HTTP. +export type JudgeInput = { + imageDataUrl: string + alt: string + context: string +} + +export interface JudgeAltText { + judge(input: JudgeInput): Promise +} + +export type JudgeMode = 'copilot' | 'azure-augmented' diff --git a/src/utils/load-image-data-url.ts b/src/utils/load-image-data-url.ts new file mode 100644 index 0000000..cf8c04f --- /dev/null +++ b/src/utils/load-image-data-url.ts @@ -0,0 +1,47 @@ +// Loads an image from any of: an http(s) URL, an absolute filesystem path, +// or a path relative to a given base directory. Returns the bytes encoded as +// a data URL suitable for vision-model `image_url` fields. +// +// Used by the offline probe (where images come from cases.json paths or URLs) +// and by the production rule (where images come from the page's +// resolved against the page URL). + +import {readFile} from 'node:fs/promises' +import {extname, isAbsolute, resolve} from 'node:path' + +const MIME_BY_EXT: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.bmp': 'image/bmp', + '.avif': 'image/avif', +} + +function guessMime(p: string): string { + return MIME_BY_EXT[extname(p).toLowerCase()] ?? 'application/octet-stream' +} + +export type LoadImageOptions = { + // Used to resolve relative filesystem paths. Ignored for URLs. + baseDir?: string +} + +export async function loadImageAsDataUrl(imageRef: string, options: LoadImageOptions = {}): Promise { + if (/^data:/i.test(imageRef)) return imageRef + + if (/^https?:\/\//i.test(imageRef)) { + const res = await fetch(imageRef) + if (!res.ok) throw new Error(`failed to fetch ${imageRef}: ${res.status} ${res.statusText}`) + const buf = Buffer.from(await res.arrayBuffer()) + const mime = res.headers.get('content-type')?.split(';')[0]?.trim() || guessMime(imageRef) + return `data:${mime};base64,${buf.toString('base64')}` + } + + const baseDir = options.baseDir ?? process.cwd() + const abs = isAbsolute(imageRef) ? imageRef : resolve(baseDir, imageRef) + const buf = await readFile(abs) + return `data:${guessMime(abs)};base64,${buf.toString('base64')}` +} From 27e00a0d9571697f826d74d687def1e81f059dd3 Mon Sep 17 00:00:00 2001 From: Keenan Zhou Date: Wed, 17 Jun 2026 11:16:49 -0700 Subject: [PATCH 03/35] Extend image extraction with structured context fields --- index.ts | 3 +- src/extract.ts | 84 ++++++++++++++++++++++++++++++++++++++------------ src/types.ts | 25 +++++++++++++-- 3 files changed, 89 insertions(+), 23 deletions(-) diff --git a/index.ts b/index.ts index fac182a..390d7da 100644 --- a/index.ts +++ b/index.ts @@ -35,7 +35,8 @@ export default async function altTextScan({page, addFinding}: PluginArgs): Promi for (const rule of enabledRules) { let results try { - results = rule.evaluate(ctx) + // Rules may be sync or async; await both shapes uniformly. + results = await rule.evaluate(ctx) } catch (err) { console.error(`[alt-text-scan] rule "${rule.id}" threw on ${url}:`, err) continue diff --git a/src/extract.ts b/src/extract.ts index b2f9880..9449553 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -1,27 +1,73 @@ import type {Page} from 'playwright' import type {ImageRecord} from './types.js' +// Maximum number of characters of nearby prose forwarded to model-backed rules. +// Long enough to convey context, short enough to keep prompts cheap and focused. +const NEARBY_TEXT_MAX = 600 + // Returns one ImageRecord per HTML element that is exposed in the accessibility tree. // Using getByRole('img') filters out elements that assistive tech cannot perceive. export async function extractImages(page: Page): Promise { - return page.getByRole('img').evaluateAll(els => - els - // getByRole('img') also matches SVG/div with role="img", so filter those out. - .filter(el => el.tagName === 'IMG') - .map(el => { - const rect = el.getBoundingClientRect() - const boundingBox = - rect.width === 0 && rect.height === 0 ? null : {x: rect.x, y: rect.y, width: rect.width, height: rect.height} - return { - src: el.getAttribute('src'), - alt: el.getAttribute('alt'), - role: el.getAttribute('role'), - ariaHidden: el.getAttribute('aria-hidden') === 'true', - ariaLabel: el.getAttribute('aria-label'), - ariaLabelledBy: el.getAttribute('aria-labelledby'), - outerHTML: el.outerHTML, - boundingBox, - } - }), + return page.getByRole('img').evaluateAll( + (els, maxNearby) => + els + // getByRole('img') also matches SVG/div with role="img", so filter those out. + .filter(el => el.tagName === 'IMG') + .map(el => { + const rect = el.getBoundingClientRect() + const boundingBox = + rect.width === 0 && rect.height === 0 + ? null + : {x: rect.x, y: rect.y, width: rect.width, height: rect.height} + + // Closest ancestor link with an href. + const linkEl = el.closest('a[href]') as HTMLAnchorElement | null + const inLink = linkEl ? {href: linkEl.getAttribute('href') ?? ''} : null + + // Closest ancestor button — either a