diff --git a/.gitignore b/.gitignore index 538786a4..76f55900 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ # testing /coverage +/test-results/ +/playwright-report/ # next.js /.next/ diff --git a/docs/parallel-review-performance.md b/docs/parallel-review-performance.md new file mode 100644 index 00000000..9f8f3f12 --- /dev/null +++ b/docs/parallel-review-performance.md @@ -0,0 +1,129 @@ +# Parallel Maia analysis in reviews + +## Why this exists + +Each review position requests Maia predictions for 21 rating levels. The +existing `Maia` engine sent those 21 inputs as one ONNX batch to one dedicated +Web Worker. Splitting the batch across a small pool lets independent ONNX +sessions use more CPU cores. Results are reassembled in the original rating +order, so move probabilities and win estimates keep their existing meaning. + +Dedicated **Web Workers** are used here. A Service Worker handles fetch and +offline lifecycle events; it is not the right owner for long-running model +inference. The project already runs Maia inference in a Web Worker, so the +pool extends the current design instead of introducing a new execution model. +See [MDN on dedicated Web Workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) +and [MDN on Service Workers](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API/Using_Service_Workers). + +The review Options panel contains an Advanced performance settings section. +The drill selection screen and live drill controls expose the same worker +control before a drill review starts. The Stockfish drill review choice keeps +depth 18 by default or permits a faster depth 12 review; shallower searches can +change move classifications. The depth 12 option was not included in the Maia +inference benchmark below. +Parallel analysis can be switched off, and the count can be set from 1 to the +device limit. The default is two workers when at least three logical cores are +reported, otherwise one. At most four workers are allowed, with one reported +core reserved for the UI and Stockfish. If core information is unavailable, +the fallback limit is two. Additional workers are created only when an analysis +batch needs them. Each worker holds a separate copy of the 44 MB model and an +ONNX session, so raising the count increases memory use. If a secondary worker +fails, its batch chunk is retried on the primary worker. If a secondary session +does not start within five seconds, analysis proceeds with the workers already +ready. Disabling parallel +analysis keeps the original single-worker path. + +Match moves still come from the `play/get_move` backend endpoint. Browser +workers cannot shorten that server request; this setting applies to local +review and drill inference only. + +Stockfish loads two NNUE weight files separately from the Maia worker pool. +On the local check, the larger file was about 75 MB and took 141 seconds to +download from the remote host. The weights are cached in IndexedDB after +download. The loading toast now shows whether it is checking the cache, +downloading weights, loading weights, or starting the engine. The Maia worker +count does not shorten this initial download or Stockfish search. + +## Project fit and checks + +This follows the project's existing React context and `Maia` engine structure, +keeps the worker script under `public/`, and uses the existing IndexedDB model +cache. No server endpoint or chess result format changes. The code is TypeScript +checked and linted. Jest tests cover device limits, persisted settings, +parallel result ordering, single-worker equivalence, failed secondary retry, +and malformed batch inputs. The Playwright browser test loads a review, uses +the real model, checks that two workers receive inference requests, and checks +setting persistence and the device cap. + +ONNX Runtime may also use WASM threads within a worker on cross-origin-isolated +pages. The worker count is deliberately capped because more sessions can +increase both CPU contention and memory use; see [ONNX Runtime's threading +guidance](https://onnxruntime.ai/docs/tutorials/web/performance-diagnosis.html). + +Run locally: + +```bash +npm test -- --runInBand +npm run test:browser +npx tsc --noEmit +``` + +Install Chromium with `npx playwright install chromium` if it is not already +available. To use an existing Chrome installation, set +`PLAYWRIGHT_CHROME_EXECUTABLE=/path/to/chrome`. + +## Performance report + +Correction: an earlier draft used 21 copies of rating 1500 and empty board +tensors. Those inputs do not represent a review, so its timings were discarded. +The measurements below replace that comparison. + +Re-measured on 2026-10-04 in headless Chrome 153 on Linux, with 32 logical cores +reported and cross-origin isolation enabled. The benchmark uses the actual +Maia-3 ONNX model, ten legal positions from a Ruy Lopez opening, and the same +21 rating inputs per position as review: 600 through 2600 in steps of 100. +Black-to-move positions are mirrored and piece channels are encoded as in the +review preprocessor. It times warm inference only; board preprocessing, +Stockfish, rendering, and model initialization are excluded. All four workers +are warmed before timing, and the worker-count order rotates between three +runs. The median is shown. Reproduce the full samples with +`node scripts/benchmark-maia-workers.mjs` while the development server is +running on port 3107. + +To inspect the feature locally, run `npm run dev -- --hostname 0.0.0.0 --port +3107` and open `http://localhost:3107`. In a completed review, expand +**Options → Advanced performance settings** to switch parallel Maia analysis +on or off and choose a worker count. + +Local verification on 2026-10-04: `/drills` returned HTTP 200 from the local +development server; all 10 Jest tests, the Playwright review-worker browser +test (1/1), and `tsc --noEmit` passed using the installed Chrome executable. + +| Review workload | 1 worker | 2 workers | 4 workers | 2 worker gain | 4 worker gain | +| --- | ---: | ---: | ---: | ---: | ---: | +| One position, 21 ratings | 483 ms | 315 ms | 222 ms | 1.53× | 2.18× | +| Five positions, 105 ratings | 2,340 ms | 1,402 ms | 1,154 ms | 1.67× | 2.03× | +| Ten positions, 210 ratings | 4,745 ms | 2,610 ms | 2,334 ms | 1.82× | 2.03× | + +Raw samples in table order: one worker `[489, 483, 463]`, +`[2317, 2371, 2340]`, `[4603, 4856, 4745]`; two workers +`[312, 316, 315]`, `[1402, 1617, 1348]`, `[2607, 2835, 2610]`; +four workers `[223, 219, 222]`, `[1128, 1171, 1154]`, +`[2334, 2327, 2804]` milliseconds. A preceding run on the same day measured +488/353/235 ms for one position, 2,424/1,621/1,243 ms for five positions, +and 5,000/3,443/2,858 ms for ten positions (one/two/four workers). These are +single-device observations, not guaranteed gains across devices or complete +reviews. + +The first worker reached ready status in 2,235 ms in this run. That includes +loading the model from the local browser cache or development server; extra +worker initialization is excluded from the warm timings. Whole-review wall +time can improve less because Stockfish search, opening-book requests, React +rendering, and model loading are not accelerated by this pool. Low-core and +memory-constrained devices may run better with one worker; the setting allows +that choice. Match performance is intentionally outside this report because +matches use the backend move service. + +For all ten sampled positions and all 21 rating levels, the maximum difference +between the one-worker and two- or four-worker move and value logits was **0**. +The worker split changed throughput in this run, not the Maia predictions. diff --git a/package-lock.json b/package-lock.json index 9e3249ce..b4922998 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ }, "devDependencies": { "@babel/core": "^7.16.7", + "@playwright/test": "^1.63.0", "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^16.3.0", @@ -820,6 +821,18 @@ "node": ">=18" } }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/runtime": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", @@ -830,6 +843,17 @@ "tslib": "^2.4.0" } }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", @@ -1988,6 +2012,19 @@ "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", "license": "MIT" }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", + "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.10.0" + } + }, "node_modules/@next/env": { "version": "15.5.7", "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.7.tgz", @@ -2216,6 +2253,26 @@ "@parcel/watcher-win32-x64": "2.5.1" } }, + "node_modules/@parcel/watcher-android-arm64": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.1.tgz", + "integrity": "sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/@parcel/watcher-darwin-arm64": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.1.tgz", @@ -2236,6 +2293,244 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/@parcel/watcher-darwin-x64": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.1.tgz", + "integrity": "sha512-1ZXDthrnNmwv10A0/3AJNZ9JGlzrF82i3gNQcWOzd7nJ8aj+ILyW1MTxVk35Db0u91oD5Nlk9MBiujMlwmeXZg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-freebsd-x64": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.1.tgz", + "integrity": "sha512-SI4eljM7Flp9yPuKi8W0ird8TI/JK6CSxju3NojVI6BjHsTyK7zxA9urjVjEKJ5MBYC+bLmMcbAWlZ+rFkLpJQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-glibc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.1.tgz", + "integrity": "sha512-RCdZlEyTs8geyBkkcnPWvtXLY44BCeZKmGYRtSgtwwnHR4dxfHRG3gR99XdMEdQ7KeiDdasJwwvNSF5jKtDwdA==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-musl": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.1.tgz", + "integrity": "sha512-6E+m/Mm1t1yhB8X412stiKFG3XykmgdIOqhjWj+VL8oHkKABfu/gjFj8DvLrYVHSBNC+/u5PeNrujiSQ1zwd1Q==", + "cpu": [ + "arm" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-glibc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.1.tgz", + "integrity": "sha512-LrGp+f02yU3BN9A+DGuY3v3bmnFUggAITBGriZHUREfNEzZh/GO06FF5u2kx8x+GBEUYfyTGamol4j3m9ANe8w==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-musl": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.1.tgz", + "integrity": "sha512-cFOjABi92pMYRXS7AcQv9/M1YuKRw8SZniCDw0ssQb/noPkRzA+HBDkwmyOJYp5wXcsTrhxO0zq1U11cK9jsFg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-glibc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.1.tgz", + "integrity": "sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-musl": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.1.tgz", + "integrity": "sha512-n0E2EQbatQ3bXhcH2D1XIAANAcTZkQICBPVaxMeaCVBtOpBZpWJuf7LwyWPSBDITb7In8mqQgJ7gH8CILCURXg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-arm64": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.1.tgz", + "integrity": "sha512-RFzklRvmc3PkjKjry3hLF9wD7ppR4AKcWNzH7kXR7GUe0Igb3Nz8fyPwtZCSquGrhU5HhUNDr/mKBqj7tqA2Vw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-ia32": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.1.tgz", + "integrity": "sha512-c2KkcVN+NJmuA7CGlaGD1qJh1cLfDnQsHjE89E60vUEMlqduHGCdCLJCID5geFVM0dOtA3ZiIO8BoEQmzQVfpQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-x64": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.1.tgz", + "integrity": "sha512-9lHBdJITeNR++EvSQVUcaZoWupyHfXe1jZvGZ06O/5MflPcuPLtEphScIBL+AiCWBO46tDSHzWyD0uDmmZqsgA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -2260,6 +2555,22 @@ "url": "https://opencollective.com/pkgr" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@popperjs/core": { "version": "2.11.8", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", @@ -2527,6 +2838,17 @@ "@testing-library/dom": ">=7.21.4" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -3186,6 +3508,34 @@ "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", "license": "ISC" }, + "node_modules/@unrs/resolver-binding-android-arm-eabi": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", + "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@unrs/resolver-binding-android-arm64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", + "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, "node_modules/@unrs/resolver-binding-darwin-arm64": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", @@ -3200,6 +3550,257 @@ "darwin" ] }, + "node_modules/@unrs/resolver-binding-darwin-x64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", + "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@unrs/resolver-binding-freebsd-x64": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", + "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", + "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", + "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", + "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-arm64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", + "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", + "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", + "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", + "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", + "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-gnu": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", + "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-x64-musl": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", + "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", + "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^0.2.11" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", + "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", + "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@unrs/resolver-binding-win32-x64-msvc": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", + "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@vercel/analytics": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.5.0.tgz", @@ -10539,6 +11140,35 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "license": "MIT" }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/popper.js": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", diff --git a/package.json b/package.json index 8aad1c4d..a139e244 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "start": "next start", "export": "next export", "lint": "eslint . --ext ts,tsx --fix", - "test": "jest" + "test": "jest", + "test:browser": "playwright test" }, "dependencies": { "@babel/preset-react": "^7.16.7", @@ -40,6 +41,7 @@ }, "devDependencies": { "@babel/core": "^7.16.7", + "@playwright/test": "^1.63.0", "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^16.3.0", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000..3e215283 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: './tests/browser', + timeout: 90_000, + use: { + baseURL: 'http://127.0.0.1:3107', + browserName: 'chromium', + launchOptions: { + executablePath: process.env.PLAYWRIGHT_CHROME_EXECUTABLE, + }, + }, + webServer: { + command: 'npm run dev -- --port 3107', + url: 'http://127.0.0.1:3107', + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, +}) diff --git a/scripts/benchmark-maia-workers.mjs b/scripts/benchmark-maia-workers.mjs new file mode 100644 index 00000000..fc656495 --- /dev/null +++ b/scripts/benchmark-maia-workers.mjs @@ -0,0 +1,227 @@ +import { chromium } from '@playwright/test' +import { Chess } from 'chess.ts' + +const baseURL = process.env.MAIA_BENCHMARK_URL || 'http://127.0.0.1:3107' +const sampleMoves = [ + 'e4', + 'e5', + 'Nf3', + 'Nc6', + 'Bb5', + 'a6', + 'Ba4', + 'Nf6', + 'O-O', +] +const sampleGame = new Chess() +const reviewFens = [sampleGame.fen()] +for (const move of sampleMoves) { + sampleGame.move(move) + reviewFens.push(sampleGame.fen()) +} +const browser = await chromium.launch({ + headless: true, + executablePath: process.env.PLAYWRIGHT_CHROME_EXECUTABLE, +}) + +try { + const page = await browser.newPage() + await page.goto(baseURL) + const report = await page.evaluate(async (fens) => { + const modelUrl = '/maia3/maia3_simplified.onnx' + const ratings = Array.from({ length: 21 }, (_, i) => 600 + i * 100) + const pieceTypes = 'PNBRQKpnbrqk' + const tokenize = (fen) => { + const [placement, turn] = fen.split(' ') + const rows = placement.split('/') + const canonicalRows = + turn === 'b' + ? rows.reverse().map((row) => + row.replace(/[a-zA-Z]/g, (piece) => + piece === piece.toUpperCase() + ? piece.toLowerCase() + : piece.toUpperCase(), + ), + ) + : rows + const tokens = new Float32Array(64 * 12) + canonicalRows.forEach((row, rank) => { + let file = 0 + for (const symbol of row) { + if (/\d/.test(symbol)) { + file += Number(symbol) + } else { + tokens[((7 - rank) * 8 + file) * 12 + pieceTypes.indexOf(symbol)] = 1 + file++ + } + } + }) + return tokens + } + const positions = fens.map(tokenize) + const workers = [] + let nextId = 1 + + const waitFor = (worker, predicate) => + new Promise((resolve, reject) => { + const onMessage = (event) => { + if (event.data.type === 'error') { + worker.removeEventListener('message', onMessage) + reject(new Error(event.data.message)) + } else if (predicate(event.data)) { + worker.removeEventListener('message', onMessage) + resolve(event.data) + } + } + worker.addEventListener('message', onMessage) + }) + + const addWorker = async () => { + const worker = new Worker('/maia-worker.js') + const initialized = waitFor( + worker, + (message) => + message.type === 'status' && + (message.status === 'ready' || message.status === 'no-cache'), + ) + worker.postMessage({ type: 'init', modelUrl, modelVersion: '3' }) + const status = await initialized + if (status.status === 'no-cache') { + const ready = waitFor( + worker, + (message) => message.type === 'status' && message.status === 'ready', + ) + worker.postMessage({ type: 'download' }) + await ready + } + workers.push(worker) + return worker + } + + const infer = (worker, position, chunkRatings) => { + const size = chunkRatings.length + const id = nextId++ + const result = waitFor( + worker, + (message) => message.type === 'inference-result' && message.id === id, + ) + const tokens = new Float32Array(size * position.length) + for (let i = 0; i < size; i++) { + tokens.set(position, i * position.length) + } + worker.postMessage({ + type: 'inference', + id, + tokens: tokens.buffer, + eloSelfs: Float32Array.from(chunkRatings).buffer, + eloOppos: Float32Array.from(chunkRatings).buffer, + batchSize: size, + }) + return result + } + + const runPosition = async (workerCount, position) => { + const chunkSize = Math.ceil(ratings.length / workerCount) + const chunks = await Promise.all( + workers.slice(0, workerCount).map(async (worker, index) => { + const offset = index * chunkSize + const chunkRatings = ratings.slice(offset, offset + chunkSize) + if (chunkRatings.length === 0) return null + return { offset, result: await infer(worker, position, chunkRatings) } + }), + ) + const logitsMove = new Float32Array(ratings.length * 4352) + const logitsValue = new Float32Array(ratings.length * 3) + for (const chunk of chunks) { + if (!chunk) continue + logitsMove.set(new Float32Array(chunk.result.logitsMove), chunk.offset * 4352) + logitsValue.set(new Float32Array(chunk.result.logitsValue), chunk.offset * 3) + } + return { logitsMove, logitsValue } + } + + const benchmark = async (workerCount, positionCount) => { + const started = performance.now() + for (let index = 0; index < positionCount; index++) { + await runPosition(workerCount, positions[index]) + } + return Math.round(performance.now() - started) + } + + const maxDifference = (left, right) => { + let largest = 0 + for (let i = 0; i < left.length; i++) { + largest = Math.max(largest, Math.abs(left[i] - right[i])) + } + return largest + } + + const coldStart = performance.now() + await addWorker() + const firstWorkerReadyMs = Math.round(performance.now() - coldStart) + while (workers.length < 4) await addWorker() + await Promise.all( + workers.map((worker) => infer(worker, positions[0], [1500])), + ) + const baseline = [] + for (const position of positions) { + baseline.push(await runPosition(1, position)) + } + const outputComparisons = [] + for (const count of [1, 2, 4]) { + let maxMoveLogitDifference = 0 + let maxValueLogitDifference = 0 + for (let index = 0; index < positions.length; index++) { + const output = await runPosition(count, positions[index]) + maxMoveLogitDifference = Math.max( + maxMoveLogitDifference, + maxDifference(baseline[index].logitsMove, output.logitsMove), + ) + maxValueLogitDifference = Math.max( + maxValueLogitDifference, + maxDifference(baseline[index].logitsValue, output.logitsValue), + ) + } + outputComparisons.push({ + workers: count, + maxMoveLogitDifference, + maxValueLogitDifference, + }) + } + const rows = [] + const orders = [ + [1, 2, 4], + [4, 2, 1], + [2, 1, 4], + ] + for (const positionCount of [1, 5, 10]) { + const samples = new Map([[1, []], [2, []], [4, []]]) + for (const order of orders) { + for (const count of order) { + samples.get(count).push(await benchmark(count, positionCount)) + } + } + for (const count of [1, 2, 4]) { + rows.push({ + workers: count, + positions: positionCount, + samplesMs: samples.get(count), + }) + } + } + workers.forEach((worker) => worker.terminate()) + return { + userAgent: navigator.userAgent, + logicalCores: navigator.hardwareConcurrency, + crossOriginIsolated: self.crossOriginIsolated, + ratingLevels: ratings, + positionCount: positions.length, + outputComparisons, + firstWorkerReadyMs, + rows, + } + }, reviewFens) + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`) +} finally { + await browser.close() +} diff --git a/src/components/Analysis/ConfigureAnalysis.tsx b/src/components/Analysis/ConfigureAnalysis.tsx index 13d67586..f2a869d8 100644 --- a/src/components/Analysis/ConfigureAnalysis.tsx +++ b/src/components/Analysis/ConfigureAnalysis.tsx @@ -2,6 +2,7 @@ import React from 'react' import { AnalyzedGame } from 'src/types' import { ContinueAgainstMaia } from 'src/components' +import { MaiaWorkerSettings } from 'src/components/Common/MaiaWorkerSettings' interface Props { currentMaiaModel: string @@ -59,6 +60,9 @@ export const ConfigureAnalysis: React.FC = ({ +
+ +
{onAnalyzeEntireGame && ( + {open && ( +
+ + + change(settings.enabled, Number(e.target.value))} + className="w-full accent-human-4 disabled:opacity-50" + /> +

+ Up to {maxWorkers} on this device. Used for local analysis in game + reviews and drills. This controls Maia only; Stockfish loads and + searches separately. +

+ {showDrillDepth && ( +
+ + +

+ Faster review searches less deeply and may change move ratings. + Model loading time is unaffected. +

+
+ )} +
+ )} + + ) +} diff --git a/src/components/Openings/OpeningSelectionModal.tsx b/src/components/Openings/OpeningSelectionModal.tsx index d3b213cd..c407b254 100644 --- a/src/components/Openings/OpeningSelectionModal.tsx +++ b/src/components/Openings/OpeningSelectionModal.tsx @@ -19,6 +19,7 @@ import { EndgamePositionDetail, } from 'src/types' import { ModalContainer } from '../Common/ModalContainer' +import { MaiaWorkerSettings } from '../Common/MaiaWorkerSettings' import { useTour } from 'src/contexts' import { tourConfigs } from 'src/constants/tours' import { WindowSizeContext } from 'src/contexts/WindowSizeContext' @@ -1415,6 +1416,8 @@ const DrillStudioPanel: React.FC<{ )} + + {/* Add Drill button */}