diff --git a/.dependency-cruiser.mjs b/.dependency-cruiser.mjs index be18293b..3b4f611a 100644 --- a/.dependency-cruiser.mjs +++ b/.dependency-cruiser.mjs @@ -103,8 +103,8 @@ export default { ), direction( "no-fs-or-process", - "core, leasing, agent-device and host-load reach the filesystem and processes only through injected ports", - "^src/(core|leasing|agent-device|host-load)/", + "core, leasing, agent-device, host-load and stream reach the filesystem and processes only through injected ports", + "^src/(core|leasing|agent-device|host-load|stream)/", { dependencyTypes: ["core"], path: fsOrProcess }, ), direction( @@ -113,6 +113,18 @@ export default { "^src/(core|leasing|drivers)/", { path: "^src/agent-device/" }, ), + direction( + "not-stream", + "core, leasing, the drivers, the gateway and the contract import nothing from the stream module", + "^src/(core|leasing|drivers|gateway|contract)/", + { path: "^src/stream/" }, + ), + direction( + "stream-not-edges", + "the stream module is a module the host loads: it imports no frontend, gateway, driver or leasing", + "^src/stream/", + { path: "^src/(http|gateway|drivers|daemon|cli|mcp|leasing)/" }, + ), direction( "host-load-imports-ports-only", "host-load reads the machine through the HostLoadProbe port and imports no other component", diff --git a/.fallowrc.json b/.fallowrc.json index 0baed307..147c7777 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -12,6 +12,8 @@ // them (budget-guard, comment-lint) import from their neighbours (usage, comments). // .dependency-cruiser.cycles.mjs is named only by src/lint-rules.test.ts, which runs depcruise // with it (ADR 0027). + // vendor/screen-capture/scripts/unframe.mjs is a diagnostic a person runs by hand against a + // captured file, the way the .agents scripts are. "entry": [ "src/cli/main.ts", "src/daemon/main.ts", @@ -20,7 +22,8 @@ "e2e/console/*.spec.ts", "vitest.mutate.config.ts", ".agents/scripts/*.mjs", - ".dependency-cruiser.cycles.mjs" + ".dependency-cruiser.cycles.mjs", + "vendor/screen-capture/scripts/*.mjs" ], "dynamicallyLoaded": ["e2e/fake-driver/index.ts"], // The web console's libraries are bundled into dist/ui at build time and ship as files, so diff --git a/AGENTS.md b/AGENTS.md index 84792d50..7775f518 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,6 +46,11 @@ changes in this repo: agents run and the checks they leave to hooks and CI; the slow lane (real devices); project-only checks. Agents and skills take every command from here. +- [vendor-and-third-party-code.md](docs/internal/agent-rules/vendor-and-third-party-code.md) + — third-party code lives under `vendor/`, one directory per upstream, each + with its license text and a `NOTICE.md`; source vendoring records the commit, + the deviations and how to diff against upstream; binaries are vendored + byte-identical with a checksum; nothing is fetched at runtime. - [always-in-scope.md](docs/internal/agent-rules/always-in-scope.md) — what every change includes without its spec asking (docs and strings it makes false, both `EVENTS.md` files, a test for every new path, Fallow entries), @@ -96,4 +101,6 @@ Maintainer/agent docs live under [docs/internal/](docs/internal/): spec session writes, and the module manual template - [EVENTS.md](docs/internal/EVENTS.md) — the same catalog with rationale and ADR references - [IDEAS.md](docs/internal/IDEAS.md) — post-v1 ideas; don't implement these unless asked +- [VENDORED-CODE.md](docs/internal/VENDORED-CODE.md) — every third-party artifact in the + repo: upstream, version, license, where it lives, who consumes it, how to re-verify - [adr/](docs/internal/adr/) — architecture decision records and their status diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ec22074b..be27dc77 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -39,6 +39,10 @@ edit yourself keeps the mode you give it until the next `config set`; run | `capabilities.agent-device.enabled` | **Both sides.** On a worker: run one agent-device daemon on this worker, started and watched by Simlock. It sees Simlock's devices and no others, refuses `boot` and `shutdown`, and keeps its files in `~/.simlock/agent-device`. If agent-device is missing, older than 0.21.28, speaks another RPC protocol or needs a newer Node than Simlock runs on, it is not started and Simlock runs on without it. A daemon a killed Simlock left behind is stopped at the next start with `capabilities.agent-device.enabled` `true`; with it off, Simlock does not look for one. With `http.enabled` also `true`, the agent-device CLI reaches it at `/v1/agent-device` (see [HTTP-API.md](HTTP-API.md#v1agent-device)). On a gateway: serve `/v1/agent-device` and send each agent to a worker whose agent-device is available; the gateway runs no daemon of its own. | `false` | | `capabilities.agent-device.leaseTtlCapMs` | **Both sides.** Caps the TTL of the leases the agent-device CLI makes and renews through Simlock: a longer `ttlMs` is lowered to it, and a lease that sends none gets it. A whole number of milliseconds from `5000` to `lease.maxTtlMs`; a value outside that range stops the daemon at start, on a gateway too. Used only when `capabilities.agent-device.enabled` is `true`. | unset (no cap) | | `capabilities.agent-device.command` | **Worker side.** The agent-device executable: a name looked up on `PATH`, or a path. A gateway ignores it and logs a warning. Used only when `capabilities.agent-device.enabled` is `true`. | `agent-device` | +| `capabilities.stream.enabled` | **Both sides.** Offer a live view of a leased simulator's or emulator's screen. On a worker: answer `GET /v1/stream/leases/{leaseId}` (see [HTTP-API.md](HTTP-API.md#get-v1streamleasesleaseid)) by capturing the screen of the device that lease holds. On a gateway: serve the same URL and relay each request to the worker that holds the lease. Off by default because a stream is a capture pinned to a leased device — with it off, nothing is running until someone asks to look, and nothing is running when they stop. Read at start; changing it needs a restart. | `false` | +| `capabilities.stream.fps` | **Worker side.** The frame rate the encoder is asked for. A capture sends full frames only when the device's screen changes, and on a still screen only small keep-alive frames, so this is a ceiling and not a promise: an untouched home screen stays far below it, which is what keeps an idle stream cheap. A whole number from `1` to `60`. | `30` | +| `capabilities.stream.maxDimension` | **Worker side.** Cap on the encoded picture's longest side, in pixels; the other side follows the device's aspect ratio. This is the lever that costs the most bandwidth — a device that encodes at about 6 Mbit/s native is nearer 1.4 Mbit/s at `720`. A whole number from `96` to `4096`. | `720` | +| `capabilities.stream.bitrateKbps` | **Worker side.** Target bitrate in kilobits per second, handed to the encoder as an average rather than a ceiling; a frame that needs more takes it. A whole number from `100` to `40000`; the iOS encoder takes nothing lower. | `1500` | | `exec.timeoutMs` | **Worker side.** How long one `device.exec` command (`simlock simctl` / `simlock adb` against a gateway or over HTTP) may run before the worker kills it and the operation fails with `EXEC_TIMEOUT`. Authoritative: it bounds the process that actually runs. | `10 minutes` | | `gateway.routing` | **Gateway side.** Which routing policy places a queued request on a worker. `warm-then-free` is the only policy in v1. Among the workers that can serve the request, in order: a worker whose health is not `running`, or that has requests of its own waiting, is left out; a worker holding a ready device that fits goes first (the device's mode must fit the request, a request with no mode fits a device that serves the worker's default mode, and its runtime must be the one requested or, with none requested, the one the worker would pick: the newest runtime the model pairs with); otherwise a worker with no free running slot for the platform or overall is left out (a running device that is not leased counts as free: the worker shuts it down to make room), a worker under its RAM budget for the platform is preferred over one at it (a worker at its budget is still asked when it is the only one left), and the one with the most free running capacity for the platform wins. A worker that is left out is busy, not unable: the request waits for it. Ties go to the lowest worker id. | `warm-then-free` | | `gateway.disconnectedRetentionMs` | **Gateway side.** How long a disconnected worker is kept (greyed, never dispatched to) before the gateway forgets it. The clock is held while the gateway still knows of gateway-issued leases on that worker, and that hold ends when the last of those leases passes its deadline. | `24 hours` | @@ -73,6 +77,7 @@ edit yourself keeps the mode you give it until the next `config set`; run | `stalledTransition.thresholdMultiplier` | Factor applied to a driver's own `provision + boot` (for `provisioning`) or `reclaim` (for `reclaiming`) estimate to get the stall threshold: past it, `simlock doctor` reports a `stalled-transition` finding and `simlock status` marks the device `stalled`. | `3` | | `stalledTransition.minimumThresholdMs` | Floor under the multiplied estimate, for a driver whose estimate is near zero. | `1 minute` | | `drivers.ios.deviceRoot` | The CoreSimulator device set Simlock owns and scopes every `simctl` call to. See [Device roots](#device-roots). | `${SIMLOCK_HOME}/devices/ios` | +| `drivers.ios.screenCaptureCommand` | The program that captures a simulator's screen for `capabilities.stream.enabled`. Unset, the one `pnpm build` put beside the daemon. | unset | | `drivers.android.deviceRoot` | The AVD home Simlock owns; exported as `ANDROID_AVD_HOME` to every `avdmanager`/`emulator` call. See [Device roots](#device-roots). | `${SIMLOCK_HOME}/devices/android` | | `drivers.android.adbServerPort` | TCP port for Simlock's own adb server. Must not be the shared server's `5037`. Startup fails closed if it is occupied. | `5038` | | `ios.defaultMode` | The device mode an iOS lease request gets when it names none: `slim` or `full`. Every worker makes both kinds whatever this says. See [Device mode: slim and full](#device-mode-slim-and-full). | `full` | @@ -166,8 +171,9 @@ off by default. capacity, drivers, downloads, idle, warmPool, health, ios, android, stalledTransition — configures devices, which a gateway does not have; each one present in a gateway's config is reported with a warning and ignored, the same treatment an -unknown key gets. The worker-side `gateway.url`/`token`/`label` and -`capabilities.agent-device.command` are warned about there too. Nothing is stripped: the same file can be copied between a +unknown key gets. The worker-side `gateway.url`/`token`/`label`, +`capabilities.agent-device.command` and `capabilities.stream.fps`/`maxDimension`/`bitrateKbps` +are warned about there too. Nothing is stripped: the same file can be copied between a worker and a gateway with only `mode` differing, and the warning is what makes the difference visible rather than silent. @@ -568,6 +574,47 @@ Only these four keys exist. There is no way to pass other emulator arguments from config, so nothing here can move an emulator's port or AVD home. +## Screen streaming + +`capabilities.stream.enabled` turns a leased device's screen into something you can look at: +a worker captures the screen of the device a lease holds, encodes it as +H.264, and the bytes reach a browser as the body of one ordinary +`GET /v1/stream/leases/{leaseId}` (see +[HTTP-API.md](HTTP-API.md#get-v1streamleasesleaseid)). On a gateway the same URL +relays the request to the worker that holds the lease, and the gateway forwards +the bytes without reading one of them. + +Nothing is captured until someone asks to look, and asking is the only +on-command there is: the request itself keeps the capture running, and closing +it — pressing Stop, navigating away, closing the tab — is the off-command. A +lease that ends by any other route stops the capture too, so a device never +goes on being filmed after it has been handed back, and no stream outlives a +viewer or a lease. + +What it costs is mostly `capabilities.stream.maxDimension` and `capabilities.stream.bitrateKbps`. Against +a device whose screen is being actively redrawn on an Apple silicon Mac, the +encoder's native settings produce about 6 Mbit/s; at `maxDimension` `720` and +`bitrateKbps` `1500` the same motion is about 1.4 Mbit/s, and a device nobody is +touching sits between 25 and 90 kbit/s because a screen that has not changed +sends almost nothing. `capabilities.stream.fps` is a ceiling rather than a rate for the same +reason. On a Mac an iOS capture costs about 50-80 MB of memory per device being +watched. + +An iOS simulator's screen and an Android emulator's screen can be streamed. A +lease on a physical iPhone or Android phone is refused with +`STREAM_NOT_SUPPORTED`, naming that as the reason rather than failing for one +the reader has to guess. + +- **iOS** needs the capture program `pnpm build` builds on a Mac with Xcode + selected. Without it an iOS stream is refused with + `STREAM_UNAVAILABLE`, saying why; `simlock status` says why too when no + platform on the worker can stream. `drivers.ios.screenCaptureCommand` names a + program at another path. +- **Android** needs nothing more than the SDK the emulators already use. Each + stream copies a small screen server into the emulator, and removes it when + the stream ends. An emulator encodes in software unless its GPU mode gives it + a hardware encoder, so expect about 20 frames a second while the screen moves. + ## Capacity strategies How many devices Simlock lets exist and run at once is decided by a capacity diff --git a/docs/CONSOLE.md b/docs/CONSOLE.md index e3cf328c..aa7fea74 100644 --- a/docs/CONSOLE.md +++ b/docs/CONSOLE.md @@ -212,6 +212,29 @@ and the id of the request that was granted the lease, while the daemon still keeps it. A lease that has been released or has expired says "Lease not found". +### Watching a lease's screen + +A lease's page ends with a **Screen** panel: one button, **Show screen**, and +while it is on, the device's picture with the facts beside it — its size and +codec, the frames painted, the rate they arrive at, the bandwidth they cost, and +how many were dropped rather than drawn behind an earlier one. + +Nothing runs while it is off, and the button is the only control. Pressing +**Stop**, navigating away, closing the tab and signing out all stop the capture +on the worker the same way, and a lease that ends — released, expired, or lost +its device — ends the picture with it rather than leaving a frame that looks +live on the page. What you are looking at is either moving or gone. + +The panel is on every lease's page, and streams wherever the worker the lease +runs on has screen streaming turned on (see [`capabilities.stream.enabled`](CONFIGURATION.md)). +Where it is off, the panel says so; where the lease is on a device a screen cannot be +captured from — a physical iPhone or Android phone — it says that a live screen +comes only from a simulator or an emulator. Either way it starts nothing. + +The first picture can take a while on a slow emulator. The panel waits up to 90 +seconds for it, longer than a worker takes to give up, so a capture that cannot +start shows the worker's reason rather than a timeout. + ### Who holds a lease A lease taken over the HTTP API belongs to the token that took it. The console diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 9ecd5094..61f1b100 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -136,6 +136,9 @@ authorizes it already emitted `lease.granted`, the device's own state is untouched, and a fleet where every `adb shell` command produced a bus event would push everything else out of the ring buffer within minutes. +**A screen stream emits no event either.** Watching a device's screen changes +no lease and no device; the daemon log records each stream opened and refused. + **Every worker's own business events are republished on the gateway's bus** with `workerId` added to the payload, under their original names — the fact happened in that worker's lease handling or reaper, and rewriting either would diff --git a/docs/HTTP-API.md b/docs/HTTP-API.md index f43a8599..f25ff98e 100644 --- a/docs/HTTP-API.md +++ b/docs/HTTP-API.md @@ -265,7 +265,7 @@ device, oldest first, in the shape [`GET /v1/lease-requests`](#get-v1lease-requests) answers. A worker always sends it, empty when nothing waits; an older daemon leaves it out. -**`capabilities`** lists each optional module the daemon knows of, with its +**`capabilities`** lists the optional modules the daemon reports, with their state, in the order it sends them (at most 16, no name twice): ```json @@ -276,7 +276,8 @@ state, in the order it sends them (at most 16, no name twice): characters. `state` is `available`, `unavailable` or `off`. `reason` is there for `unavailable` only: 1 to 500 characters, with control characters replaced by spaces. A worker with `capabilities.agent-device.enabled` `false` or unset sends -`agent-device` as `off`. A status read that takes over 1 s or fails shows +`agent-device` as `off`, and one with `capabilities.stream.enabled` `false` or unset +sends `stream` as `off`. A status read that takes over 1 s or fails shows `unavailable` with `could not read 's status`. A gateway's own answer leaves `capabilities` out: each worker's list is on its entry in `workers`. @@ -903,6 +904,56 @@ Paths in `args` resolve on the daemon's filesystem that path to exist *there*). Getting a file onto that machine is not part of this API in this version — see [Not implemented](#not-implemented). +### `GET /v1/stream/leases/{leaseId}` + +Streams the screen of the device one lease holds, for as long as the +connection stays open. Offered only where `capabilities.stream.enabled` is `true` (see +[CONFIGURATION.md](CONFIGURATION.md)); a daemon without it never mounts this +route and answers it like any path it has not heard of. + +The response has no declared length and does not end on its own. Its headers +describe the picture before a byte of it is read: + +| Header | | +|---|---| +| `content-type` | `application/vnd.simlock.stream+h264`, or `+mjpeg` | +| `simlock-stream-codec` | `h264` or `mjpeg` | +| `simlock-stream-width` / `-height` | The encoded picture's size in pixels | +| `simlock-stream-fps` | The frame rate the encoder was asked for | +| `x-simlock-lease` | The lease id this stream is | + +The body is a sequence of frames, each one `4-byte big-endian length`, then a +1-byte tag, then the payload. The length covers the tag byte. Tag `0x05` is a +JSON frame naming the device's `codec`, `width`, `height` and `fps`, and it is +always the first frame; `0x01` is the codec's parameter sets, `0x02` a key +frame, `0x03` a delta. A key frame's and a delta's payload is length-prefixed +AVCC NAL data, which is what a browser's `VideoDecoder` consumes directly, and +the `0x01` payload is an `avcC` blob for its `configure` call — the bytes a +capture wrote are the bytes a reader gets, re-framed by nothing on the way. + +**Closing the request is the only stop-command.** There is no stop endpoint and +no stop frame: holding the connection open is what keeps the capture running, +and a client that goes away — aborting the fetch, closing the tab, losing the +network — ends it. A lease ending by any route stops it too, including while a +viewer is still connected, so a stream cannot outlive the lease that authorises +it. Nothing is captured before a request arrives, which is why this route is +worth asking for twice rather than holding open in advance. + +Authorization is the same as every other lease route: a token sees its own +leases, an operator sees any, and anything else is a `404`. `STREAM_NOT_SUPPORTED` +(415) means the device behind the lease can never produce a picture here — a +physical phone. `STREAM_UNAVAILABLE` (503) means it could but cannot now: a +capture program that is not installed, or a device that painted nothing within +30 seconds; asking again is the right move. A stream that stops receiving frames +for 30 seconds ends with an error rather than freezing on its last picture. `WORKER_UNREACHABLE` on +a gateway means the worker that holds the lease is gone. + +The body is the same bytes on a gateway as on the worker: a gateway relays a +stream rather than reading one, so a client pointed at a gateway needs no second +code path. A refusal, by contrast, keeps the shape every other route uses and +the worker's own code — a viewer told `STREAM_NOT_SUPPORTED` by a gateway +learned that from the machine that looked at the device. + ### `POST /v1/components/install` Role: `operator`. Installs one iOS simulator runtime or Android system image @@ -1700,7 +1751,7 @@ Every failure is the same shape the daemon protocol uses: | 400 | `BAD_REQUEST` (malformed body, bad query param, validation) | | 401 | `UNAUTHENTICATED` (missing or unrecognized token) | | 403 | `FORBIDDEN` (role doesn't permit the route — including a `worker` token on any `/v1` route other than `/v1/uplink`, and an `agent`/`operator` token at `/v1/uplink`; a `/v1/lease-requests/*` route whose request another token sent; or `POST /v1/leases/{id}/renew`/`DELETE /v1/leases/{id}`/`POST /v1/leases/{id}/exec` naming another requester's still-live lease), `DOWNLOADS_DISABLED` (`POST /v1/components/install` under `downloads.policy: "never"`) | -| 404 | `UNKNOWN_WORKER` (`POST`/`DELETE /v1/workers/{id}/drain` naming a worker the gateway does not know), `UNKNOWN_LEASE_REQUEST` (unknown request id), `UNKNOWN_LEASE` (unknown lease id, expired/released, **or `GET /v1/leases/{id}`/`GET /v1/leases/{id}/events` naming another requester's lease** — see [`GET /v1/leases/{id}`](#get-v1leasesid)) , `UNKNOWN_DEVICE` (`DELETE /v1/devices/{id}` naming a device that is not enrolled) | +| 404 | `UNKNOWN_WORKER` (`POST`/`DELETE /v1/workers/{id}/drain` naming a worker the gateway does not know), `UNKNOWN_LEASE_REQUEST` (unknown request id), `UNKNOWN_LEASE` (unknown lease id, expired/released, **or `GET /v1/leases/{id}`/`GET /v1/leases/{id}/events` naming another requester's lease** — see [`GET /v1/leases/{id}`](#get-v1leasesid); also `GET /v1/stream/leases/{leaseId}` naming a lease unknown, released, or another requester's — a stream reveals no lease that asking could not already list), `UNKNOWN_DEVICE` (`DELETE /v1/devices/{id}` naming a device that is not enrolled) | | 409 | `REQUESTER_ALREADY_LEASED` (body names the existing lease id; fleet-wide on a gateway), `LEASE_ID_TAKEN` (a `leaseId` an active lease or a waiting request holds; body carries `leaseId`; on a gateway, one of the gateway's own leases or requests, or a worker that refused it), `IDEMPOTENCY_CONFLICT` (an `Idempotency-Key` repeated with a different device or `leaseId`), `REQUEST_NOT_CANCELLABLE` (body names the lease id if the request had already been granted), `WORKER_CONNECTED` (`DELETE /v1/workers/{id}` while its uplink is open), `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE` (body carries `devices` and `foreignDevices`), `COMPONENT_BUSY` (the three refusals of `DELETE /v1/components/{platform}/{version}`), `DEVICE_BUSY` (`POST /v1/devices`, `DELETE /v1/devices/{id}` of a leased, reclaiming or held device; body carries `id` and `reason`) | | 422 | `UNKNOWN_MODEL`, `RUNTIME_MISSING`, `NO_DRIVER`, `CAPABILITY_MISSING` (a request required capabilities no worker able to serve it has available; body carries `capabilities`), `DEVICE_NOT_ENROLLABLE` (`POST /v1/devices`; body carries `id`, `platform`, `reason`, and `canonicalId` when known), `PASSTHROUGH_REFUSED` (a refused `exec` verb or `devicectl` command, a caller-supplied `--set`/`-P`, a bare `adb shell`), `UNKNOWN_PASSTHROUGH_TOOL` | | 501 | `UNSUPPORTED_IN_GATEWAY_MODE` (an operation that acts on one machine, asked of a gateway: `POST /v1/components/install`, `GET /v1/components`, `DELETE /v1/components/{platform}/{version}`, `POST /v1/devices`, `DELETE /v1/devices/{id}`), `UNSUPPORTED_IN_WORKER_MODE` (an operation on a gateway's workers, asked of a single host: `POST`/`DELETE /v1/workers/{id}/drain`, `DELETE /v1/workers/{id}`, `POST /v1/components/install` with `workers`) | diff --git a/docs/internal/ARCHITECTURE.md b/docs/internal/ARCHITECTURE.md index 49beff7e..e3c0699e 100644 --- a/docs/internal/ARCHITECTURE.md +++ b/docs/internal/ARCHITECTURE.md @@ -42,10 +42,11 @@ worker daemon ── uplink (WebSocket to /v1/uplink, worker dials) ──┘ | core | Registry, device lifecycle, reclaim, quarantine, cleanup, components, doctor | [core](../../src/core/AGENTS.md) | | core/warm-pool | What stays booted, and warm targets | [warm-pool](../../src/core/warm-pool/AGENTS.md) | | core/capacity | Limits, RAM budget, reservations | [capacity](../../src/core/capacity/AGENTS.md) | -| drivers | The `Driver` port's two implementations | [drivers](../../src/drivers/AGENTS.md) · [android](../../src/drivers/android/AGENTS.md) · [ios](../../src/drivers/ios/AGENTS.md) | +| drivers | The `Driver` port's two implementations | [drivers](../../src/drivers/AGENTS.md) · [android](../../src/drivers/android/AGENTS.md) · [android screen](../../src/drivers/android/screen/AGENTS.md) · [ios](../../src/drivers/ios/AGENTS.md) | | gateway | Fleet queue, routing, forwarding to workers, relay | [gateway](../../src/gateway/AGENTS.md) · [routing](../../src/gateway/routing/AGENTS.md) | | link | Byte channels over the uplink | [link](../../src/link/AGENTS.md) | | agent-device | Optional edge: the agent-device daemon beside a worker, its HTTP front | [agent-device](../../src/agent-device/AGENTS.md) | +| stream | Optional edge: a leased device's live screen over HTTP, relayed by a gateway | [stream](../../src/stream/AGENTS.md) | | http | HTTP frontend: routes, tokens, SSE | [http](../../src/http/AGENTS.md) | | cli | Operator command surface | [cli](../../src/cli/AGENTS.md) | | mcp | stdio MCP server for one agent session | [mcp](../../src/mcp/AGENTS.md) | @@ -69,12 +70,13 @@ Arrows point from the optional to the required (architecture rule 15). | `past-index` | Outside a component, import only its index file (tests may also import its testing file) | | `core-not-leasing` | Core declares a port; the composition root connects leasing to it | | `core-not-drivers` | Core and leasing reach a platform only through the `Driver` interface | -| `no-fs-or-process` | Core, leasing, agent-device and host-load use ports, never `fs` or `child_process` | +| `no-fs-or-process` | Core, leasing, agent-device, host-load and stream use ports, never `fs` or `child_process` | | `definition-is-light` | A module's definition file imports its own module only for types or by dynamic import | | `modules-imports-little`, `modules-imports-bus-types-only` | `src/modules` imports only ports, contract and core, and the bus for types | | `not-modules` | Core, leasing, drivers, gateway and contract import nothing from modules or module-registry | | `not-agent-device` | Core, leasing and drivers import nothing from agent-device | | `host-load-imports-ports-only`, `not-host-load` | host-load stands alone | +| `not-stream`, `stream-not-edges` | Nothing required imports stream; stream imports no frontend, gateway or driver | | `gateway-not-edges`, `gateway-only-dispatch` | Gateway imports no driver or frontend, and from daemon only `src/daemon/dispatch.ts` | | `only-main-knows-gateway` | In daemon, only `src/daemon/main.ts` imports the gateway | | `link-is-portable`, `link-imports-port-types-only` | The link uses no Node module or package, and only the types of ports | @@ -93,6 +95,7 @@ never import each other, directly or through a third. | Startup and stop | [startup](flows/startup.md) | | `device.exec`, local and through a gateway | [device-exec](flows/device-exec.md) | | agent-device through simlock | [agent-device](../../src/agent-device/AGENTS.md) | +| A screen stream, local and through a gateway | [stream](../../src/stream/AGENTS.md) | ## Rules that cut across modules diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index 119b54a5..768583e6 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -218,6 +218,11 @@ else out of a 1000-entry ring buffer within minutes. An audit trail of what agents ran on their devices is a different feature with different retention needs, not a line in this catalogue. +**A screen stream emits no event either** (ADR 0031). Watching a device's +screen changes no lease and no device, the same as `device.exec`, and a viewer +toggling the console's Screen pane would fill the ring buffer with nothing +behind it. The daemon log records each stream opened and each one refused. + **Every worker's own business events are republished on the gateway's bus** with `workerId` added to the payload, under their original names and with their original emitting module — the fact happened in that worker's lease diff --git a/docs/internal/VENDORED-CODE.md b/docs/internal/VENDORED-CODE.md new file mode 100644 index 00000000..1b05eeb5 --- /dev/null +++ b/docs/internal/VENDORED-CODE.md @@ -0,0 +1,82 @@ +# Vendored code + +Every piece of third-party code this repository carries, and how to re-verify +each one. The rules that decide what goes in this list are in +[agent-rules/vendor-and-third-party-code.md](agent-rules/vendor-and-third-party-code.md); +this page is the inventory, one row per dependency. A change under `vendor/` +updates the affected row here, its directory's `NOTICE.md`, and +[`vendor/README.md`](../../vendor/README.md)'s index, in the same change. + +The convention, in one line: `vendor//` holds the artifact(s), the +license text(s) they require under `LICENSES/`, and a `NOTICE.md`. + +## The inventory + +| What | Upstream | Version / commit | License | Where it lives | Consumed by | Re-verify with | +|---|---|---|---|---|---|---| +| `@agent-device/proxy` — the transport proxy in front of an agent-device daemon, packed from a branch of agent-device that is not on npm | `callstack/agent-device`, `simlock-integration` branch | commit `253dca023edf13410f7928c76e9e604e5e5871a0` | MIT (Callstack) | `vendor/agent-device-proxy-0.21.28-dev-253dca0.tgz`; license inside the archive at `package/LICENSE` | `package.json`'s `"@agent-device/proxy": "file:vendor/…tgz"`, which `src/agent-device/host/` imports | `tar -tzf vendor/agent-device-proxy-*.tgz \| grep LICENSE`; the commit is what `scripts/slow-e2e.sh` checks a local checkout against, so keep it the first 40-char hash in `vendor/README.md` | +| serve-sim's capture and encode pipeline — CoreSimulator framebuffer capture and the H.264 encode path, patched by us, with four files rewritten by us that stay serve-sim's modified files | `EvanBacon/serve-sim` | commit `6d009686ed08f7d8bd19bca9f9f67a9237931f44` | Apache-2.0 for serve-sim's files and our rewrites of four of them; MIT for the three files that are ours entirely | `vendor/screen-capture/Sources/simlock-screen-capture/`, license at `vendor/screen-capture/LICENSES/serve-sim.APACHE-2.0.txt` | the `simlock-screen-capture` program `pnpm build` puts in dist/bin, run by `src/drivers/ios/screen-capture.ts` | the `curl … \| diff -u -` command in [`vendor/screen-capture/NOTICE.md`](../../vendor/screen-capture/NOTICE.md), per file | +| `scrcpy-server` — the Android screen-streaming server artifact, byte-identical to upstream's release asset | `Genymobile/scrcpy` | release `v5.0.1`; revision built `dc8a55a24c27ae78ae202b5a3288da681d0f141a` | Apache-2.0; the Kotlin standard library inside it adds BSD-3-Clause (ThreeTen-BP) and BSL-1.0 (Boost) code | `vendor/scrcpy/scrcpy-server-v5.0.1`, licenses at `vendor/scrcpy/LICENSES/scrcpy.APACHE-2.0.txt` and `vendor/scrcpy/LICENSES/threetenbp.BSD-3-Clause.txt`, digest in `vendor/scrcpy/SHA256SUMS.txt` | the Android driver's screen capture, which pushes it unmodified to a leased emulator; the client that drives it is ours, in `src/drivers/android/`, and is not vendored | `cd vendor/scrcpy && shasum -a 256 -c SHA256SUMS.txt`, and the same against upstream's own release `SHA256SUMS.txt`, whose GPG signature `vendor/scrcpy/NOTICE.md` says how to check | + +`vendor/scrcpy/NOTICE.md` names what is *inside* that artifact besides scrcpy's +own classes — the Kotlin standard library 2.2.10 and the JetBrains/IntelliJ +annotation classes — because they are in the file we distribute even though we +did not choose them one by one. Not all of it is Apache-2.0: Kotlin's own +`license/README.md` marks `kotlin.time` as derived from ThreeTen-BP (BSD-3-Clause, +whose notice binary redistribution must carry, so its text is in +`vendor/scrcpy/LICENSES/`) and `MathJVM.kt` as derived from Boost (BSL-1.0, which +exempts object code). Upstream scrcpy ships the artifact without the ThreeTen-BP +text. + +## What is third-party but not vendored + +These are third-party code the repo depends on and ships or installs through +their own distribution, so `vendor/` is the wrong place for them and this is the +reason they are not above. + +| What | Upstream | Version | License | Where it is | Why it is not vendored | +|---|---|---|---|---|---| +| React, React DOM, Hono, `ws`, Zod, the MCP SDK, TanStack table/virtual, Recharts | npm | as `pnpm-lock.yaml` pins | per package | `node_modules/`, resolved from the lockfile | A registry gives us these, with their licenses; vendoring a tree we never patch only loses upstream's fixes | +| The Geist and Geist Mono variable fonts | `vercel/geist-font`, repackaged as `@fontsource-variable/geist` | `^5.3.0` | OFL-1.1 | `ui/src/main.tsx` imports them; the built console serves the `.woff2` files out of `dist/ui/assets/` | Installed from npm and bundled at build time. **Known gap:** the font license travels with the npm package but Vite does not copy it beside the emitted `.woff2`, and OFL-1.1 wants the license to accompany the font; the fix is to ship the license with the console build, not to vendor the fonts | +| The agent-device CLI itself | `callstack/agent-device` | the commit above | MIT | a separate checkout, at the absolute path `SIMLOCK_E2E_AGENT_DEVICE` gives, on machines that run the slow lane | It is a tool we invoke, not code we carry; only its proxy is vendored, and `toolchain.md` says how to read its source at that commit | + +The license question for the first row is answered by npm, which ships each +package's own license text next to it; nothing in that row is a file in this +repo, which is what separates it from the three rows above. + +## External tools are not vendored + +`simctl` and the rest of Xcode's command-line tools, the Android SDK's +`emulator`, `avdmanager` and `sdkmanager`, and `adb` are **prerequisites this +project invokes and does not carry**. They arrive with the user's Xcode or Android +SDK installation, at whatever version that installation happens to be, and +simlock never downloads, bundles, or pins one. `docs/CONFIGURATION.md` and the +driver prerequisite checks are where their absence is reported. + +Worth keeping the two ideas apart, because the confusion is cheap to make: a +vendored artifact is a byte we distribute and are therefore obliged to license +correctly, while an external tool is something we shell out to and can only ask +for. That is why the scrcpy *server* is vendored and `adb` — which is how it +reaches the device — is not; and why the capture program's Swift source is +vendored while the Xcode that compiles it is not. + +## How a new vendored thing gets here + +1. Decide it cannot stay an upstream dependency: a registry does not carry the + line we need, or we must patch it. +2. Make `vendor//`, put the artifact(s) in it, upstream's verbatim + license text in `LICENSES/..txt`, and `NOTICE.md` written + to the list in rule 4 of + [agent-rules/vendor-and-third-party-code.md](agent-rules/vendor-and-third-party-code.md). + A binary also gets `SHA256SUMS.txt` in `shasum -c` format. +3. Add the row here and the directory to `vendor/README.md`'s index. Keep the + agent-device commit the first 40-character hash in that file — the slow lane + reads the first one it finds. +4. `pnpm exec vitest run --project unit src/source-hygiene.test.ts` checks the + shape (license text, `NOTICE.md`, the digest, and that a vendored binary is + excluded from the text scan for a reason it can prove). + +Nothing here is fetched at install, at build or at run time: an artifact is in +the tree because a person or an agent fetched it once, checked it, and committed +it, which is what safety rule 4's consent requirement looks like for vendored +code. diff --git a/docs/internal/agent-rules/always-in-scope.md b/docs/internal/agent-rules/always-in-scope.md index d1e9e2b5..96db5655 100644 --- a/docs/internal/agent-rules/always-in-scope.md +++ b/docs/internal/agent-rules/always-in-scope.md @@ -55,6 +55,14 @@ spec says otherwise in its own words, the spec wins. 11. **Comments say why.** Every comment the change adds or edits starts with `why:` and says why, or is deleted; the commit hook and CI reject the rest (architecture rule 17). +12. **A vendored artifact's notice and row.** A change under `vendor/` updates + that directory's `NOTICE.md`, its row in `docs/internal/VENDORED-CODE.md` + and `vendor/README.md`'s index in the same change + (vendor-and-third-party-code rule 11). A new first-level `vendor/` + directory carries its license text under `LICENSES/` and a `NOTICE.md`; + `src/source-hygiene.test.ts` enforces that shape, and counts a vendored + binary as excluded from the text scan only where a `SHA256SUMS.txt` pins + its digest (vendor-and-third-party-code rules 2, 3, 7). **CI proves the suite.** CI runs the checks [toolchain.md](toolchain.md) lists on every push, and the merge gate refuses a PR whose checks are not diff --git a/docs/internal/agent-rules/toolchain.md b/docs/internal/agent-rules/toolchain.md index edf0a5ac..8a4bde3c 100644 --- a/docs/internal/agent-rules/toolchain.md +++ b/docs/internal/agent-rules/toolchain.md @@ -61,6 +61,25 @@ that goes through the reviews its diff's kind gets and the merge gate: | Comment gardening | `garden-comments` | weekly | | Review calibration | `calibrate-reviews` | weekly | +## The screen capture programs + +`pnpm build` runs `scripts/build-capture.mjs`: it copies the vendored scrcpy +server into `dist/vendor/scrcpy/` after checking its pinned SHA-256, and on a +Mac with Xcode selected it builds `vendor/screen-capture/` (Swift) into +`dist/bin/simlock-screen-capture`. Elsewhere the Swift build is skipped and iOS +streams report unavailable. + +Neither runs in the unit, e2e or console lanes: both read a booted device's +screen, which means real devices and the slow lane. The lanes cover what this +repo writes -- the stream module, the drivers' capture code against fakes, the +envelope -- and the console lane runs a stand-in program that writes the same +envelope from stills (`e2e/console/screen-capture.ts`), named by the fake +driver's `screenCaptureCommand`. A change to the Swift program is checked by +hand on a machine with a simulator, where +`node vendor/screen-capture/scripts/unframe.mjs out.264` turns the +bytes it wrote into Annex-B and counts each tag, and `ffprobe out.264` says +whether they decode. + ## Slow lane Tests that start real simulators or emulators. Run them only through @@ -113,7 +132,7 @@ the tool's docs alone. | Tool | Version | Read it | | ---- | ------- | ------- | -| agent-device | the version `package.json` pins for `@agent-device/proxy` (tag `v`) | `git -C show v:` in a clone of `callstackincubator/agent-device` (on a machine that runs the slow lane, the one `.slow-lane.env` names). No clone: `gh api 'repos/callstackincubator/agent-device/contents/?ref=v' -H 'Accept: application/vnd.github.raw'`. The proxy as simlock bundles it: `node_modules/@agent-device/proxy/dist` | +| agent-device | the version `package.json` pins for `@agent-device/proxy` (tag `v`) | `git -C show v:` in a clone of `callstack/agent-device` (on a machine that runs the slow lane, the one `.slow-lane.env` names). No clone: `gh api 'repos/callstack/agent-device/contents/?ref=v' -H 'Accept: application/vnd.github.raw'`. The proxy as simlock bundles it: `node_modules/@agent-device/proxy/dist` | | Android SDK tools, Xcode tools | the installed ones | run them: ` --help`, or the command itself | | simlock's own CLI | `main` | `pnpm build`, then `node dist/cli/main.js --help` or the command itself | diff --git a/docs/internal/agent-rules/vendor-and-third-party-code.md b/docs/internal/agent-rules/vendor-and-third-party-code.md new file mode 100644 index 00000000..f1acb368 --- /dev/null +++ b/docs/internal/agent-rules/vendor-and-third-party-code.md @@ -0,0 +1,116 @@ +# Agent rules: vendored and third-party code + +Rules for anything third-party this repository carries in its own tree: +someone else's source, someone else's binary, someone else's package. The +inventory of what is here today is [../VENDORED-CODE.md](../VENDORED-CODE.md); +this file is the rule that keeps the inventory true. + +1. **Third-party code lives under `vendor/`, one directory per upstream.** + `src/`, `ui/`, `scripts/` and `e2e/` hold no file this project did not + write. A dependency npm resolves stays an npm dependency — vendoring is for + what a registry does not give us: a fork at an unreleased commit, a source + tree we patch, an artifact we ship. + +2. **A first-level directory under `vendor/` is third-party _or derived-from_ + third-party code, and never either by accident.** It holds the artifact(s) we + took or derived, every license text they require, and a `NOTICE.md`. A + directory here may also hold a build of that code — a tool compiled from + vendored sources, with our CLI and envelope around it — because keeping a + derivative beside what it derives from is what keeps the provenance in `rule + 5`'s diff readable; `vendor/screen-capture/` is that case. What a directory + under `vendor/` must never hold is code with no upstream at all: that belongs + in `src/`, `ui/`, `scripts/` or `e2e/`. `NOTICE.md`'s per-file origin table + is what separates ours from theirs, and rule 6 makes that table mandatory + precisely because the directory itself no longer does. The check + `src/source-hygiene.test.ts` runs is the licensing half of this claim, so a + directory that fails it misrepresents itself. A file sitting at `vendor/`'s + root is a packed archive, not a directory, and carries its license inside it + (the tarball rule 1's npm case does not cover) — that is why the check is per + directory. + +3. **A license text is upstream's verbatim text, named for what it covers.** + It lives in `/LICENSES/..txt`. Do not name it + `LICENSE`: a bare `LICENSE` in a directory states that it licenses + everything there, and a vendored directory often holds our own files + beside theirs. Where upstream ships no `NOTICE` file, say so in `NOTICE.md` + — that is the difference between Apache §4(d) being satisfied and being + vacuous, and a reader cannot tell from the archive. + +4. **`NOTICE.md` states these, in this order.** Upstream repository and + author; the exact commit or release every line came from; what we vendored + and, as its own list, what we deliberately left out; every deviation we made + from upstream, each with the reason that forced it; the one command that + re-verifies provenance against upstream; and what in the directory is ours. + Each of those is a checkable item, so the file is either complete or it is + not. The reason a deviation needs its reason written down is that the only + thing worse than an unexplained patch is a patch someone "cleans up" back to + upstream's behaviour, breaking the feature that needed it. + +5. **Vendored source records the commit and the diff command; it never stores + a pristine copy.** `NOTICE.md` names the commit and prints the command that + fetches upstream's version of a file and diffs it against ours. Committing + pristine copies too gives a source file two homes, and sooner or later + somebody edits the wrong one. The diff against upstream is also the + early-warning system for a vendored file that reached an API the upstream + platform moved — a major toolchain update shows up there before it shows up + as a failure in the field. + +6. **Vendored source names its origin per file, never by a count.** `NOTICE.md` + carries a table of file → origin, and the line counts stay out of it: they + are the first thing a rewrite invalidates, and a stale count reads as + authority. Origin is not remembered, it is measured — `NOTICE.md` gives the + command, and the row is wrong the moment the command says it is. Where a + file is ours in shape and upstream's in idea, that is what the row says. + +7. **A vendored binary is byte-identical to what upstream published, with its + checksum beside it.** `SHA256SUMS.txt`, in upstream's own + ` ` form, so `shasum -a 256 -c SHA256SUMS.txt` verifies + the directory from inside it; the CI check runs that verification, so the + claim is proved rather than asserted. Never patch, re-pack, re-sign or + strip a vendored binary — an artifact we cannot reproduce byte for byte from + the release is an artifact nobody can re-verify. When upstream publishes a + checksum file for the release, check ours against it and say in `NOTICE.md` + that we did. + +8. **A vendored binary names what is inside it.** A packaged artifact carries + other people's code we did not choose file by file — a runtime library, + annotation jars, a bundled dependency. Name every one of those in + `NOTICE.md` with its license, whether or not it conflicts with ours: it is + in the artifact we distribute, so attribution reaches it, and the next + reader cannot see it without unpacking the file. + + Whatever the package ships from `vendor//` — the artifact itself or a + program built from it — ships with that directory's `NOTICE.md` and + `LICENSES/`, copied into `dist/vendor//` by `scripts/build-capture.mjs` + (tested in `src/build-capture.test.ts`). The package publishes `dist/`, and a + license kept only in the repository never reaches whoever installs it. + +9. **Nothing arrives over the network without the consent + [safety.md](safety.md) rule 4 requires.** Vendoring is a commit: a person or + an agent fetched the artifact once, checked it, and put it in the tree. + Nothing in `vendor/` is downloaded at install, at build, or when the daemon + starts, and no postinstall step reaches a package registry or a release URL + for it. That rule is about multi-GB downloads the user never asked for; the + same reason covers a 700 KB jar — a fetch nobody consented to is a failure + nobody can attribute, and it makes a build's output depend on the state of + someone else's server. Pinning a version and fetching it lazily is not a + smaller version of this rule: it is the thing the rule forbids. + +10. **A vendored artifact nothing consumes yet says so in its own notice.** A + notice that reads as though the artifact were in use is a claim about the + code, and it is false. Name the planned consumer and that it does not + exist yet; the alternative — leaving the artifact unexplained — reads as + somebody else's mistake and invites its deletion. + +11. **A change under `vendor/` updates the notice and the inventory in the same + change** (architecture rule 13, always-in-scope item 12): `NOTICE.md`, the + row in [../VENDORED-CODE.md](../VENDORED-CODE.md), and `vendor/README.md`'s + index. A toolchain upgrade of a vendored artifact is a new commit or + release in those three places, not an edit of the old one: what the + previous line was is the information a later diff needs. + +12. **An external tool is not vendored, and is not listed as if it were.** + Xcode's `simctl`, the Android SDK, `adb` and an emulator image are + prerequisites we invoke and do not carry. They belong in the note that + separates the two, not in the inventory's rows; conflating them is how a + "we vendor it" claim ends up in a doc that a user then cannot install from. diff --git a/e2e/README.md b/e2e/README.md index 3fe5e828..bb1669e4 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -67,5 +67,5 @@ running test body silently never fires — see the comment in `helpers/env.ts`. 60s limit on a cold boot ([#206](https://github.com/callstackincubator/simlock/issues/206)). Treat the slow lane as a manual, one-flow-at-a-time tool until both are fixed. - Android is exercised through the fake driver in the fast lane; only - `slow-android-smoke.test.ts`, `slow-warm-pool.test.ts`, `slow-agent-device.test.ts` and - `slow-agent-device-gateway.test.ts` touch a real emulator. + `slow-android-smoke.test.ts`, `slow-android-stream.test.ts`, `slow-warm-pool.test.ts`, + `slow-agent-device.test.ts` and `slow-agent-device-gateway.test.ts` touch a real emulator. diff --git a/e2e/console/screen-capture.ts b/e2e/console/screen-capture.ts new file mode 100644 index 00000000..e683f55b --- /dev/null +++ b/e2e/console/screen-capture.ts @@ -0,0 +1,72 @@ +import { chmod, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +// why: the browser lane needs a capture program for the fake driver's screen to run, and the real +// one is Swift, needs a booted simulator, and cannot run on a CI runner. This writes the same envelope +// that tool writes -- a meta frame naming `mjpeg`, then one JPEG per delta frame -- from a process that +// owes its life to nothing but the daemon that started it. Stills rather than H.264 so the lane decodes +// in every browser, WebCodecs being Chromium's for now; the H.264 path is pinned by unit tests. + +// why: three solid-colour stills -- red, lime, blue, made by +// `ffmpeg -f lavfi -i color=red:s=120x240 -frames:v 1 -q:v 30 f.jpg` -- held as base64 so the program +// below turns them back into bytes with no fixture file to ship. They differ because the lane asserts +// the pane shows a picture that *changes*, and one colour painted twice is not a change. +const STILLS = [ + "/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYxLjE5LjEwMAD/2wBDAAg8PEc8R1JSUlJSUmFaYWVlZWFhYWFlZWVsbGx/f39sbGxlZWxseHh/f4qOioODf4OOjpaWlrS0rKzS0tn/////xABNAAEBAAAAAAAAAAAAAAAAAAAABgEBAQEAAAAAAAAAAAAAAAAAAAYHEAEAAAAAAAAAAAAAAAAAAAAAEQEAAAAAAAAAAAAAAAAAAAAA/8AAEQgA8AB4AwEiAAIRAAMRAP/aAAwDAQACEQMRAD8AiwEo38AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB/9k=", + "/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYxLjE5LjEwMAD/2wBDAAg8PEc8R1JSUlJSUmFaYWVlZWFhYWFlZWVsbGx/f39sbGxlZWxseHh/f4qOioODf4OOjpaWlrS0rKzS0tn/////xABMAAEBAAAAAAAAAAAAAAAAAAAABgEBAQAAAAAAAAAAAAAAAAAAAAYQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCADwAHgDASIAAhEAAxEA/9oADAMBAAIRAxEAPwCLASqJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAf/2Q==", + "/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYxLjE5LjEwMAD/2wBDAAg8PEc8R1JSUlJSUmFaYWVlZWFhYWFlZWVsbGx/f39sbGxlZWxseHh/f4qOioODf4OOjpaWlrS0rKzS0tn/////xABNAAEBAAAAAAAAAAAAAAAAAAAABwEBAQEAAAAAAAAAAAAAAAAAAAUHEAEAAAAAAAAAAAAAAAAAAAAAEQEAAAAAAAAAAAAAAAAAAAAA/8AAEQgA8AB4AwEiAAIRAAMRAP/aAAwDAQACEQMRAD8AjgDf0sAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB/9k=", +]; + +export async function writeCaptureProgram(dir: string): Promise { + const path = join(dir, "mjpeg-capture.mjs"); + await writeFile( + path, + `#!/usr/bin/env node +// why: the lane's stand-in for simlock's screen capture program. It writes the same envelope, exits +// when the daemon closes its stdin, and logs when it was started and when it went away -- which is how +// the lane proves a viewer leaving or a lease ending stops a capture rather than only a picture. +import { appendFileSync } from "node:fs"; + +const LOG = ${JSON.stringify(join(dir, "capture.log"))}; +const args = process.argv.slice(2); +const address = args.at(-1); +const stills = ${JSON.stringify(STILLS)}.map((text) => Buffer.from(text, "base64")); + +function envelope(tag, payload) { + const head = Buffer.alloc(5); + head.writeUInt32BE(payload.length + 1, 0); + head[4] = tag; + return Buffer.concat([head, payload]); +} + +const meta = Buffer.from( + JSON.stringify({ udid: address, codec: "mjpeg", width: 120, height: 240, fps: 5, deviceSet: "lane" }), +); + +appendFileSync(LOG, \`opened \${JSON.stringify(args)}\\n\`); +process.stdout.write(envelope(0x05, meta)); + +let next = 0; +const timer = setInterval(() => { + process.stdout.write(envelope(0x03, stills[next++ % stills.length])); +}, 100); + +// why: stdin is the liveness handle the real tool documents, so an "end" on it is the one stop a +// daemon gives a capture, and the line below is the lane's evidence that the request was what held it. +process.stdin.on("end", () => { + appendFileSync(LOG, "closed\\n"); + clearInterval(timer); + process.exit(0); +}); +process.stdin.resume(); +process.stdout.on("error", () => process.exit(0)); +`, + ); + await chmod(path, 0o755); + return path; +} + +export async function captureLog(dir: string): Promise { + const { readFile } = await import("node:fs/promises"); + return readFile(join(dir, "capture.log"), "utf8").catch(() => ""); +} diff --git a/e2e/console/screen.spec.ts b/e2e/console/screen.spec.ts new file mode 100644 index 00000000..6c24c45a --- /dev/null +++ b/e2e/console/screen.spec.ts @@ -0,0 +1,251 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Page } from "@playwright/test"; + +import { expect, type RunningDaemon, signIn, startDaemon, test } from "./fixtures.js"; +import { captureLog, writeCaptureProgram } from "./screen-capture.js"; + +// why: the capture program this lane writes is its witness as well as its picture source -- it records +// when the daemon started it and when it went away. That is what lets these tests say a viewer opening +// a pane *started a capture on the worker* and that the same viewer leaving stopped one, which is the +// one promise this surface makes and a screenshot cannot prove. + +const IOS = { availableOsVersions: ["18.4"], knownModels: ["iPhone 16"] }; +const LEASE = ["lease", "--platform", "ios", "--device", "iPhone 16", "--os", "18.4"] as const; + +interface StreamingHost { + readonly command: string; + readonly dir: string; + readonly host: RunningDaemon; + readonly lease: string; +} + +async function streamingHost(): Promise { + const dir = await mkdtemp(join(tmpdir(), "simlock-screen-")); + const command = await writeCaptureProgram(dir); + let host: RunningDaemon | undefined; + try { + host = await startDaemon("worker", { + config: { + capabilities: { stream: { enabled: true, fps: 5 } }, + }, + driverScript: { ios: { ...IOS, screenCaptureCommand: command } }, + }); + const granted = await host.cli([...LEASE, "--agent-id", "console-screen", "--detach"]); + expect(granted.code, granted.stderr).toBe(0); + return { + command, + dir, + host, + lease: (granted.json as { lease: { id: string } }).lease.id, + }; + } catch (error) { + await rm(dir, { force: true, recursive: true }); + await host?.dispose(); + throw error; + } +} + +async function cleanup(started: StreamingHost): Promise { + await started.host.dispose(); + await rm(started.dir, { force: true, recursive: true }); +} + +async function openLeasePage(page: Page, host: RunningDaemon, lease: string): Promise { + await page.goto(new URL(`/leases/${encodeURIComponent(lease)}`, host.url).toString()); + await signIn(page, host.tokens.operator); + await expect(page.getByRole("heading", { level: 2, name: "Screen" })).toBeVisible(); +} + +function showButton(page: Page) { + return page.getByRole("button", { name: /^(Show screen|Stop)$/ }); +} + +function screen(page: Page) { + return page.getByLabel("The leased device's screen"); +} + +// why: the mean colour of what the canvas holds, so "the picture changed" is a number rather than a +// hope -- three solid colours that alternate mean three distinct readings, and two equal ones mean a +// canvas painted once and left alone. +async function averageColour(page: Page): Promise { + const rgb = await screen(page).evaluate((element) => { + const canvas = element as HTMLCanvasElement; + const image = canvas.getContext("2d")?.getImageData(0, 0, canvas.width, canvas.height); + if (image === undefined) throw new Error("the screen canvas has no 2d context"); + const data = image.data; + let red = 0; + let green = 0; + let blue = 0; + for (let i = 0; i < data.length; i += 4) { + red += data[i] ?? 0; + green += data[i + 1] ?? 0; + blue += data[i + 2] ?? 0; + } + const count = data.length / 4; + return [red, green, blue].map((total) => Math.round(total / count)); + }); + return rgb.join(","); +} + +async function fact(page: Page, label: string): Promise { + return page + .locator(".facts > div") + .filter({ has: page.getByText(label, { exact: true }) }) + .first() + .getByRole("definition") + .innerText(); +} + +async function frames(page: Page): Promise { + const shown = await fact(page, "Frames"); + const parsed = Number.parseInt(shown, 10); + return Number.isNaN(parsed) ? 0 : parsed; +} + +async function streamCapability(host: RunningDaemon): Promise { + const listed = await host.cli(["worker", "list", "--json"]); + expect(listed.code, listed.stderr).toBe(0); + const views = (listed.json as { workers: { capabilities?: { name: string; state: string }[] }[] }) + .workers; + return views[0]?.capabilities?.find((entry) => entry.name === "stream")?.state; +} + +async function captureLogLines(dir: string, word: string): Promise { + const log = await captureLog(dir); + return log.split("\n").filter((line) => line.startsWith(word)).length; +} + +test.describe("the Screen pane", () => { + test("shows nothing and starts no capture until asked", async ({ page }) => { + const started = await streamingHost(); + try { + await openLeasePage(page, started.host, started.lease); + await expect(screen(page)).toHaveCount(0); + await expect(page.getByText("No picture while this is off.")).toBeVisible(); + await expect.poll(() => fact(page, "Picture")).toBe("—"); + await expect.poll(() => fact(page, "State")).toBe("off"); + expect(await captureLogLines(started.dir, "opened")).toBe(0); + expect(await streamCapability(started.host)).toBe("available"); + } finally { + await cleanup(started); + } + }); + + test("Show screen starts the capture, paints it, and reports what it is showing", async ({ + page, + }) => { + const started = await streamingHost(); + try { + await openLeasePage(page, started.host, started.lease); + await showButton(page).click(); + await expect(screen(page)).toBeVisible(); + await logReached(started.dir, "opened", 1); + + // why: these numbers are the capture program's own, so the pane sizing itself from them proves it + // read the meta frame rather than guessing a size from whatever arrived first. + await expect.poll(() => fact(page, "Picture"), { timeout: 10_000 }).toBe("120 × 240 mjpeg"); + await expect.poll(async () => (await frames(page)) > 0, { timeout: 10_000 }).toBe(true); + await expect.poll(() => fact(page, "State"), { timeout: 10_000 }).toBe("live"); + + // why: three different colours arrive in turn, so readings that keep differing prove frames are + // still arriving rather than one picture painted once and left on the canvas. + const seen = new Set(); + await expect + .poll( + async () => { + seen.add(await averageColour(page)); + return seen.size; + }, + { timeout: 10_000 }, + ) + .toBeGreaterThanOrEqual(2); + } finally { + await cleanup(started); + } + }); + + test("Stop stops the capture, and so does leaving the page", async ({ page }) => { + const started = await streamingHost(); + try { + await openLeasePage(page, started.host, started.lease); + await showButton(page).click(); + await logReached(started.dir, "opened", 1); + await expect.poll(async () => (await frames(page)) > 0, { timeout: 10_000 }).toBe(true); + + await showButton(page).click(); + await logReached(started.dir, "closed", 1); + await expect(screen(page)).toHaveCount(0); + await expect(showButton(page)).toHaveText("Show screen"); + + await showButton(page).click(); + await logReached(started.dir, "opened", 2); + await page.goto(new URL("/leases", started.host.url).toString()); + await logReached(started.dir, "closed", 2); + + // why: arriving back at the lease page must not resume a picture nobody asked for -- a third + // "opened" here is a capture running for a viewer that only opened a page. + await page.goto( + new URL(`/leases/${encodeURIComponent(started.lease)}`, started.host.url).toString(), + ); + await expect(showButton(page)).toHaveText("Show screen"); + expect(await captureLogLines(started.dir, "opened")).toBe(2); + } finally { + await cleanup(started); + } + }); + + test("a worker with streaming off refuses the picture and says what to do", async ({ page }) => { + const dir = await mkdtemp(join(tmpdir(), "simlock-screen-")); + let host: RunningDaemon | undefined; + try { + host = await startDaemon("worker", { driverScript: { ios: IOS } }); + const granted = await host.cli([...LEASE, "--agent-id", "console-screen", "--detach"]); + expect(granted.code, granted.stderr).toBe(0); + const lease = (granted.json as { lease: { id: string } }).lease.id; + await openLeasePage(page, host, lease); + await showButton(page).click(); + await expect(page.getByRole("alert")).toContainText("capabilities.stream.enabled is on"); + expect(await streamCapability(host)).toBe("off"); + await expect(screen(page)).toHaveCount(0); + expect(await captureLog(dir)).toBe(""); + } finally { + await host?.dispose(); + await rm(dir, { force: true, recursive: true }); + } + }); + + test("a release takes the pane away and leaves no capture running", async ({ page }) => { + const started = await streamingHost(); + try { + await openLeasePage(page, started.host, started.lease); + await showButton(page).click(); + await logReached(started.dir, "opened", 1); + await expect.poll(async () => (await frames(page)) > 0, { timeout: 10_000 }).toBe(true); + + const released = await started.host.cli(["release", started.lease]); + expect(released.code, released.stderr).toBe(0); + + // why: a lease that is gone has no page to watch, so the pane goes with it and its abort stops the + // capture. What this proves is the promise an operator can see: leaving a lease behind never + // leaves a device being filmed. The guarantee that a lease ending stops a capture whose viewer is + // still connected is not observable from a browser -- it is pinned in + // `src/stream/captures.test.ts`, where a capture can be held open past the lease. + await logReached(started.dir, "closed", 1); + await expect(page.getByText("Lease not found")).toBeVisible(); + expect(await captureLogLines(started.dir, "opened")).toBe(1); + } finally { + await cleanup(started); + } + }); +}); + +async function logReached(dir: string, word: string, count: number): Promise { + // why: a capture stopping is the daemon's act, not the browser's, and it is the fact under test -- so + // a lane that did not wait for it would pass with a capture that never started or never stopped. + await expect + .poll(async () => (await captureLogLines(dir, word)) >= count, { timeout: 10_000 }) + .toBe(true); +} diff --git a/e2e/fake-driver/fake-driver.ts b/e2e/fake-driver/fake-driver.ts index 4c31b4ed..37b79523 100644 --- a/e2e/fake-driver/fake-driver.ts +++ b/e2e/fake-driver/fake-driver.ts @@ -35,7 +35,11 @@ import { type DeviceSpec, type VirtualSpec, type Platform, + type ScreenCapture, + ScreenNotSupportedError, + ScreenUnavailableError, } from "../../dist/core/index.js"; +import { NodeBytePipeLauncher } from "../../dist/ports/index.js"; import type { FakeDriverErrorSpec, FakeDriverOperation, @@ -221,6 +225,7 @@ export class OutOfProcessFakeDriver implements Driver { /** Synthetic: this driver owns no real devices, and nothing validates or creates it. */ readonly deviceRoot: string; readonly passthroughTools: readonly string[]; + readonly screenCapture: ScreenCapture; readonly componentFootprint: { readonly path: string; readonly bytes: number }; readonly #clock: FakeDriverClock; /** Versions `installComponent` put in place, with each one's receipt. */ @@ -242,6 +247,30 @@ export class OutOfProcessFakeDriver implements Driver { this.#clock = options.clock; this.#logPath = options.logPath; this.#scriptPath = options.scriptPath; + this.screenCapture = { + open: async (device) => { + if (device.physical) + throw new ScreenNotSupportedError("A physical phone cannot be streamed"); + const command = (await this.#readScript()).screenCaptureCommand; + if (command === undefined) throw new ScreenUnavailableError("No screen capture program"); + const child = new NodeBytePipeLauncher().start(command, [device.deviceId], { + env: process.env, + }); + let closing: Promise | undefined; + return { + body: child.stdout, + close: () => + (closing ??= (async () => { + child.closeStdin(); + await child.exited; + })()), + }; + }, + unavailable: async () => + (await this.#readScript()).screenCaptureCommand === undefined + ? "No screen capture program" + : undefined, + }; } /** diff --git a/e2e/fake-driver/types.ts b/e2e/fake-driver/types.ts index 473db9e5..4bb24e09 100644 --- a/e2e/fake-driver/types.ts +++ b/e2e/fake-driver/types.ts @@ -139,6 +139,10 @@ export interface FakeDriverPlatformScript { * Defaults to `-address`. */ readonly address?: string; + // why: the browser lane has no simulator to film, so a script names a stand-in program that writes + // the capture envelope; unset, this platform's screen is unavailable, as on a host whose capture + // program was never built. + readonly screenCaptureCommand?: string; readonly missingPrerequisites?: readonly { readonly prerequisite: string; readonly message: string; diff --git a/e2e/slow-android-stream.test.ts b/e2e/slow-android-stream.test.ts new file mode 100644 index 00000000..84849dd6 --- /dev/null +++ b/e2e/slow-android-stream.test.ts @@ -0,0 +1,243 @@ +import { execFile } from "node:child_process"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { DEVICE_READY_MS, waitFor, withHttpDaemon } from "./helpers/index.js"; + +const ANDROID_HOME = + process.env.ANDROID_HOME ?? + process.env.ANDROID_SDK_ROOT ?? + join(homedir(), "Library/Android/sdk"); +const hasAndroidSdk = + existsSync(ANDROID_HOME) && existsSync(join(ANDROID_HOME, "platform-tools", "adb")); + +const TAG = { alive: 0x06, description: 0x01, keyframe: 0x02, delta: 0x03, meta: 0x05 } as const; + +interface Frame { + readonly tag: number; + readonly payload: Uint8Array; +} + +// why: the body is an endless stream, so this reads frames until it has what it came for and then +// lets go, rather than reading to an end that never comes. +async function readFrames( + body: ReadableStream, + enough: (frames: readonly Frame[]) => boolean, +): Promise { + const reader = body.getReader(); + const frames: Frame[] = []; + let buffer = new Uint8Array(0); + try { + while (!enough(frames)) { + const step = await reader.read(); + if (step.done) break; + const joined = new Uint8Array(buffer.length + step.value.length); + joined.set(buffer); + joined.set(step.value, buffer.length); + buffer = joined; + while (buffer.length >= 5) { + const length = new DataView(buffer.buffer, buffer.byteOffset).getUint32(0); + if (buffer.length < 4 + length) break; + frames.push({ payload: buffer.slice(5, 4 + length), tag: buffer[4]! }); + buffer = buffer.slice(4 + length); + } + } + } finally { + await reader.cancel().catch(() => undefined); + } + return frames; +} + +describe.skipIf(!hasAndroidSdk)( + "Android screen stream (real emulator)", + { tags: ["slow", "android"] }, + () => { + it( + "streams a leased emulator's screen as meta, an avcC description and an H.264 key frame, leaves no server or jar once the viewer goes, keeps a still screen open 15 s past the silence bound with alive frames, and ends when its lease is released under a viewer", + { timeout: 480_000 }, + async () => { + const { env, port } = await withHttpDaemon((httpPort) => ({ + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port: httpPort }, + capabilities: { + stream: { bitrateKbps: 1500, enabled: true, fps: 30, maxDimension: 720 }, + }, + }, + driver: "real", + })); + const baseUrl = `http://127.0.0.1:${String(port)}`; + try { + const lease = await env.cli( + [ + "lease", + "--platform", + "android", + "--device", + "Pixel 9", + "--agent-id", + "android-stream", + "--detach", + ], + { timeout: DEVICE_READY_MS }, + ); + expect(lease.code, `lease failed: ${lease.stdout} ${lease.stderr}`).toBe(0); + const leaseId = (lease.json as { lease: { id: string } }).lease.id; + const created = await env.cli(["token", "create", "--role", "operator"]); + expect(created.code, created.stderr).toBe(0); + const authorization = `Bearer ${(created.json as { secret: string }).secret}`; + + const viewer = new AbortController(); + const response = await fetch( + `${baseUrl}/v1/stream/leases/${encodeURIComponent(leaseId)}`, + { + headers: { authorization }, + signal: viewer.signal, + }, + ); + expect( + response.status, + await response + .clone() + .text() + .catch(() => ""), + ).toBe(200); + expect(response.headers.get("simlock-stream-codec")).toBe("h264"); + if (response.body === null) throw new Error("the stream answered without a body"); + + const frames = await readFrames(response.body, (read) => + read.some((frame) => frame.tag === TAG.keyframe), + ); + viewer.abort(); + + expect(frames.map((frame) => frame.tag).slice(0, 3)).toEqual([ + TAG.meta, + TAG.description, + TAG.keyframe, + ]); + const meta = JSON.parse(new TextDecoder().decode(frames[0]!.payload)) as { + codec: string; + width: number; + height: number; + }; + expect(meta.codec).toBe("h264"); + expect(Math.max(meta.width, meta.height)).toBeLessThanOrEqual(720); + expect(Math.min(meta.width, meta.height)).toBeGreaterThan(0); + const description = frames[1]!.payload; + expect(description[0]).toBe(1); + expect(description[4]).toBe(0xff); + const key = frames[2]!.payload; + const firstNal = new DataView(key.buffer, key.byteOffset).getUint32(0); + expect(firstNal).toBeGreaterThan(0); + expect(firstNal).toBeLessThanOrEqual(key.length - 4); + + await waitFor( + async () => { + const processes = await env.cli(["adb", "shell", "ps", "-A", "-o", "ARGS"]); + const files = await env.cli(["adb", "shell", "ls", "/data/local/tmp"]); + return ( + processes.code === 0 && + !processes.stdout.includes("com.genymobile.scrcpy.Server") && + files.code === 0 && + !files.stdout.includes("simlock-scrcpy-") + ); + }, + { + interval: 500, + label: "no scrcpy server process and no pushed jar left on the emulator", + timeout: 30_000, + }, + ); + + // why: an untouched emulator's encoder sends nothing once its picture is still, so a stream + // held well past the 30 s silence bound proves the still screen is kept alive on a real + // device, and the lease released under that viewer proves the stop on a device too. + const watching = await fetch( + `${baseUrl}/v1/stream/leases/${encodeURIComponent(leaseId)}`, + { headers: { authorization } }, + ); + expect(watching.status).toBe(200); + if (watching.body === null) throw new Error("the stream answered without a body"); + const watched = watching.body.getReader(); + const tags: number[] = []; + let buffered = new Uint8Array(0); + let ended = false; + const reading = (async () => { + for (;;) { + const step = await watched.read().catch(() => ({ done: true, value: undefined })); + if (step.done || step.value === undefined) break; + const joined = new Uint8Array(buffered.length + step.value.length); + joined.set(buffered); + joined.set(step.value, buffered.length); + buffered = joined; + while (buffered.length >= 5) { + const length = new DataView(buffered.buffer, buffered.byteOffset).getUint32(0); + if (buffered.length < 4 + length) break; + tags.push(buffered[4]!); + buffered = buffered.slice(4 + length); + } + } + ended = true; + })(); + await new Promise((resolve) => setTimeout(resolve, 45_000)); + expect(ended, "a still screen's stream ended inside the silence bound").toBe(false); + expect(tags).toContain(TAG.keyframe); + // why: an `alive` frame is the evidence the screen went still and the stream was kept open + // for it; a screen that kept painting would pass the line above without proving anything. + expect(tags, "no alive frame: the screen never went still").toContain(TAG.alive); + + // why: the viewer leaves while the screen is still, the case where the server writes + // nothing and only the close can end it; the device must be left with no server and no jar. + await watched.cancel(); + await reading; + await waitFor( + async () => { + const processes = await env.cli(["adb", "shell", "ps", "-A", "-o", "ARGS"]); + const files = await env.cli(["adb", "shell", "ls", "/data/local/tmp"]); + return ( + processes.code === 0 && + !processes.stdout.includes("com.genymobile.scrcpy.Server") && + files.code === 0 && + !files.stdout.includes("simlock-scrcpy-") + ); + }, + { + interval: 500, + label: "no scrcpy server and no jar left after a viewer left a still screen", + timeout: 30_000, + }, + ); + + const last = await fetch(`${baseUrl}/v1/stream/leases/${encodeURIComponent(leaseId)}`, { + headers: { authorization }, + }); + expect(last.status).toBe(200); + const lastReader = last.body?.getReader(); + if (lastReader === undefined) throw new Error("the stream answered without a body"); + await lastReader.read(); + let lastEnded = false; + const draining = (async () => { + for (;;) { + const step = await lastReader.read().catch(() => ({ done: true })); + if (step.done) break; + } + lastEnded = true; + })(); + const release = await env.cli(["release", leaseId]); + expect(release.code, release.stderr).toBe(0); + await Promise.race([draining, new Promise((resolve) => setTimeout(resolve, 15_000))]); + expect(lastEnded, "the stream outlived its lease").toBe(true); + const hostShells = await new Promise((resolve) => { + execFile("pgrep", ["-fl", "[s]imlock-scrcpy-"], (_error, stdout) => resolve(stdout)); + }); + expect(hostShells, "a host adb shell for the capture outlived the lease").toBe(""); + } finally { + await env + .cli(["nuke", "--delete-devices", "--yes"], { timeout: 120_000 }) + .catch(() => undefined); + } + }, + ); + }, +); diff --git a/e2e/slow-ios-stream.test.ts b/e2e/slow-ios-stream.test.ts new file mode 100644 index 00000000..27f46f4e --- /dev/null +++ b/e2e/slow-ios-stream.test.ts @@ -0,0 +1,160 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +import { DEVICE_READY_MS, waitFor, withHttpDaemon } from "./helpers/index.js"; + +const execFileAsync = promisify(execFile); + +const TAG = { description: 0x01, keyframe: 0x02, meta: 0x05 } as const; + +interface Frame { + readonly tag: number; + readonly payload: Uint8Array; +} + +// why: the body is an endless stream, so this reads frames until it has what it came for and then +// lets go, rather than reading to an end that never comes. +async function readFrames( + reader: ReadableStreamDefaultReader, + enough: (frames: readonly Frame[]) => boolean, +): Promise { + const frames: Frame[] = []; + let buffer = new Uint8Array(0); + while (!enough(frames)) { + const step = await reader.read(); + if (step.done) break; + const joined = new Uint8Array(buffer.length + step.value.length); + joined.set(buffer); + joined.set(step.value, buffer.length); + buffer = joined; + while (buffer.length >= 5) { + const length = new DataView(buffer.buffer, buffer.byteOffset).getUint32(0); + if (buffer.length < 4 + length) break; + frames.push({ payload: buffer.slice(5, 4 + length), tag: buffer[4]! }); + buffer = buffer.slice(4 + length); + } + } + return frames; +} + +// why: `pgrep -x` matches the program's own name only; a pattern match would also find the shell +// running this check, which is a capture that looks alive and is not. +async function capturesRunning(): Promise { + const found = await execFileAsync("pgrep", ["-x", "simlock-screen-capture"]).catch(() => ({ + stdout: "", + })); + return found.stdout.split("\n").filter((line) => line.trim() !== "").length; +} + +describe.skipIf(process.platform !== "darwin")( + "iOS screen stream (real simulator)", + { tags: ["slow", "ios"] }, + () => { + it( + "streams a leased simulator's screen as meta, an avcC description and an H.264 key frame; a still screen stays open past the silence bound; a viewer leaving and a lease ending under a viewer each stop the capture", + { timeout: 480_000 }, + async () => { + expect(await capturesRunning(), "a capture was already running before the test").toBe(0); + const { env, port } = await withHttpDaemon((httpPort) => ({ + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port: httpPort }, + capabilities: { + stream: { bitrateKbps: 1500, enabled: true, fps: 30, maxDimension: 720 }, + }, + }, + driver: "real", + })); + const baseUrl = `http://127.0.0.1:${String(port)}`; + try { + const lease = await env.cli( + ["lease", "--platform", "ios", "--agent-id", "ios-stream", "--detach"], + { timeout: DEVICE_READY_MS }, + ); + expect(lease.code, `lease failed: ${lease.stdout} ${lease.stderr}`).toBe(0); + const leaseId = (lease.json as { lease: { id: string } }).lease.id; + const created = await env.cli(["token", "create", "--role", "operator"]); + expect(created.code, created.stderr).toBe(0); + const authorization = `Bearer ${(created.json as { secret: string }).secret}`; + const open = async (signal: AbortSignal) => { + const response = await fetch( + `${baseUrl}/v1/stream/leases/${encodeURIComponent(leaseId)}`, + { headers: { authorization }, signal }, + ); + expect( + response.status, + await response + .clone() + .text() + .catch(() => ""), + ).toBe(200); + if (response.body === null) throw new Error("the stream answered without a body"); + return { reader: response.body.getReader(), response }; + }; + + const viewer = new AbortController(); + const first = await open(viewer.signal); + expect(first.response.headers.get("simlock-stream-codec")).toBe("h264"); + const frames = await readFrames(first.reader, (read) => + read.some((frame) => frame.tag === TAG.keyframe), + ); + expect(await capturesRunning()).toBe(1); + viewer.abort(); + + expect(frames.map((frame) => frame.tag).slice(0, 3)).toEqual([ + TAG.meta, + TAG.description, + TAG.keyframe, + ]); + const meta = JSON.parse(new TextDecoder().decode(frames[0]!.payload)) as { + codec: string; + width: number; + height: number; + }; + expect(meta.codec).toBe("h264"); + expect(Math.max(meta.width, meta.height)).toBeLessThanOrEqual(720); + expect(Math.min(meta.width, meta.height)).toBeGreaterThan(0); + expect(frames[1]!.payload[0]).toBe(1); + await waitFor(async () => (await capturesRunning()) === 0, { + interval: 250, + label: "the capture stopped once its viewer went away", + timeout: 10_000, + }); + + const second = await open(new AbortController().signal); + await readFrames(second.reader, (read) => + read.some((frame) => frame.tag === TAG.keyframe), + ); + // why: nothing touches the simulator here, so its screen is still; the stream must stay + // open well past the 30 s silence bound on the capture's own idle repaints. + const stillUntil = Date.now() + 35_000; + const held = await readFrames(second.reader, () => Date.now() > stillUntil); + expect(held.length, "a still simulator's stream sent nothing for 35 s").toBeGreaterThan( + 0, + ); + const after = await second.reader.read(); + expect(after.done, "a still simulator's stream ended inside 35 s").toBe(false); + const release = await env.cli(["release", leaseId]); + expect(release.code, release.stderr).toBe(0); + const ended = await Promise.race([ + readFrames(second.reader, () => false).then( + () => "ended", + () => "ended", + ), + new Promise((resolve) => setTimeout(() => resolve("still streaming"), 15_000)), + ]); + expect(ended).toBe("ended"); + await waitFor(async () => (await capturesRunning()) === 0, { + interval: 250, + label: "the capture stopped once its lease ended under a connected viewer", + timeout: 10_000, + }); + } finally { + await env + .cli(["nuke", "--delete-devices", "--yes"], { timeout: 120_000 }) + .catch(() => undefined); + } + }, + ); + }, +); diff --git a/package.json b/package.json index 900b8087..c812d51c 100644 --- a/package.json +++ b/package.json @@ -38,9 +38,10 @@ } }, "scripts": { - "build": "pnpm run build:daemon && pnpm run build:ui", + "build": "pnpm run build:daemon && pnpm run build:ui && pnpm run build:capture", "build:daemon": "tsc -p tsconfig.build.json", "build:ui": "vite build ui", + "build:capture": "node scripts/build-capture.mjs", "build:e2e": "tsc -p e2e/fake-driver/tsconfig.json", "dev:ui": "vite ui", "test": "pnpm run test:unit && pnpm run test:timing && pnpm run test:e2e", diff --git a/scripts/build-capture.mjs b/scripts/build-capture.mjs new file mode 100644 index 00000000..5b6c7e51 --- /dev/null +++ b/scripts/build-capture.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// `pnpm build:capture`: puts the screen-capture artifacts beside the compiled daemon, which looks for +// them there. The scrcpy server is copied into dist/vendor/scrcpy/ once its SHA-256 matches the pin in +// vendor/scrcpy/SHA256SUMS.txt; a mismatch fails the build. On macOS with a Swift toolchain the iOS +// capture program in vendor/screen-capture/ is built and copied to dist/bin/simlock-screen-capture. +// Anywhere else it is skipped with a note, and the daemon reports iOS streams as unavailable. +// Each artifact ships with its upstream's NOTICE.md and LICENSES/ in dist/vendor//, because the +// package publishes dist/ and Apache-2.0 §4(a) asks that whoever gets a binary gets its license too. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, copyFileSync, cpSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const dist = join(root, "dist"); + +function copyNotices(dir) { + const target = join(dist, "vendor", dir); + mkdirSync(target, { recursive: true }); + copyFileSync(join(root, "vendor", dir, "NOTICE.md"), join(target, "NOTICE.md")); + cpSync(join(root, "vendor", dir, "LICENSES"), join(target, "LICENSES"), { recursive: true }); +} + +function copyScrcpyServer() { + const sums = readFileSync(join(root, "vendor/scrcpy/SHA256SUMS.txt"), "utf8").trim().split("\n"); + for (const line of sums) { + const [expected, name] = line.trim().split(/\s+/u); + const source = join(root, "vendor/scrcpy", name); + const actual = createHash("sha256").update(readFileSync(source)).digest("hex"); + if (actual !== expected) { + console.error(`build-capture: ${name} does not match its pinned SHA-256 (${actual})`); + process.exit(1); + } + const target = join(dist, "vendor/scrcpy", name); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(source, target); + } + copyNotices("scrcpy"); +} + +function buildIosCapture() { + if (process.platform !== "darwin") { + console.log("build-capture: not macOS, skipping the iOS screen-capture program"); + return; + } + const packagePath = join(root, "vendor/screen-capture"); + const scratch = join(packagePath, ".build/out"); + const built = spawnSync( + "swift", + ["build", "-c", "release", "--package-path", packagePath, "--scratch-path", scratch], + { stdio: "inherit" }, + ); + if (built.error !== undefined || built.status !== 0) { + console.warn( + "build-capture: could not build the iOS screen-capture program (is Xcode installed and selected?); " + + "iOS screen streams will be unavailable", + ); + return; + } + const target = join(dist, "bin/simlock-screen-capture"); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(join(scratch, "release/simlock-screen-capture"), target); + chmodSync(target, 0o755); + copyNotices("screen-capture"); +} + +copyScrcpyServer(); +buildIosCapture(); diff --git a/src/build-capture.test.ts b/src/build-capture.test.ts new file mode 100644 index 00000000..ec28188f --- /dev/null +++ b/src/build-capture.test.ts @@ -0,0 +1,130 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmodSync, + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const SCRIPT = join(import.meta.dirname, "..", "scripts", "build-capture.mjs"); +let root: string | undefined; + +afterEach(() => { + if (root !== undefined) rmSync(root, { force: true, recursive: true }); + root = undefined; +}); + +// why: a copy of the script beside a scratch vendor/ tree, because the script finds the repo from its +// own location and the claim is about what it does with the files it finds there. +function scratchRepo(server: string, digest: string): string { + root = mkdtempSync(join(tmpdir(), "simlock-build-capture-")); + mkdirSync(join(root, "scripts")); + mkdirSync(join(root, "vendor", "scrcpy"), { recursive: true }); + cpSync(SCRIPT, join(root, "scripts", "build-capture.mjs")); + writeFileSync(join(root, "vendor", "scrcpy", "scrcpy-server-v5.0.1"), server); + writeFileSync( + join(root, "vendor", "scrcpy", "SHA256SUMS.txt"), + `${digest} scrcpy-server-v5.0.1\n`, + ); + for (const [dir, license] of [ + ["scrcpy", "scrcpy.APACHE-2.0.txt"], + ["screen-capture", "serve-sim.APACHE-2.0.txt"], + ] as const) { + mkdirSync(join(root, "vendor", dir, "LICENSES"), { recursive: true }); + writeFileSync(join(root, "vendor", dir, "NOTICE.md"), `notice of ${dir}`); + writeFileSync(join(root, "vendor", dir, "LICENSES", license), `license of ${dir}`); + } + return root; +} + +// why: a `swift` that writes the program where `swift build` would, so the claim about what ships +// beside the iOS program is checked without a Swift toolchain. +function fakeSwift(repo: string): string { + const bin = join(repo, "fake-bin"); + mkdirSync(bin); + writeFileSync( + join(bin, "swift"), + [ + "#!/bin/sh", + 'while [ "$1" != "--scratch-path" ]; do shift; done', + '/bin/mkdir -p "$2/release"', + 'echo program > "$2/release/simlock-screen-capture"', + ].join("\n"), + ); + chmodSync(join(bin, "swift"), 0o755); + return bin; +} + +function sha256(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +describe("scripts/build-capture.mjs", () => { + it("fails the build, and copies nothing, when the scrcpy server does not match its pinned SHA-256", () => { + const repo = scratchRepo("tampered", sha256("the real server")); + const run = spawnSync(process.execPath, [join(repo, "scripts", "build-capture.mjs")], { + encoding: "utf8", + env: { ...process.env, PATH: "" }, + }); + expect(run.status).toBe(1); + expect(run.stderr).toContain("does not match its pinned SHA-256"); + expect(existsSync(join(repo, "dist", "vendor", "scrcpy", "scrcpy-server-v5.0.1"))).toBe(false); + }); + + it("copies a scrcpy server that matches its pin into dist, byte for byte", () => { + const repo = scratchRepo("the real server", sha256("the real server")); + execFileSync(process.execPath, [join(repo, "scripts", "build-capture.mjs")], { + env: { ...process.env, PATH: "" }, + stdio: "pipe", + }); + expect( + readFileSync(join(repo, "dist", "vendor", "scrcpy", "scrcpy-server-v5.0.1"), "utf8"), + ).toBe("the real server"); + }); + + it("ships the scrcpy server with its NOTICE.md and every license it needs", () => { + const repo = scratchRepo("the real server", sha256("the real server")); + execFileSync(process.execPath, [join(repo, "scripts", "build-capture.mjs")], { + env: { ...process.env, PATH: "" }, + stdio: "pipe", + }); + const shipped = join(repo, "dist", "vendor", "scrcpy"); + expect(readFileSync(join(shipped, "NOTICE.md"), "utf8")).toBe("notice of scrcpy"); + expect(readFileSync(join(shipped, "LICENSES", "scrcpy.APACHE-2.0.txt"), "utf8")).toBe( + "license of scrcpy", + ); + }); + + it.skipIf(process.platform !== "darwin")( + "ships the iOS capture program with serve-sim's NOTICE.md and license, and leaves them out when it is not built", + () => { + const repo = scratchRepo("the real server", sha256("the real server")); + const run = (path: string) => + execFileSync(process.execPath, [join(repo, "scripts", "build-capture.mjs")], { + env: { ...process.env, PATH: path }, + stdio: "pipe", + }); + const shipped = join(repo, "dist", "vendor", "screen-capture"); + + run(""); + expect(existsSync(join(repo, "dist", "bin", "simlock-screen-capture"))).toBe(false); + expect(existsSync(shipped)).toBe(false); + + run(fakeSwift(repo)); + expect(existsSync(join(repo, "dist", "bin", "simlock-screen-capture"))).toBe(true); + expect(readFileSync(join(shipped, "NOTICE.md"), "utf8")).toBe("notice of screen-capture"); + expect(readFileSync(join(shipped, "LICENSES", "serve-sim.APACHE-2.0.txt"), "utf8")).toBe( + "license of screen-capture", + ); + }, + ); +}); diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index c675c993..ba8a7689 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -31,7 +31,7 @@ See also: [leasing](../leasing/AGENTS.md) (lease flow, when reclaim runs), [warm | **Startup, health, drivers** | | | `startup-read.ts`, `startup-converger.ts` | `readStartup`; `StartupConverger`, the device half of startup | | `doctor.ts`, `host-facts.ts` | `Doctor` findings and fixes; `HostFactsReader` for status | -| `driver.ts`, `driver-catalog.ts` | `Driver` interface, errors, `DiskSpaceGuard`; `DriverCatalog` | +| `driver.ts`, `driver-catalog.ts`, `screen.ts` | `Driver`, errors, `DiskSpaceGuard`; `DriverCatalog`; `ScreenCapture`, frame tags | | `catalog-match.ts` | Model name, class and paired runtimes against a catalog entry | | `component-installer.ts` | `ComponentInstaller`: install, list, remove | | `physical-enrollment.ts`, `physical-presence.ts` | `device add`/`remove`; every presence read, `latestPresence` | diff --git a/src/core/driver-catalog.ts b/src/core/driver-catalog.ts index 4129dc2d..1bd3d797 100644 --- a/src/core/driver-catalog.ts +++ b/src/core/driver-catalog.ts @@ -10,6 +10,7 @@ import type { PhysicalDriverDevice, VirtualDriverDevice, } from "./driver.js"; +import { type ScreenFeed, ScreenNotSupportedError, type ScreenSettings } from "./screen.js"; import { stableError } from "./stable-error.js"; /** Thrown when no installed driver can serve the requested platform. */ @@ -43,6 +44,11 @@ export interface PlatformCatalog extends DriverCatalogEntry { readonly classDefaults: Readonly>>; } +export interface ScreenAvailability { + readonly platform: Platform; + readonly reason?: string; +} + /** Immutable platform-to-driver lookup used by the lease path. */ export class DriverCatalog { readonly #drivers: ReadonlyMap; @@ -87,6 +93,37 @@ export class DriverCatalog { throw new UnknownPassthroughToolError(tool); } + async openScreen( + platform: Platform, + device: VirtualDriverDevice | PhysicalDriverDevice, + settings: ScreenSettings, + signal: AbortSignal, + ): Promise { + const capture = this.#drivers.get(platform)?.screenCapture; + if (capture === undefined) { + throw new ScreenNotSupportedError(`A ${platform} device's screen cannot be streamed`); + } + return await capture.open(device, settings, signal); + } + + async screenAvailability(): Promise { + return await Promise.all( + [...this.#drivers.values()].flatMap((driver) => + driver.screenCapture === undefined + ? [] + : [ + driver.screenCapture + .unavailable() + .then((reason) => + reason === undefined + ? { platform: driver.platform } + : { platform: driver.platform, reason }, + ), + ], + ), + ); + } + async resolveSpec(request: ExactDeviceRequest): Promise { return this.get(request.platform).resolveSpec(request); } diff --git a/src/core/driver.ts b/src/core/driver.ts index 035887da..82a622a9 100644 --- a/src/core/driver.ts +++ b/src/core/driver.ts @@ -1,6 +1,7 @@ import type { EventMap } from "../bus/index.js"; import type { Filesystem } from "../ports/index.js"; import type { RootRejectionReason } from "./device-root.js"; +import type { ScreenCapture } from "./screen.js"; import type { AbsentReason, DeviceClass, @@ -354,6 +355,9 @@ export interface Driver { dispose?(): Promise; /** Spelled `| undefined` so a driver that decides at construction can still say it has none. */ readonly passthroughTools?: readonly string[] | undefined; + // why: optional because a platform whose screen cannot be taken leaves it out, and the stream + // module learns that from its absence rather than from a platform name. + readonly screenCapture?: ScreenCapture | undefined; /** why: `device.exec` has no pseudo-terminal (ADR 0005 §19c), so a driver may refuse there what it allows locally. */ passthrough?( tool: string, diff --git a/src/core/fake-driver.ts b/src/core/fake-driver.ts index dcb9d55d..d291927a 100644 --- a/src/core/fake-driver.ts +++ b/src/core/fake-driver.ts @@ -38,6 +38,7 @@ import { UnknownModelError, uninstallPlan, } from "./driver.js"; +import type { ScreenCapture } from "./screen.js"; export type FakeDriverOperation = | "resolveSpec" @@ -126,6 +127,7 @@ export interface FakeDriverOptions { readonly leaseEnvironment?: Readonly>; /** why: absent by default, so two fakes in one catalog do not both answer to the same tool. */ readonly passthroughTools?: readonly string[]; + readonly screenCapture?: ScreenCapture; readonly passthrough?: ( tool: string, device: VirtualDriverDevice | PhysicalDriverDevice | undefined, @@ -188,6 +190,7 @@ export class FakeDriver implements Driver { readonly #latencyMs: FakeDriverOptions["latencyMs"]; readonly #leaseEnvironment: Readonly>; readonly passthroughTools: readonly string[] | undefined; + readonly screenCapture: ScreenCapture | undefined; readonly #passthrough: FakeDriverOptions["passthrough"]; #nextDeviceNumber = 1; readonly #pendingMakeReady: (() => void)[] = []; @@ -225,6 +228,7 @@ export class FakeDriver implements Driver { this.#leaseEnvironment = options.leaseEnvironment ?? {}; this.#legacyDevices = new Map(Object.entries(options.legacyDevices ?? {})); this.passthroughTools = options.passthroughTools; + this.screenCapture = options.screenCapture; this.#passthrough = options.passthrough; this.platform = options.platform; this.deviceRoot = options.deviceRoot ?? `/fake/${options.platform}`; diff --git a/src/core/index.ts b/src/core/index.ts index 39d0686b..f857c564 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -90,8 +90,21 @@ export { DriverCatalog, NoDriverError, type ModelPreferences, + type ScreenAvailability, UnknownPassthroughToolError, } from "./driver-catalog.js"; +export { + encodeScreenFrame, + encodeScreenMeta, + type ScreenCapture, + type ScreenFeed, + SCREEN_FRAME_HEADER_BYTES, + SCREEN_FRAME_TAG, + type ScreenMeta, + ScreenNotSupportedError, + type ScreenSettings, + ScreenUnavailableError, +} from "./screen.js"; // fallow-ignore-next-line unused-type -- wire-visible rejection vocabulary for Simlock's own adb server. export type { AdbServerRejectionReason, DriverRejectionReason } from "./driver.js"; export { diff --git a/src/core/screen.test.ts b/src/core/screen.test.ts new file mode 100644 index 00000000..a00c1c2d --- /dev/null +++ b/src/core/screen.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import { FakeClock } from "../ports/index.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import { FakeDriver } from "./fake-driver.js"; +import { + encodeScreenFrame, + encodeScreenMeta, + type ScreenCapture, + ScreenNotSupportedError, + SCREEN_FRAME_TAG, +} from "./screen.js"; + +const settings = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; +const device = { address: "a", deviceId: "d", driverData: {}, physical: false } as const; + +describe("encodeScreenFrame", () => { + it("prefixes the payload with a big-endian length that counts the tag byte", () => { + const frame = encodeScreenFrame(SCREEN_FRAME_TAG.keyframe, new Uint8Array([9, 8, 7])); + expect([...frame]).toEqual([0, 0, 0, 4, SCREEN_FRAME_TAG.keyframe, 9, 8, 7]); + }); + + it("writes a meta frame whose payload is the meta as JSON", () => { + const frame = encodeScreenMeta({ codec: "h264", fps: 30, height: 720, width: 330 }); + expect(frame[4]).toBe(SCREEN_FRAME_TAG.meta); + expect(JSON.parse(new TextDecoder().decode(frame.subarray(5)))).toEqual({ + codec: "h264", + fps: 30, + height: 720, + width: 330, + }); + }); +}); + +describe("DriverCatalog screens", () => { + it("opens a screen through the driver of the device's platform, with the caller's settings and signal", async () => { + const opened: string[] = []; + let passed: { settings: unknown; signal: AbortSignal } | undefined; + const capture: ScreenCapture = { + open: async (target, given, signal) => { + opened.push(target.deviceId); + passed = { settings: given, signal }; + return { body: new ReadableStream(), close: async () => undefined }; + }, + unavailable: async () => undefined, + }; + const catalog = new DriverCatalog([ + new FakeDriver({ clock: new FakeClock(), platform: "ios", screenCapture: capture }), + new FakeDriver({ clock: new FakeClock(), platform: "android" }), + ]); + const signal = new AbortController().signal; + await catalog.openScreen("ios", device, settings, signal); + expect(opened).toEqual(["d"]); + expect(passed?.settings).toEqual(settings); + expect(passed?.signal).toBe(signal); + }); + + it("refuses a platform whose driver offers no screen as not supported", async () => { + const catalog = new DriverCatalog([ + new FakeDriver({ clock: new FakeClock(), platform: "android" }), + ]); + await expect( + catalog.openScreen("android", device, settings, new AbortController().signal), + ).rejects.toBeInstanceOf(ScreenNotSupportedError); + }); + + it("reports availability only for drivers that offer a screen, with the reason one cannot", async () => { + const catalog = new DriverCatalog([ + new FakeDriver({ + clock: new FakeClock(), + platform: "ios", + screenCapture: { + open: async () => { + throw new Error("unused"); + }, + unavailable: async () => "No capture program at /x", + }, + }), + new FakeDriver({ clock: new FakeClock(), platform: "android" }), + ]); + expect(await catalog.screenAvailability()).toEqual([ + { platform: "ios", reason: "No capture program at /x" }, + ]); + }); +}); diff --git a/src/core/screen.ts b/src/core/screen.ts new file mode 100644 index 00000000..1f0b7354 --- /dev/null +++ b/src/core/screen.ts @@ -0,0 +1,81 @@ +import type { PhysicalDriverDevice, VirtualDriverDevice } from "./driver.js"; + +// why: the one table in TypeScript for a capture's bytes: the Android driver writes it and the stream +// module reads it. The Swift capture program, the console and the e2e tests keep their own copies; +// the console's test pins its copy to this one, so a tag that drifts fails a test instead of a viewer +// decoding garbage. +export const SCREEN_FRAME_TAG = { + description: 0x01, + keyframe: 0x02, + delta: 0x03, + seed: 0x04, + meta: 0x05, + // why: an empty frame a capture sends when it is alive with nothing new to paint, so a reader's + // silence bound tells a still screen from a dead capture; a reader that has no use for it skips it. + alive: 0x06, +} as const; + +export const SCREEN_FRAME_HEADER_BYTES = 5; + +export interface ScreenMeta { + readonly codec: "h264" | "mjpeg"; + readonly width: number; + readonly height: number; + readonly fps: number; +} + +export interface ScreenSettings { + readonly fps: number; + readonly maxDimension: number; + readonly bitrateKbps: number; +} + +// why: the one shape every platform's capture is handed out in, so the module that serves a stream +// never learns how a picture was taken. `body` carries frames in the table above, meta first; the +// stream module, not the driver, decides how long it may stay quiet. +export interface ScreenFeed { + readonly body: ReadableStream; + // why: resolves once the device holds nothing of this capture any more -- a server process, a + // forward, a pushed file -- so a lease's reclaim can wait for it rather than race it. Bounded by + // the driver and idempotent: a viewer leaving, the lease ending and the body ending all call it. + close(): Promise; +} + +export interface ScreenCapture { + // why: a reason when this driver cannot capture right now (its program is missing), so a + // status read can say why without opening a stream to find out. + unavailable(): Promise; + // why: `signal` aborts an open still in progress -- the viewer left or the lease ended -- so a + // driver stops pushing files to and starting processes on a device that is being handed back. + open( + device: VirtualDriverDevice | PhysicalDriverDevice, + settings: ScreenSettings, + signal: AbortSignal, + ): Promise; +} + +export class ScreenNotSupportedError extends Error { + constructor(message: string) { + super(message); + this.name = "ScreenNotSupportedError"; + } +} + +export class ScreenUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = "ScreenUnavailableError"; + } +} + +export function encodeScreenFrame(tag: number, payload: Uint8Array): Uint8Array { + const frame = new Uint8Array(SCREEN_FRAME_HEADER_BYTES + payload.length); + new DataView(frame.buffer).setUint32(0, payload.length + 1); + frame[4] = tag; + frame.set(payload, SCREEN_FRAME_HEADER_BYTES); + return frame; +} + +export function encodeScreenMeta(meta: ScreenMeta): Uint8Array { + return encodeScreenFrame(SCREEN_FRAME_TAG.meta, new TextEncoder().encode(JSON.stringify(meta))); +} diff --git a/src/daemon/AGENTS.md b/src/daemon/AGENTS.md index 175432b8..06d740de 100644 --- a/src/daemon/AGENTS.md +++ b/src/daemon/AGENTS.md @@ -120,7 +120,7 @@ Security model: cooperative identity, not a hostile-process boundary. load reading. Backoff from 1 s, ×2, cap 30 s, jittered; retries forever. - `DaemonServer.acceptUplink` serves it as one more connection, role `admin`. - `relay.request` runs only on that connection; targets come from `relayTargets` - (today `agent-device`). See [gateway](../gateway/AGENTS.md). + (today `agent-device` and `stream`). See [gateway](../gateway/AGENTS.md). ## Logging and event records diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 4dcee9d2..58c74c15 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -1,6 +1,6 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { EVENT_FILE_NAME, EventBus, EventHistory, type EventBusLogger } from "../bus/index.js"; import { fitHostFacts, fitHostLoad } from "../contract/index.js"; @@ -72,7 +72,9 @@ import { NodeProcessRunner, NodeProcessSupervisor, NodeSystemStats, + NodeBytePipeLauncher, NodeTcpProbe, + NodeTcpStreamConnector, resolveDaemonSocketPath, readPackageVersion, resolveSimlockHome, @@ -346,6 +348,8 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise driver.platform), processRunner, processSupervisor, + registry, + screens: new DriverCatalog(drivers), tcpProbe, }, definitions: moduleDefinitions, @@ -790,7 +794,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise logger.debug("Screen capture", { line }), + }, onSlimSkipped: (fact) => { logger.warn("Skipped iOS device slim", { deviceId: fact.deviceId, @@ -1158,8 +1179,10 @@ async function discoverAndroidDriver( instanceId: options.instanceId, processRunner: options.processRunner, processSupervisor: options.processSupervisor, + scrcpyServerPath: PACKAGED_SCRCPY_SERVER, simlockHome: options.simlockHome, tcpProbe: options.tcpProbe, + tcpStreamConnector: new NodeTcpStreamConnector(), // Ambient like `homedir()` above, and read here rather than in the driver so the // composition root stays the only place that touches process state. ...(process.getuid === undefined ? {} : { uid: process.getuid() }), diff --git a/src/daemon/relay-gateway.e2e.test.ts b/src/daemon/relay-gateway.e2e.test.ts index 90be06e2..2b99d176 100644 --- a/src/daemon/relay-gateway.e2e.test.ts +++ b/src/daemon/relay-gateway.e2e.test.ts @@ -128,7 +128,9 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { await cleanUp(daemons, directories); }); - async function startPair(options: { readonly deadlineMs?: number } = {}) { + async function startPair( + options: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) { const fixture: WorkerFixture = await startWorker(directories, daemons, { "agent-device": target, echo: target, @@ -147,10 +149,13 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } : undefined, }; - const makeRelay = (relayOptions: { readonly deadlineMs?: number } = {}) => + const makeRelay = ( + relayOptions: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) => new GatewayRelay({ directory, ...(relayOptions.deadlineMs === undefined ? {} : { deadlineMs: relayOptions.deadlineMs }), + ...(relayOptions.idleMs === undefined ? {} : { idleMs: relayOptions.idleMs }), }); const relay = makeRelay(options); const opened = () => pair.gatewayTap.binaries(TYPE.OPEN).length; @@ -548,7 +553,7 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { await poll(() => signal?.aborted).toBe(true); }, 30_000); - it("a target that never answers is cancelled after a 200 ms deadline, and the caller gets WORKER_UNREACHABLE with message `worker did not finish the relayed request within 200 ms`; one that stalls after its head gets its body stream errored with that message", async () => { + it("a target that never answers is cancelled after a 200 ms deadline, and the caller gets WORKER_UNREACHABLE with message `worker did not finish the relayed request within 200 ms`; one that stalls after its head, under a 10 s deadline and a 200 ms idle bound, gets its body stream errored with WORKER_UNREACHABLE `worker sent nothing on the relayed response for 200 ms`", async () => { let signal: AbortSignal | undefined; behavior = async (request) => { signal = request.signal; @@ -569,7 +574,7 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { }), ), ); - const stalling = await startPair({ deadlineMs: 200 }); + const stalling = await startPair({ deadlineMs: 10_000, idleMs: 200 }); const response = await stalling.relay.relay(WORKER_ID, buildCall({})); const reader = response.body?.getReader(); await reader?.read(); @@ -577,7 +582,11 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { const message = `worker ${WORKER_ID} did not finish the relayed request within 200 ms`; expect(noHead).toMatchObject({ code: "WORKER_UNREACHABLE", message, state: "rejected" }); - expect(stalled).toMatchObject({ message, state: "rejected" }); + expect(stalled).toMatchObject({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} sent nothing on the relayed response for 200 ms`, + state: "rejected", + }); }, 30_000); it("closing the uplink mid-relay fails the caller with WORKER_UNREACHABLE, or errors its body stream after the head, and aborts the target's request signal", async () => { diff --git a/src/daemon/stream-gateway.e2e.test.ts b/src/daemon/stream-gateway.e2e.test.ts new file mode 100644 index 00000000..58fcd14f --- /dev/null +++ b/src/daemon/stream-gateway.e2e.test.ts @@ -0,0 +1,204 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { encodeScreenFrame, encodeScreenMeta, SCREEN_FRAME_TAG } from "../core/index.js"; +import type { ScreenCapture } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { MemoryFilesystem, NoopLogger, SystemClock } from "../ports/index.js"; +import type { DispatchSession } from "./dispatch.js"; +import { startDaemon } from "./main.js"; +import type { DaemonServer } from "./server.js"; + +async function freeLoopbackPort(): Promise { + const server = createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + await new Promise((resolve) => server.close(() => resolve())); + if (typeof address !== "object" || address === null) throw new Error("no port bound"); + return address.port; +} + +const admin: DispatchSession = { + manageEventSubscription: () => undefined, + principal: "test-operator", + role: "admin", +}; + +const agent: DispatchSession = { + manageEventSubscription: () => undefined, + principal: "agent-1", + role: "agent", +}; + +// why: a screen that paints a meta frame and a key frame, then keeps the stream open until it is +// closed, which is what a real capture does; `closes` is the test's witness that a stop reached it. +function paintingScreen(): { readonly capture: ScreenCapture; readonly closes: () => number } { + let closes = 0; + return { + capture: { + open: async () => { + let controller!: ReadableStreamDefaultController; + const body = new ReadableStream({ + start: (given) => { + controller = given; + given.enqueue(encodeScreenMeta({ codec: "h264", fps: 30, height: 64, width: 32 })); + given.enqueue( + encodeScreenFrame(SCREEN_FRAME_TAG.keyframe, new Uint8Array([0, 0, 0, 1, 7])), + ); + }, + }); + return { + body, + close: async () => { + closes += 1; + try { + controller.close(); + } catch { + // why: a body its reader already cancelled is closed; the stop still counts. + } + }, + }; + }, + unavailable: async () => undefined, + }, + closes: () => closes, + }; +} + +describe("a screen stream through a gateway", () => { + const daemons: DaemonServer[] = []; + const directories: string[] = []; + + afterEach(async () => { + await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + directories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })), + ); + }); + + async function fleet(workerStreams: boolean) { + const gatewayDirectory = await mkdtemp(join(tmpdir(), "simlock-e2e-stream-gateway-")); + directories.push(gatewayDirectory); + const port = await freeLoopbackPort(); + const gateway = await startDaemon({ + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port }, + mode: "gateway", + capabilities: { stream: { enabled: true } }, + }, + dataDirectory: gatewayDirectory, + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + statePath: join(gatewayDirectory, "state.json"), + version: "1.0.0-e2e", + }); + daemons.push(gateway); + const token = (await gateway.dispatch("token.create", { label: "w", role: "worker" }, admin)) + .secret; + const operator = ( + await gateway.dispatch("token.create", { label: "op", role: "operator" }, admin) + ).secret; + const workerDirectory = await mkdtemp(join(tmpdir(), "simlock-e2e-stream-worker-")); + directories.push(workerDirectory); + const screen = paintingScreen(); + const worker = await startDaemon({ + configOverrides: { + gateway: { label: "w", token, url: `ws://127.0.0.1:${String(port)}` }, + capabilities: { stream: { enabled: workerStreams } }, + }, + dataDirectory: workerDirectory, + drivers: [ + new FakeDriver({ + availableOsVersions: ["18.0"], + clock: new SystemClock(), + knownModels: ["Pixel-A"], + platform: "android", + screenCapture: screen.capture, + }), + ], + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + statePath: join(workerDirectory, "state.json"), + version: "1.0.0-e2e", + }); + daemons.push(worker); + await vi.waitFor( + async () => { + const { workers } = await gateway.dispatch("worker.list", {}, admin); + expect(workers[0]?.catalog?.length).toBeGreaterThan(0); + }, + { timeout: 10_000 }, + ); + const granted = await gateway.dispatch( + "lease.request", + { model: "Pixel-A", platform: "android" }, + agent, + ); + const leaseId = (granted as { lease: { id: string } }).lease.id; + const stream = (signal?: AbortSignal) => + fetch(`http://127.0.0.1:${String(port)}/v1/stream/leases/${encodeURIComponent(leaseId)}`, { + headers: { authorization: `Bearer ${operator}` }, + ...(signal === undefined ? {} : { signal }), + }); + return { gateway, leaseId, screen, stream }; + } + + it( + "relays the worker's picture, and the lease ending on the gateway stops the capture on the worker", + { timeout: 30_000 }, + async () => { + const { gateway, leaseId, screen, stream } = await fleet(true); + const response = await stream(); + expect(response.status).toBe(200); + expect(response.headers.get("simlock-stream-codec")).toBe("h264"); + expect(response.headers.get("simlock-stream-width")).toBe("32"); + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("no body"); + const first = await reader.read(); + expect(first.value?.[4]).toBe(SCREEN_FRAME_TAG.meta); + + await gateway.dispatch("lease.release", { leaseId }, agent); + await vi.waitFor(() => expect(screen.closes()).toBe(1), { timeout: 10_000 }); + await vi.waitFor( + async () => { + const step = await reader.read().catch(() => ({ done: true })); + expect(step.done).toBe(true); + }, + { timeout: 10_000 }, + ); + }, + ); + + it( + "a viewer leaving the gateway stops the capture on the worker", + { timeout: 30_000 }, + async () => { + const { screen, stream } = await fleet(true); + const viewer = new AbortController(); + const response = await stream(viewer.signal); + expect(response.status).toBe(200); + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("no body"); + await reader.read(); + viewer.abort(); + await vi.waitFor(() => expect(screen.closes()).toBe(1), { timeout: 10_000 }); + }, + ); + + it( + "answers 503 STREAM_UNAVAILABLE naming the worker when that worker does not stream", + { timeout: 30_000 }, + async () => { + const { stream } = await fleet(false); + const response = await stream(); + expect(response.status).toBe(503); + const body = (await response.json()) as { error: { code: string; message: string } }; + expect(body.error.code).toBe("STREAM_UNAVAILABLE"); + expect(body.error.message).toMatch(/^Worker /u); + }, + ); +}); diff --git a/src/drivers/AGENTS.md b/src/drivers/AGENTS.md index 3000a16d..4f830800 100644 --- a/src/drivers/AGENTS.md +++ b/src/drivers/AGENTS.md @@ -17,6 +17,7 @@ Manuals: [Android](android/AGENTS.md), [iOS](ios/AGENTS.md). Flows: [device sele | `inspectPhysical`, `listPresentPhysical` | Enrollment read and presence of physical devices | | `leaseEnvironment`, `virtualLeaseEnvironment`, `passthrough` | What a holder needs to reach its device | | `dispose` | Daemon stop. Android reaps its adb server here | +| `screenCapture` (optional) | A device's screen as core's frame envelope (`core/screen.ts`); absent = cannot stream | ## Rules both drivers keep diff --git a/src/drivers/android/AGENTS.md b/src/drivers/android/AGENTS.md index edf8851e..43e1d458 100644 --- a/src/drivers/android/AGENTS.md +++ b/src/drivers/android/AGENTS.md @@ -12,6 +12,8 @@ Parent: [drivers](../AGENTS.md). Flows: [device selection](../../../docs/interna | `index.ts` | `AndroidDriver`: every `Driver` verb, console-port allocator, passthrough rules | | `adb-server.ts` | `AdbServerSupervisor`: start, adopt, reap Simlock's adb server | | `adb-registrar.ts` | `AdbRegistrar`: announces an emulator (`host:emulator:`) | +| `screen/` | Screen capture: scrcpy client ([screen](screen/AGENTS.md)) | +| `adb-host-protocol.ts` | adb host-protocol framing (`adbHostRequest`, `OKAY`/`FAIL`), for both callers | | `default-adb-server.ts` | `DefaultAdbServerGate`: protocol check of the server on 5037 | | `physical-handler.ts` | `AndroidPhysicalHandler`: enrol, presence, reclaim, passthrough for phones | | `device-profile-source.ts` | `DeviceProfileRegistry`; SDK and devices.xml profile sources | @@ -106,9 +108,11 @@ Lease environment: virtual `ANDROID_ADB_SERVER_PORT=`; physical No slim: `resolveSpec` ignores `mode` and returns a full spec. `fresh` is core's. -Plain facts: `avdmanager list device` runs on every `listCatalog`, every resolve of a model -the library lacks, and every library provision (seed); nothing is cached. The sdkmanager -catalog is never read. Existing devices are not re-checked against the fit rule. +Screen — `screenCapture`, only when `create` got `scrcpyServerPath` and `tcpStreamConnector`: [screen](screen/AGENTS.md). + +Plain facts: `avdmanager list device` runs on every `listCatalog`, every resolve of a model the library +lacks, and every library provision (seed); nothing is cached. The sdkmanager catalog is never read. +Existing devices are not re-checked against the fit rule. ## State and timers @@ -135,9 +139,8 @@ Reads only: the user's AVD home, devices.xml, the SDK. ## Known gaps -- **Simlock's adb server is stopped by pid only.** `ADB_REJECT_KILL_SERVER=1` refuses Simlock's own - `kill-server`. A listening server with no record is `occupied`: find it with `lsof`, or move - `drivers.android.adbServerPort`. Two instances need distinct ports. +- **Simlock's adb server is stopped by pid only.** `ADB_REJECT_KILL_SERVER=1` refuses its `kill-server`. A + server with no record is `occupied`: find it with `lsof`, or move `drivers.android.adbServerPort` (one per instance). - **A physical command can restart a 5037 server of another protocol (ADR 0022 §5).** One started between the gate and the command is restarted by adb's client. Run every adb on the host from one platform-tools. - **Port checks are partial.** After a restart `shutdown` sends `emu kill` to the stored serial diff --git a/src/drivers/android/adb-host-protocol.ts b/src/drivers/android/adb-host-protocol.ts new file mode 100644 index 00000000..314787df --- /dev/null +++ b/src/drivers/android/adb-host-protocol.ts @@ -0,0 +1,13 @@ +// why: adb's host protocol is a length-prefixed request answered by `OKAY` or by `FAIL` and a +// length-prefixed reason, the same framing in both directions: +// https://android.googlesource.com/platform/packages/modules/adb/+/refs/heads/main/adb_io.cpp +const LENGTH_PREFIX_DIGITS = 4; + +export const ADB_OKAY = "OKAY"; +export const ADB_FAIL = "FAIL"; +export const ADB_STATUS_BYTES = 4; +export const ADB_LENGTH_BYTES = LENGTH_PREFIX_DIGITS; + +export function adbHostRequest(service: string): string { + return `${service.length.toString(16).padStart(LENGTH_PREFIX_DIGITS, "0")}${service}`; +} diff --git a/src/drivers/android/adb-registrar.ts b/src/drivers/android/adb-registrar.ts index f69e9cd4..253e0ad8 100644 --- a/src/drivers/android/adb-registrar.ts +++ b/src/drivers/android/adb-registrar.ts @@ -1,10 +1,7 @@ import type { TcpProbe } from "../../ports/index.js"; +import { ADB_FAIL, adbHostRequest } from "./adb-host-protocol.js"; -/** adb frames a host service as four hex digits of length, big-endian, then the service. */ -const LENGTH_PREFIX_DIGITS = 4; const SERVICE_PREFIX = "host:emulator:"; -/** An error reply is `FAIL`; a success reply is `OKAY` or nothing. */ -const FAILURE_PREFIX = "FAIL"; export interface AdbRegistrarOptions { readonly serverPort: number; @@ -36,16 +33,12 @@ export class AdbRegistrar { */ async register(adbPort: number): Promise { const service = `${SERVICE_PREFIX}${adbPort}`; - const reply = await this.#options.tcp.send(this.#options.serverPort, frame(service)); + const reply = await this.#options.tcp.send(this.#options.serverPort, adbHostRequest(service)); // adb answers `OKAY`, or closes without answering at all ("we don't even need to send a // reply" in its own source), so silence is success here rather than a timeout to report. - if (reply.startsWith(FAILURE_PREFIX)) { - throw new Error(`adb refused ${service}: ${reply.slice(FAILURE_PREFIX.length)}`); + if (reply.startsWith(ADB_FAIL)) { + throw new Error(`adb refused ${service}: ${reply.slice(ADB_FAIL.length)}`); } } } - -function frame(service: string): string { - return `${service.length.toString(16).padStart(LENGTH_PREFIX_DIGITS, "0")}${service}`; -} diff --git a/src/drivers/android/index.test.ts b/src/drivers/android/index.test.ts index 94e6970c..636dd440 100644 --- a/src/drivers/android/index.test.ts +++ b/src/drivers/android/index.test.ts @@ -15,6 +15,7 @@ import { FakeClock, FakeProcessSupervisor, FakeTcpProbe, + FakeTcpStreamConnector, type Filesystem, MemoryFilesystem, NodeFilesystem, @@ -2973,6 +2974,83 @@ describe("AndroidDriver", () => { }); }); +describe("AndroidDriver screen capture", () => { + const serverPath = "/simlock/vendor/scrcpy-server-v5.0.1"; + const running = { + address: "emulator-5586", + deviceId: "simlock_one", + driverData: { avdName: "simlock_one", configHash: "h", port: 5586, serial: "emulator-5586" }, + physical: false, + } as const; + const settings = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; + + it("is offered only when both the server path and a TCP stream connector are given", async () => { + const both = await createDriver(await androidFilesystem(), new ScriptedProcessRunner([]), { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + expect(both.screenCapture).toBeDefined(); + const noPath = await createDriver(await androidFilesystem(), new ScriptedProcessRunner([]), { + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + expect(noPath.screenCapture).toBeUndefined(); + const noConnector = await createDriver( + await androidFilesystem(), + new ScriptedProcessRunner([]), + { + scrcpyServerPath: serverPath, + }, + ); + expect(noConnector.screenCapture).toBeUndefined(); + }); + + it("pushes the server with this driver's adb, its scoped environment and the device's serial", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp("/simlock/vendor"); + await filesystem.writeFileAtomic(serverPath, "jar"); + const runner = new ScriptedProcessRunner([ + { + match: { + args: ["-s", "emulator-5586", "push", serverPath, /simlock-scrcpy-/], + command: binaries.adb, + }, + result: { code: 1, stderr: "error: closed", stdout: "" }, + }, + { + match: { + args: ["-s", "emulator-5586", "shell", "rm", "-f", /simlock-scrcpy-/], + command: binaries.adb, + }, + }, + ]); + const driver = await createDriver(filesystem, runner, { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + await expect( + driver.screenCapture?.open(running, settings, new AbortController().signal), + ).rejects.toThrow(/error: closed/); + expect(runner.calls[0]?.options.env).toEqual(scopedEnv); + expect(runner.calls[1]?.options.env).toEqual(scopedEnv); + }); + + it("refuses an emulator that is not running, before running anything", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp("/simlock/vendor"); + await filesystem.writeFileAtomic(serverPath, "jar"); + const runner = new ScriptedProcessRunner([]); + const driver = await createDriver(filesystem, runner, { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + const stopped = { ...running, driverData: { ...running.driverData, port: 0, serial: "" } }; + await expect( + driver.screenCapture?.open(stopped, settings, new AbortController().signal), + ).rejects.toThrow(/not running/); + expect(runner.calls).toEqual([]); + }); +}); + describe("AndroidDriver.create", () => { it("creates and marks its own AVD home under SIMLOCK_HOME when none exists yet", async () => { const filesystem = await androidFilesystem({ withDeviceRoot: false }); @@ -5176,7 +5254,9 @@ async function createDriver( readonly ids?: readonly string[]; readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; readonly readinessTimeoutMs?: number; + readonly scrcpyServerPath?: string; readonly tcpProbe?: FakeTcpProbe; + readonly tcpStreamConnector?: FakeTcpStreamConnector; } = {}, ) { let nextId = 0; @@ -5219,8 +5299,16 @@ function onlyProvided(options: { readonly emulator?: AndroidEmulatorLaunchOptions; readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; readonly readinessTimeoutMs?: number; + readonly scrcpyServerPath?: string; + readonly tcpStreamConnector?: FakeTcpStreamConnector; }) { return { + ...(options.scrcpyServerPath === undefined + ? {} + : { scrcpyServerPath: options.scrcpyServerPath }), + ...(options.tcpStreamConnector === undefined + ? {} + : { tcpStreamConnector: options.tcpStreamConnector }), ...(options.acceptAndroidLicenses === undefined ? {} : { acceptAndroidLicenses: options.acceptAndroidLicenses }), diff --git a/src/drivers/android/index.ts b/src/drivers/android/index.ts index 3ec7813e..1f0780ad 100644 --- a/src/drivers/android/index.ts +++ b/src/drivers/android/index.ts @@ -39,6 +39,7 @@ import { type PassthroughContext, type ReclaimResult, removeListedComponent, + type ScreenCapture, RuntimeMissingError, sameReceipt, } from "../../core/index.js"; @@ -53,6 +54,7 @@ import { type ProcessRunner, type ProcessSupervisor, type TcpProbe, + type TcpStreamConnector, type TimerHandle, } from "../../ports/index.js"; import { AdbRegistrar } from "./adb-registrar.js"; @@ -72,6 +74,7 @@ import { } from "./device-profile-source.js"; import { fits, isPlayStoreImage, misfitReason } from "./image-fit.js"; import { LibraryDeviceProfileSource } from "./profiles/index.js"; +import { createAndroidScreenCapture } from "./screen/index.js"; import { type AndroidSdkPaths, locateSdk } from "./sdk-paths.js"; export { AdbServerUnavailableError } from "./adb-server.js"; @@ -197,6 +200,10 @@ export interface AndroidDriverOptions { /** `SIMLOCK_HOME`: the default device root and the adb server record are derived here, not in the core. */ readonly simlockHome: string; readonly tcpProbe: TcpProbe; + // why: both are optional so a host that leaves either out gets a driver without screen streams + // rather than one that fails each stream it is asked for. + readonly scrcpyServerPath?: string | undefined; + readonly tcpStreamConnector?: TcpStreamConnector | undefined; /** `process.getuid?.()`; `undefined` skips the root's ownership check. */ readonly uid?: number; /** @@ -486,6 +493,7 @@ export class AndroidDriver implements Driver { readonly #sdk: AndroidSdkPaths; readonly #tcpProbe: TcpProbe; readonly componentFootprint: { readonly path: string; readonly bytes: number }; + readonly screenCapture: ScreenCapture | undefined; private constructor( options: AndroidDriverOptions, @@ -523,6 +531,7 @@ export class AndroidDriver implements Driver { this.#tcpProbe = options.tcpProbe; // System images land under the SDK root, so that is the volume an install is reserved on. this.componentFootprint = { bytes: ANDROID_SYSTEM_IMAGE_MIN_FREE_BYTES, path: sdk.root }; + this.screenCapture = this.#screenCaptureFor(options, sdk.adb, adbServerPort); // Where an AVD Simlock made before it owned a root still sits: the AVD home the user // had configured then, or the SDK's own default. Used only by `findLegacy` / // `destroyLegacy` -- the fallback CP3 deleted from the driver proper, kept exactly here @@ -542,6 +551,27 @@ export class AndroidDriver implements Driver { ); } + #screenCaptureFor( + options: AndroidDriverOptions, + adb: string, + adbServerPort: number, + ): ScreenCapture | undefined { + if (options.scrcpyServerPath === undefined || options.tcpStreamConnector === undefined) { + return undefined; + } + return createAndroidScreenCapture({ + adb, + adbServerPort, + clock: options.clock, + env: () => this.#env(), + filesystem: options.filesystem, + processRunner: options.processRunner, + serialOf: (device) => this.#dataFor(device).serial, + serverPath: options.scrcpyServerPath, + tcp: options.tcpStreamConnector, + }); + } + /** * Establishes everything containment rests on before the driver can be asked to do * anything: the AVD home this instance owns, and the private adb server its emulators diff --git a/src/drivers/android/screen/AGENTS.md b/src/drivers/android/screen/AGENTS.md new file mode 100644 index 00000000..20211930 --- /dev/null +++ b/src/drivers/android/screen/AGENTS.md @@ -0,0 +1,75 @@ +# Android screen capture + +Does: an Android emulator's screen as core's screen frames, through the vendored scrcpy server and our own client. +Entry: `index.ts` → `createAndroidScreenCapture(options)`, built by `AndroidDriver` as its `screenCapture` when `create` +gets `scrcpyServerPath` and `tcpStreamConnector` (`daemon/main.ts` passes the jar `pnpm build` copies to dist/vendor/scrcpy). +Uses: core (`ScreenCapture`, frame encoder), ports (`ProcessRunner`, `TcpStreamConnector`, `Clock`), `../adb-host-protocol.ts`. +Parent: [Android driver](../AGENTS.md). Consumer: [stream](../../../stream/AGENTS.md). + +## Files + +| File | Role | +| ----------------- | ------------------------------------------------------------------------------ | +| `index.ts` | `createAndroidScreenCapture`: the driver's `ScreenCapture`, entered only here | +| `session.ts` | One stream: push, server, socket, body, close | +| `adb-socket.ts` | `openDeviceSocket`: `host:transport` + `localabstract` on Simlock's adb server | +| `scrcpy-video.ts` | `ScrcpyVideoParser`: scrcpy's video socket (codec id, session, media packets) | +| `h264.ts` | Annex-B split, AVCC, the `avcC` record | +| `envelope.ts` | `EnvelopeMapper`: scrcpy events → core's screen frames | + +## How it works + +1. Physical → `ScreenNotSupportedError`. Jar missing at `scrcpyServerPath` → `ScreenUnavailableError` + (`unavailable()` gives the same reason). No serial (not running) → `ScreenUnavailableError`. +2. `scid`: random 31-bit, written `%08x`. `adb -s push /data/local/tmp/simlock-scrcpy-.jar`. +3. `adb -s shell CLASSPATH= app_process / com.genymobile.scrcpy.Server 5.0.1 scid= +log_level=info tunnel_forward=true video=true audio=false control=false video_codec=h264 +send_device_meta=false send_dummy_byte=true send_stream_meta=true send_frame_meta=true cleanup=true +max_size= max_fps= video_bit_rate=`. No option that changes a + device setting (`show_touches`, `stay_awake`, `screen_off_timeout`, `power_off_on_close`); with + `control=false` the server never powers the screen on. +4. No `adb forward`: a TCP connection to Simlock's adb server sends `host:transport:`, then + `localabstract:scrcpy_`. `FAIL` on the second (nothing listening yet), a refused connect, or a + socket that closes before the server's zero byte → retry every 100 ms. `FAIL` on the first → refuse. +5. Then the codec id (must be `h264`) and the first session packet (width, height) → `open` resolves. + One 15 s budget from the push's return to here. Past it, or when the shell exits first → + `ScreenUnavailableError` with the server's last 20 output lines. +6. Body: meta (session size, configured fps) on every session packet; the `avcC` description before + the first picture after config packets; key/delta frames as 4-byte-length AVCC. Pull-based + (`highWaterMark: 0`): the socket is read only when the body's reader pulls; nothing is dropped. + A body whose socket ends errors with the server's last output and closes the feed. scrcpy sends + nothing for a still screen, so after 5 s of quiet the body sends an empty `alive` frame. +7. `close` (idempotent, ≤ 5 s): close the socket, wait 1 s for the shell, then SIGKILL it, + `adb shell rm -f `. The body's end, error or cancel calls it. + +A rotation starts a new scrcpy session: a new meta frame, then a new description before the next +picture; the browser decoder re-reads both mid-stream. + +Stop: `open` takes an `AbortSignal`. A stop before the push runs nothing; one during the push (it +cannot be interrupted) starts no server after it; one while connecting ends the retries. A stopped +open cleans up (shell killed, jar removed) before it throws. `alive` frames begin only after the +first key frame, so an encoder that never paints meets the stream's silence bound. + +## State and timers + +| What | Default | Key | +| -------------------- | ------------------------------- | ----------------------------------------------------------- | +| Push / start / close | 30 s / 15 s, retry 100 ms / 5 s | `PUSH_TIMEOUT_MS`, `START_DEADLINE_MS`, `CLOSE_DEADLINE_MS` | +| Quiet before `alive` | 5 s | `ALIVE_AFTER_MS` | + +Writes on the device: `/data/local/tmp/simlock-scrcpy-.jar`, removed by the server's clean-up +process at its start and by `rm -f` on close. + +## Rules + +- A screen stream touches only its own `scid`: its jar path and socket name; never another stream's. +- A screen stream changes no device setting and drops no frame. + +## Known gaps + +- **The slow-lane stream test has not passed yet.** `e2e/slow-android-stream.test.ts` failed at the + lease on a nearly full disk. A live check on a Pixel 9 emulator streamed, and once the host + `adb shell` died no scrcpy process and no jar were left on the device. +- **Key frames every 10 s** (scrcpy's default): a viewer that resets its decoder waits up to 10 s. +- **The `avcC` record omits the High-profile extension** (chroma format, bit depths); browsers accept it. +- **Physical Android devices have no screen stream**, and the jar's digest is not checked at push. diff --git a/src/drivers/android/screen/CLAUDE.md b/src/drivers/android/screen/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/src/drivers/android/screen/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/src/drivers/android/screen/adb-socket.ts b/src/drivers/android/screen/adb-socket.ts new file mode 100644 index 00000000..56491b2c --- /dev/null +++ b/src/drivers/android/screen/adb-socket.ts @@ -0,0 +1,122 @@ +import type { TcpStream, TcpStreamConnector } from "../../../ports/index.js"; +import { + ADB_FAIL, + ADB_LENGTH_BYTES, + ADB_OKAY, + ADB_STATUS_BYTES, + adbHostRequest, +} from "../adb-host-protocol.js"; +import { ByteQueue } from "./byte-queue.js"; + +// why: adb's replies are short, so a reason longer than this is not a reply but a desynced stream. +const MAX_FAIL_REASON_BYTES = 4096; + +export class ChunkReader { + readonly #reader: ReadableStreamDefaultReader; + readonly #queue = new ByteQueue(); + + constructor(readable: ReadableStream) { + this.#reader = readable.getReader(); + } + + // why: a reply and the bytes after it often share one TCP read, and a chunk smaller than what is + // asked for is normal; both have to be kept rather than dropped or waited past. + async readExactly(count: number): Promise { + while (this.#queue.length < count) { + const step = await this.#readSource(); + if (step === undefined) return undefined; + this.#queue.push(step); + } + return this.#queue.take(count); + } + + async read(): Promise { + if (this.#queue.length > 0) return this.#queue.drain(); + return await this.#readSource(); + } + + async #readSource(): Promise { + try { + const step = await this.#reader.read(); + return step.done ? undefined : step.value; + } catch { + return undefined; + } + } +} + +type DeviceSocketOutcome = + | { readonly kind: "open"; readonly stream: TcpStream; readonly reader: ChunkReader } + | { readonly kind: "refused"; readonly reason: string } + | { readonly kind: "no-transport"; readonly reason: string }; + +interface DeviceSocketRequest { + readonly tcp: TcpStreamConnector; + readonly adbServerPort: number; + readonly serial: string; + readonly socketName: string; + readonly timeoutMs: number; + readonly signal: AbortSignal; +} + +// why: speaking adb's host protocol directly reaches the device socket through Simlock's own server +// without an `adb forward`, so a daemon that dies mid-stream leaves no listening port on the host. +// adb answers `localabstract:` with OKAY only once adbd's connect to that socket succeeded, and with +// FAIL "closed" while nothing listens there yet: +// https://android.googlesource.com/platform/packages/modules/adb/+/refs/heads/main/sockets.cpp +export async function openDeviceSocket(request: DeviceSocketRequest): Promise { + const stream = await request.tcp.connect(request.adbServerPort, request.timeoutMs); + const onAbort = () => { + stream.close(); + }; + request.signal.addEventListener("abort", onAbort, { once: true }); + try { + if (request.signal.aborted) stream.close(); + const reader = new ChunkReader(stream.readable); + await stream.write(adbHostRequest(`host:transport:${request.serial}`)); + const transport = await readReply(reader); + if (transport !== undefined) { + stream.close(); + return { kind: "no-transport", reason: transport }; + } + await stream.write(adbHostRequest(`localabstract:${request.socketName}`)); + const socket = await readReply(reader); + if (socket !== undefined) { + stream.close(); + return { kind: "refused", reason: socket }; + } + return { kind: "open", reader, stream }; + } catch (error) { + stream.close(); + throw error; + } finally { + request.signal.removeEventListener("abort", onAbort); + } +} + +class AdbReplyError extends Error { + constructor(message: string) { + super(message); + this.name = "AdbReplyError"; + } +} + +async function readReply(reader: ChunkReader): Promise { + const status = await reader.readExactly(ADB_STATUS_BYTES); + if (status === undefined) return "adb closed the connection before it answered"; + const text = ascii(status); + if (text === ADB_OKAY) return undefined; + if (text !== ADB_FAIL) throw new AdbReplyError(`adb answered ${JSON.stringify(text)}`); + const length = await reader.readExactly(ADB_LENGTH_BYTES); + if (length === undefined) return "adb failed without a reason"; + const size = Number.parseInt(ascii(length), 16); + if (!/^[0-9a-fA-F]{4}$/.test(ascii(length)) || size > MAX_FAIL_REASON_BYTES) { + throw new AdbReplyError("adb's failure reason is not length-prefixed"); + } + const reason = await reader.readExactly(size); + return reason === undefined ? "adb failed without a reason" : ascii(reason) || "adb failed"; +} + +function ascii(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes); +} diff --git a/src/drivers/android/screen/byte-queue.ts b/src/drivers/android/screen/byte-queue.ts new file mode 100644 index 00000000..23a34d28 --- /dev/null +++ b/src/drivers/android/screen/byte-queue.ts @@ -0,0 +1,35 @@ +export class ByteQueue { + readonly #chunks: Uint8Array[] = []; + #length = 0; + + get length(): number { + return this.#length; + } + + push(chunk: Uint8Array): void { + if (chunk.length === 0) return; + this.#chunks.push(chunk); + this.#length += chunk.length; + } + + take(count: number): Uint8Array { + if (count > this.#length) throw new RangeError("ByteQueue.take past its end"); + const out = new Uint8Array(count); + let filled = 0; + while (filled < count) { + const head = this.#chunks[0]; + if (head === undefined) break; + const used = Math.min(head.length, count - filled); + out.set(head.subarray(0, used), filled); + filled += used; + if (used === head.length) this.#chunks.shift(); + else this.#chunks[0] = head.subarray(used); + } + this.#length -= count; + return out; + } + + drain(): Uint8Array { + return this.take(this.#length); + } +} diff --git a/src/drivers/android/screen/envelope.test.ts b/src/drivers/android/screen/envelope.test.ts new file mode 100644 index 00000000..31fbdeea --- /dev/null +++ b/src/drivers/android/screen/envelope.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; + +import { SCREEN_FRAME_HEADER_BYTES, SCREEN_FRAME_TAG } from "../../../core/index.js"; +import { EnvelopeMapper } from "./envelope.js"; +import { avcDecoderConfiguration, splitAnnexB, toAvcc } from "./h264.js"; +import { ScrcpyVideoParser } from "./scrcpy-video.js"; +import { + codecId, + concatBytes, + mediaPacket, + SAMPLE, + sampleStream, + sessionPacket, +} from "./test-scrcpy.js"; + +interface Frame { + readonly tag: number; + readonly payload: Uint8Array; +} + +function frames(stream: Uint8Array, fps = 30): Frame[] { + const parser = new ScrcpyVideoParser(); + const mapper = new EnvelopeMapper(fps); + return parser + .push(stream) + .flatMap((event) => mapper.map(event)) + .map((frame) => { + expect(new DataView(frame.buffer).getUint32(0)).toBe( + frame.length - SCREEN_FRAME_HEADER_BYTES + 1, + ); + return { payload: frame.subarray(SCREEN_FRAME_HEADER_BYTES), tag: frame[4]! }; + }); +} + +const json = (bytes: Uint8Array): unknown => JSON.parse(new TextDecoder().decode(bytes)); + +describe("EnvelopeMapper", () => { + it("writes meta, then the description, then the key frame and its deltas", () => { + expect(frames(sampleStream()).map((frame) => frame.tag)).toEqual([ + SCREEN_FRAME_TAG.meta, + SCREEN_FRAME_TAG.description, + SCREEN_FRAME_TAG.keyframe, + SCREEN_FRAME_TAG.delta, + SCREEN_FRAME_TAG.delta, + SCREEN_FRAME_TAG.keyframe, + SCREEN_FRAME_TAG.delta, + ]); + }); + + it("names the session's size and the asked-for frame rate in the meta frame", () => { + expect(json(frames(sampleStream(), 24)[0]!.payload)).toEqual({ + codec: "h264", + fps: 24, + height: 64, + width: 96, + }); + }); + + it("describes the stream with the avcC record of the config packet's parameter sets", () => { + const [sps, pps] = splitAnnexB(SAMPLE.config); + expect([...frames(sampleStream())[1]!.payload]).toEqual([ + ...avcDecoderConfiguration([sps!], [pps!]), + ]); + }); + + it("carries every unit of a picture as AVCC, in-band parameter sets included", () => { + const written = frames(sampleStream()); + expect([...written[2]!.payload]).toEqual([...toAvcc(splitAnnexB(SAMPLE.idr))]); + expect([...written[3]!.payload]).toEqual([...toAvcc(splitAnnexB(SAMPLE.p1))]); + expect([...written[5]!.payload]).toEqual([...toAvcc(splitAnnexB(SAMPLE.idrWithParameterSets))]); + expect(written[5]!.payload.indexOf(0x67)).toBeGreaterThan(0); + }); + + it("starts a rotated session with a fresh meta and description before its first picture", () => { + const rotated = concatBytes( + sampleStream(), + sessionPacket(64, 96), + mediaPacket({ config: true }, SAMPLE.config), + mediaPacket({ key: true }, SAMPLE.idr), + ); + const tail = frames(rotated).slice(-3); + expect(tail.map((frame) => frame.tag)).toEqual([ + SCREEN_FRAME_TAG.meta, + SCREEN_FRAME_TAG.description, + SCREEN_FRAME_TAG.keyframe, + ]); + expect(json(tail[0]!.payload)).toMatchObject({ height: 96, width: 64 }); + }); + + it("joins parameter sets an encoder hands over in two config packets", () => { + const [sps, pps] = splitAnnexB(SAMPLE.config); + const split = concatBytes( + codecId(), + sessionPacket(96, 64), + mediaPacket({ config: true }, SAMPLE.config.subarray(0, 27)), + mediaPacket({ config: true }, SAMPLE.config.subarray(27)), + mediaPacket({ key: true }, SAMPLE.idr), + ); + expect([...frames(split)[1]!.payload]).toEqual([...avcDecoderConfiguration([sps!], [pps!])]); + }); + + it("refuses a picture before the session named its size or before any parameter sets", () => { + const sized = concatBytes( + codecId(), + sessionPacket(96, 64), + mediaPacket({ config: true }, SAMPLE.config), + ); + expect(frames(concatBytes(sized, mediaPacket({ key: true }, SAMPLE.idr)))).toHaveLength(3); + expect(() => frames(concatBytes(codecId(), mediaPacket({ key: true }, SAMPLE.idr)))).toThrow( + /size/, + ); + expect(() => + frames(concatBytes(codecId(), sessionPacket(96, 64), mediaPacket({ key: true }, SAMPLE.idr))), + ).toThrow(/parameter sets/); + }); +}); diff --git a/src/drivers/android/screen/envelope.ts b/src/drivers/android/screen/envelope.ts new file mode 100644 index 00000000..cda89de8 --- /dev/null +++ b/src/drivers/android/screen/envelope.ts @@ -0,0 +1,84 @@ +import { encodeScreenFrame, encodeScreenMeta, SCREEN_FRAME_TAG } from "../../../core/index.js"; +import { + avcDecoderConfiguration, + H264FormatError, + NAL_TYPE_PPS, + NAL_TYPE_SPS, + nalType, + splitAnnexB, + toAvcc, +} from "./h264.js"; +import type { ScrcpyVideoEvent } from "./scrcpy-video.js"; + +// why: the browser refuses a frame longer than this, and a stream it refuses mid-way is a viewer left +// with a frozen picture; refusing here names the cause on the daemon's side instead. +const MAX_FRAME_BYTES = 8 * 1024 * 1024; + +export class EnvelopeMapper { + readonly #fps: number; + #sps: Uint8Array[] = []; + #pps: Uint8Array[] = []; + #descriptionDue = false; + #sessionSeen = false; + + constructor(fps: number) { + this.#fps = fps; + } + + // fallow-ignore-next-line complexity -- one branch per scrcpy event, each the frames it becomes. + map(event: ScrcpyVideoEvent): Uint8Array[] { + if (event.kind === "session") { + this.#sessionSeen = true; + // why: a new session is a new encoder (the device rotated, or the server downsized after an + // encoder error), so the parameter sets of the last one say nothing about the next picture. + this.#sps = []; + this.#pps = []; + this.#descriptionDue = false; + return [ + encodeScreenMeta({ + codec: "h264", + fps: this.#fps, + height: event.height, + width: event.width, + }), + ]; + } + if (!this.#sessionSeen) { + throw new H264FormatError("The screen server sent video before it named the picture's size"); + } + const nals = splitAnnexB(event.payload); + if (event.config) { + // why: an encoder may hand its sequence and picture parameter sets over in one config packet or + // in two, so the description waits for the first picture rather than for the first config. + const sps = nals.filter((nal) => nalType(nal) === NAL_TYPE_SPS); + const pps = nals.filter((nal) => nalType(nal) === NAL_TYPE_PPS); + if (sps.length > 0) this.#sps = sps; + if (pps.length > 0) this.#pps = pps; + this.#descriptionDue = true; + return []; + } + const frames: Uint8Array[] = []; + if (this.#descriptionDue) { + frames.push( + encodeScreenFrame( + SCREEN_FRAME_TAG.description, + avcDecoderConfiguration(this.#sps, this.#pps), + ), + ); + this.#descriptionDue = false; + } else if (this.#sps.length === 0) { + throw new H264FormatError("The screen server sent a picture before its parameter sets"); + } + const picture = encodeScreenFrame( + event.key ? SCREEN_FRAME_TAG.keyframe : SCREEN_FRAME_TAG.delta, + toAvcc(nals), + ); + if (picture.length > MAX_FRAME_BYTES) { + throw new H264FormatError( + `A ${String(picture.length)}-byte frame is past what a viewer reads`, + ); + } + frames.push(picture); + return frames; + } +} diff --git a/src/drivers/android/screen/h264.test.ts b/src/drivers/android/screen/h264.test.ts new file mode 100644 index 00000000..c871c084 --- /dev/null +++ b/src/drivers/android/screen/h264.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { avcDecoderConfiguration, H264FormatError, splitAnnexB, toAvcc } from "./h264.js"; +import { SAMPLE_H264 } from "./test-fixture.js"; + +const bytes = (...values: number[]) => new Uint8Array(values); + +describe("splitAnnexB", () => { + it("splits on three- and four-byte start codes alike, keeping neither in a unit", () => { + const stream = bytes(0, 0, 0, 1, 0x67, 0xaa, 0, 0, 1, 0x68, 0xbb, 0, 0, 0, 1, 0x65, 0xcc); + expect(splitAnnexB(stream).map((nal) => [...nal])).toEqual([ + [0x67, 0xaa], + [0x68, 0xbb], + [0x65, 0xcc], + ]); + }); + + it("drops the trailing zero bytes an encoder pads a unit with", () => { + expect(splitAnnexB(bytes(0, 0, 1, 0x65, 0x11, 0, 0)).map((nal) => [...nal])).toEqual([ + [0x65, 0x11], + ]); + }); + + it("keeps emulation-prevention bytes, which are part of the unit", () => { + expect([...splitAnnexB(bytes(0, 0, 1, 0x67, 0, 0, 3, 1, 0x80))[0]!]).toEqual([ + 0x67, 0, 0, 3, 1, 0x80, + ]); + }); + + it("refuses a packet that does not start with a start code", () => { + expect(splitAnnexB(bytes(0, 0, 1, 0x65)).length).toBe(1); + expect(() => splitAnnexB(bytes(0x65, 0, 0, 1, 0x65))).toThrow(H264FormatError); + }); + + it("splits the fixture into the nine units its encoder wrote, in order", () => { + expect(splitAnnexB(SAMPLE_H264).map((nal) => nal[0]! & 0x1f)).toEqual([ + 7, 8, 5, 1, 1, 7, 8, 5, 1, + ]); + }); +}); + +describe("toAvcc", () => { + it("prefixes every unit with its length as four big-endian bytes", () => { + expect([...toAvcc([bytes(0x65, 1, 2), bytes(0x41)])]).toEqual([ + 0, 0, 0, 3, 0x65, 1, 2, 0, 0, 0, 1, 0x41, + ]); + }); +}); + +describe("avcDecoderConfiguration", () => { + const [sps, pps] = splitAnnexB(SAMPLE_H264); + + it("takes profile, compatibility and level from the SPS and declares four-byte lengths", () => { + const record = avcDecoderConfiguration([sps!], [pps!]); + expect([...record.subarray(0, 6)]).toEqual([1, 0x42, 0xc0, 0x1e, 0xff, 0xe1]); + expect(new DataView(record.buffer).getUint16(6)).toBe(sps!.length); + expect([...record.subarray(8, 8 + sps!.length)]).toEqual([...sps!]); + const ppsAt = 8 + sps!.length; + expect(record[ppsAt]).toBe(1); + expect(new DataView(record.buffer).getUint16(ppsAt + 1)).toBe(pps!.length); + expect([...record.subarray(ppsAt + 3)]).toEqual([...pps!]); + }); + + it("declares four-byte lengths whatever the SPS's own bytes say", () => { + const odd = bytes(0x67, 0x64, 0x00, 0x28, 0x00); + expect(avcDecoderConfiguration([odd], [bytes(0x68, 1)])[4]).toBe(0xff); + }); + + it("counts every SPS and PPS it carries", () => { + const record = avcDecoderConfiguration([sps!, sps!], [pps!, pps!, pps!]); + expect(record[5]).toBe(0xe2); + expect(record[6 + 2 * (2 + sps!.length)]).toBe(3); + }); + + it("refuses a record without a picture parameter set", () => { + expect(() => avcDecoderConfiguration([sps!], [])).toThrow(/picture parameter set/); + }); +}); diff --git a/src/drivers/android/screen/h264.ts b/src/drivers/android/screen/h264.ts new file mode 100644 index 00000000..2ffd5a54 --- /dev/null +++ b/src/drivers/android/screen/h264.ts @@ -0,0 +1,101 @@ +export const NAL_TYPE_SPS = 7; +export const NAL_TYPE_PPS = 8; + +// why: every length prefix this module writes is four bytes, so the record that tells a decoder how +// to read them says so as a constant. Deriving it from anything in the parameter sets is the bug +// that made a decoder read three-byte lengths out of four-byte ones. +const AVCC_LENGTH_SIZE_MINUS_ONE_BYTE = 0xff; +const AVCC_LENGTH_BYTES = 4; +const MAX_SPS_COUNT = 31; +const MAX_PPS_COUNT = 255; + +export class H264FormatError extends Error { + constructor(message: string) { + super(message); + this.name = "H264FormatError"; + } +} + +export function nalType(nal: Uint8Array): number { + return (nal[0] ?? 0) & 0x1f; +} + +export function splitAnnexB(bytes: Uint8Array): Uint8Array[] { + const starts: { readonly at: number; readonly payload: number }[] = []; + for (let index = 0; index + 2 < bytes.length; index += 1) { + if (bytes[index] === 0 && bytes[index + 1] === 0 && bytes[index + 2] === 1) { + starts.push({ at: index, payload: index + 3 }); + index += 2; + } + } + const first = starts[0]; + if (first === undefined || !onlyZeros(bytes, 0, first.at)) { + throw new H264FormatError("The encoder's packet does not start with an Annex-B start code"); + } + const nals: Uint8Array[] = []; + starts.forEach((start, position) => { + let end = starts[position + 1]?.at ?? bytes.length; + // why: a four-byte start code is a zero byte and a three-byte one, and Annex-B lets an encoder pad + // a NAL unit with trailing zeros; neither belongs to the unit a decoder is handed. + while (end > start.payload && bytes[end - 1] === 0) end -= 1; + if (end > start.payload) nals.push(bytes.subarray(start.payload, end)); + }); + return nals; +} + +export function toAvcc(nals: readonly Uint8Array[]): Uint8Array { + const total = nals.reduce((sum, nal) => sum + AVCC_LENGTH_BYTES + nal.length, 0); + const out = new Uint8Array(total); + const view = new DataView(out.buffer); + let offset = 0; + for (const nal of nals) { + view.setUint32(offset, nal.length); + out.set(nal, offset + AVCC_LENGTH_BYTES); + offset += AVCC_LENGTH_BYTES + nal.length; + } + return out; +} + +export function avcDecoderConfiguration( + sps: readonly Uint8Array[], + pps: readonly Uint8Array[], +): Uint8Array { + const first = sps[0]; + if (first === undefined || first.length < 4) { + throw new H264FormatError("The encoder sent no sequence parameter set"); + } + if (pps.length === 0) throw new H264FormatError("The encoder sent no picture parameter set"); + if (sps.length > MAX_SPS_COUNT || pps.length > MAX_PPS_COUNT) { + throw new H264FormatError("The encoder sent more parameter sets than an avcC record holds"); + } + const size = + 7 + + sps.reduce((sum, nal) => sum + 2 + nal.length, 0) + + pps.reduce((sum, nal) => sum + 2 + nal.length, 0); + const out = new Uint8Array(size); + const view = new DataView(out.buffer); + out[0] = 1; + out[1] = first[1] ?? 0; + out[2] = first[2] ?? 0; + out[3] = first[3] ?? 0; + out[4] = AVCC_LENGTH_SIZE_MINUS_ONE_BYTE; + out[5] = 0xe0 | sps.length; + let offset = 6; + for (const nal of sps) offset = writeParameterSet(out, view, offset, nal); + out[offset] = pps.length; + offset += 1; + for (const nal of pps) offset = writeParameterSet(out, view, offset, nal); + return out; +} + +function writeParameterSet(out: Uint8Array, view: DataView, offset: number, nal: Uint8Array) { + if (nal.length > 0xffff) throw new H264FormatError("A parameter set is longer than avcC allows"); + view.setUint16(offset, nal.length); + out.set(nal, offset + 2); + return offset + 2 + nal.length; +} + +function onlyZeros(bytes: Uint8Array, from: number, to: number): boolean { + for (let index = from; index < to; index += 1) if (bytes[index] !== 0) return false; + return true; +} diff --git a/src/drivers/android/screen/index.ts b/src/drivers/android/screen/index.ts new file mode 100644 index 00000000..408559af --- /dev/null +++ b/src/drivers/android/screen/index.ts @@ -0,0 +1 @@ +export { createAndroidScreenCapture } from "./session.js"; diff --git a/src/drivers/android/screen/scrcpy-video.test.ts b/src/drivers/android/screen/scrcpy-video.test.ts new file mode 100644 index 00000000..11b815ec --- /dev/null +++ b/src/drivers/android/screen/scrcpy-video.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vitest"; + +import { + MAX_SCRCPY_PACKET_BYTES, + ScrcpyProtocolError, + type ScrcpyVideoEvent, + ScrcpyVideoParser, +} from "./scrcpy-video.js"; +import { + codecId, + concatBytes, + mediaPacket, + SAMPLE, + sampleStream, + sessionPacket, +} from "./test-scrcpy.js"; + +function parseInChunks(stream: Uint8Array, cuts: readonly number[]): ScrcpyVideoEvent[] { + const parser = new ScrcpyVideoParser(); + const events: ScrcpyVideoEvent[] = []; + let from = 0; + for (const cut of [...cuts, stream.length]) { + events.push(...parser.push(stream.subarray(from, cut))); + from = cut; + } + return events; +} + +function summary(events: readonly ScrcpyVideoEvent[]) { + return events.map((event) => + event.kind === "session" + ? { height: event.height, kind: event.kind, width: event.width } + : { + config: event.config, + key: event.key, + kind: event.kind, + payload: [...event.payload], + pts: event.pts, + }, + ); +} + +describe("ScrcpyVideoParser", () => { + const stream = sampleStream(); + const whole = summary(parseInChunks(stream, [])); + + it("reads the session packet, then config, key and delta packets with their PTS", () => { + expect( + whole.map((event) => ("config" in event ? [event.config, event.key, event.pts] : event)), + ).toEqual([ + { height: 64, kind: "session", width: 96 }, + [true, false, 0], + [false, true, 0], + [false, false, 100_000], + [false, false, 200_000], + [false, true, 300_000], + [false, false, 400_000], + ]); + expect(whole[3]).toMatchObject({ payload: [...SAMPLE.p1] }); + }); + + it("reads the same events however the bytes are split in two, at every offset", () => { + // why: small payloads keep every packet kind and every header boundary while making each of the + // cuts cheap to compare, so the test proves the framing and not the machine's speed. + const small = concatBytes( + codecId(), + sessionPacket(96, 64), + mediaPacket({ config: true }, new Uint8Array([0, 0, 0, 1, 0x67, 1])), + mediaPacket({ key: true, pts: 0 }, new Uint8Array([0, 0, 0, 1, 0x65, 2])), + mediaPacket({ pts: 100_000 }, new Uint8Array([0, 0, 0, 1, 0x41, 3])), + sessionPacket(64, 96), + ); + const expected = summary(parseInChunks(small, [])); + expect(expected).toHaveLength(5); + for (let cut = 1; cut < small.length; cut += 1) { + expect(summary(parseInChunks(small, [cut])), `cut at ${String(cut)}`).toEqual(expected); + } + }); + + it("reads the same events from one byte at a time", () => { + const cuts = Array.from({ length: stream.length - 1 }, (_, index) => index + 1); + expect(summary(parseInChunks(stream, cuts))).toEqual(whole); + }); + + it("reads a large PTS whose high bits sit beside the flags", () => { + const pts = 2 ** 40 + 7; + const events = parseInChunks( + concatBytes(codecId(), mediaPacket({ key: true, pts }, new Uint8Array([0, 0, 1, 0x65]))), + [], + ); + expect(events[0]).toMatchObject({ config: false, key: true, pts }); + }); + + it("holds a packet back until its last byte arrives", () => { + const parser = new ScrcpyVideoParser(); + const packet = mediaPacket({ key: true }, SAMPLE.idr); + expect(parser.push(concatBytes(codecId(), packet.subarray(0, packet.length - 1)))).toEqual([]); + expect(parser.push(packet.subarray(packet.length - 1))).toHaveLength(1); + }); + + it("refuses a stream the server disabled, failed to configure, or encodes in another codec", () => { + expect(parseInChunks(codecId(), [])).toEqual([]); + expect(() => parseInChunks(codecId(0), [])).toThrow(/disabled/); + expect(() => parseInChunks(codecId(1), [])).toThrow(/could not configure/); + expect(() => parseInChunks(codecId(0x68323635), [])).toThrow(ScrcpyProtocolError); + }); + + it("refuses a packet claiming more than a viewer reads, before buffering it", () => { + const header = (size: number) => mediaPacket({ key: true }, new Uint8Array(0), size); + expect(parseInChunks(concatBytes(codecId(), header(MAX_SCRCPY_PACKET_BYTES)), [])).toEqual([]); + expect(() => + parseInChunks(concatBytes(codecId(), header(MAX_SCRCPY_PACKET_BYTES + 1)), []), + ).toThrow(ScrcpyProtocolError); + }); + + it("refuses a session with no picture", () => { + expect(parseInChunks(concatBytes(codecId(), sessionPacket(1, 1)), [])).toHaveLength(1); + expect(() => parseInChunks(concatBytes(codecId(), sessionPacket(0, 64)), [])).toThrow( + ScrcpyProtocolError, + ); + }); +}); diff --git a/src/drivers/android/screen/scrcpy-video.ts b/src/drivers/android/screen/scrcpy-video.ts new file mode 100644 index 00000000..5a30acb3 --- /dev/null +++ b/src/drivers/android/screen/scrcpy-video.ts @@ -0,0 +1,110 @@ +import { ByteQueue } from "./byte-queue.js"; + +// why: "h264" as a big-endian u32 -- the server writes its codec's id as the stream's first four bytes, +// and an id of 0 or 1 instead says it disabled the stream or failed to configure it. +export const SCRCPY_CODEC_H264 = 0x68323634; +const SCRCPY_STREAM_DISABLED = 0; +const SCRCPY_STREAM_CONFIG_ERROR = 1; + +const CODEC_ID_BYTES = 4; +const PACKET_HEADER_BYTES = 12; +const SESSION_FLAG = 0x80000000; +const CONFIG_FLAG = 0x40000000; +const KEY_FLAG = 0x20000000; +const PTS_HIGH_MASK = 0x1fffffff; + +// why: the browser that reads this stream refuses a frame over 8 MiB, so a packet that claims more is a +// desynced stream, not a big picture; the bound also caps what one half-read packet can hold in memory. +export const MAX_SCRCPY_PACKET_BYTES = 8 * 1024 * 1024; +const MAX_VIDEO_SIDE = 16_384; + +export type ScrcpyVideoEvent = + | { readonly kind: "session"; readonly width: number; readonly height: number } + | { + readonly kind: "media"; + readonly config: boolean; + readonly key: boolean; + readonly pts: number; + readonly payload: Uint8Array; + }; + +export class ScrcpyProtocolError extends Error { + constructor(message: string) { + super(message); + this.name = "ScrcpyProtocolError"; + } +} + +export class ScrcpyVideoParser { + readonly #queue = new ByteQueue(); + #codecRead = false; + #pending: + | { + readonly config: boolean; + readonly key: boolean; + readonly pts: number; + readonly size: number; + } + | undefined; + + push(chunk: Uint8Array): ScrcpyVideoEvent[] { + this.#queue.push(chunk); + const events: ScrcpyVideoEvent[] = []; + for (;;) { + const event = this.#next(); + if (event === undefined) return events; + if (event !== "codec") events.push(event); + } + } + + // fallow-ignore-next-line complexity -- one branch per packet kind scrcpy's video socket carries. + #next(): ScrcpyVideoEvent | "codec" | undefined { + if (!this.#codecRead) { + if (this.#queue.length < CODEC_ID_BYTES) return undefined; + this.#readCodec(new DataView(this.#queue.take(CODEC_ID_BYTES).buffer).getUint32(0)); + return "codec"; + } + if (this.#pending !== undefined) { + if (this.#queue.length < this.#pending.size) return undefined; + const { config, key, pts, size } = this.#pending; + this.#pending = undefined; + return { config, key, kind: "media", payload: this.#queue.take(size), pts }; + } + if (this.#queue.length < PACKET_HEADER_BYTES) return undefined; + const header = new DataView(this.#queue.take(PACKET_HEADER_BYTES).buffer); + const high = header.getUint32(0); + if ((high & SESSION_FLAG) !== 0) { + const width = header.getUint32(4); + const height = header.getUint32(8); + if (width === 0 || height === 0 || width > MAX_VIDEO_SIDE || height > MAX_VIDEO_SIDE) { + throw new ScrcpyProtocolError( + `The screen server announced a ${String(width)}x${String(height)} picture`, + ); + } + return { height, kind: "session", width }; + } + const size = header.getUint32(8); + if (size > MAX_SCRCPY_PACKET_BYTES) { + throw new ScrcpyProtocolError(`The screen server sent a ${String(size)}-byte packet`); + } + this.#pending = { + config: (high & CONFIG_FLAG) !== 0, + key: (high & KEY_FLAG) !== 0, + pts: (high & PTS_HIGH_MASK) * 2 ** 32 + header.getUint32(4), + size, + }; + return this.#next(); + } + + #readCodec(id: number): void { + this.#codecRead = true; + if (id === SCRCPY_CODEC_H264) return; + if (id === SCRCPY_STREAM_DISABLED) { + throw new ScrcpyProtocolError("The screen server disabled its video stream"); + } + if (id === SCRCPY_STREAM_CONFIG_ERROR) { + throw new ScrcpyProtocolError("The screen server could not configure its video stream"); + } + throw new ScrcpyProtocolError(`The screen server chose codec 0x${id.toString(16)}, not H.264`); + } +} diff --git a/src/drivers/android/screen/session.test.ts b/src/drivers/android/screen/session.test.ts new file mode 100644 index 00000000..1961dde5 --- /dev/null +++ b/src/drivers/android/screen/session.test.ts @@ -0,0 +1,618 @@ +import { describe, expect, it } from "vitest"; + +import { + type PhysicalDriverDevice, + SCREEN_FRAME_TAG, + ScreenNotSupportedError, + ScreenUnavailableError, + type VirtualDriverDevice, +} from "../../../core/index.js"; +import { + FakeClock, + MemoryFilesystem, + ScriptedProcessRunner, + type ScriptedProcessExpectation, + type TcpStream, + type TcpStreamConnector, +} from "../../../ports/index.js"; +import { + ALIVE_AFTER_MS, + CLOSE_DEADLINE_MS, + CONNECT_RETRY_MS, + createAndroidScreenCapture, + START_DEADLINE_MS, +} from "./session.js"; +import { codecId, concatBytes, mediaPacket, SAMPLE, sessionPacket } from "./test-scrcpy.js"; + +const ADB = "/sdk/platform-tools/adb"; +const SERVER = "/simlock/scrcpy-server-v5.0.1"; +const SERIAL = "emulator-5586"; +const PORT = 5038; +const SCID = 0x1234abcd; +const DEVICE_JAR = "/data/local/tmp/simlock-scrcpy-1234abcd.jar"; +const SETTINGS = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; +const DEVICE: VirtualDriverDevice = { + address: SERIAL, + deviceId: "simlock_1", + driverData: {}, + physical: false, +}; +const ENV = { ANDROID_ADB_SERVER_PORT: String(PORT), ANDROID_AVD_HOME: "/root", PATH: "/bin" }; + +type Behaviour = "accept" | "refuse" | "no-transport" | "connection-refused" | "silent"; + +class Peer { + readonly written: string[] = []; + closed = false; + pulls = 0; + readonly #queue: Uint8Array[] = []; + #wake: (() => void) | undefined; + readonly stream: TcpStream; + + constructor() { + const readable = new ReadableStream( + { + cancel: () => { + this.close(); + }, + pull: async (controller) => { + this.pulls += 1; + while (this.#queue.length === 0 && !this.closed) { + await new Promise((resolve) => { + this.#wake = resolve; + }); + } + const next = this.#queue.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(next); + }, + }, + { highWaterMark: 0 }, + ); + this.stream = { + close: () => { + this.close(); + }, + readable, + write: async (bytes) => { + this.written.push(typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes)); + }, + }; + } + + send(...chunks: (Uint8Array | string)[]): void { + for (const chunk of chunks) { + this.#queue.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk); + } + this.#wake?.(); + } + + close(): void { + this.closed = true; + this.#wake?.(); + } +} + +class AdbServer implements TcpStreamConnector { + readonly peers: Peer[] = []; + readonly ports: number[] = []; + readonly #behaviours: Behaviour[]; + + constructor( + behaviours: Behaviour[] = [], + readonly otherwise: Behaviour = "accept", + ) { + this.#behaviours = behaviours; + } + + async connect(port: number): Promise { + this.ports.push(port); + const behaviour = this.#behaviours.shift() ?? this.otherwise; + if (behaviour === "connection-refused") throw new Error("connect ECONNREFUSED 127.0.0.1"); + const peer = new Peer(); + this.peers.push(peer); + if (behaviour === "no-transport") peer.send("FAIL0020device 'emulator-5586' not found"); + if (behaviour === "refuse") peer.send("OKAY", "FAIL0006closed"); + if (behaviour === "accept") peer.send("OKAY", "OKAY", new Uint8Array([0])); + return peer.stream; + } + + get last(): Peer { + const peer = this.peers.at(-1); + if (peer === undefined) throw new Error("no connection"); + return peer; + } +} + +function push(result: Partial<{ code: number; stderr: string }> = {}): ScriptedProcessExpectation { + return { + match: { + args: ["-s", SERIAL, "push", SERVER, /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/], + command: ADB, + }, + result: { code: result.code ?? 0, stderr: result.stderr ?? "", stdout: "" }, + }; +} + +function shell(extra: Partial = {}): ScriptedProcessExpectation { + return { + hangs: true, + match: { + args: ["-s", SERIAL, "shell", ...Array.from({ length: 20 }, () => /.*/)], + command: ADB, + }, + ...extra, + }; +} + +function remove(): ScriptedProcessExpectation { + return { + match: { + args: [ + "-s", + SERIAL, + "shell", + "rm", + "-f", + /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/, + ], + command: ADB, + }, + }; +} + +async function setup( + options: { + readonly expectations?: ScriptedProcessExpectation[]; + readonly server?: AdbServer; + readonly jar?: boolean; + readonly scid?: (() => number) | null; + } = {}, +) { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + if (options.jar !== false) { + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(SERVER, "jar"); + } + const runner = new ScriptedProcessRunner(options.expectations ?? [push(), shell(), remove()]); + const server = options.server ?? new AdbServer(); + const capture = createAndroidScreenCapture({ + adb: ADB, + adbServerPort: PORT, + clock, + env: () => ({ ...ENV }), + filesystem, + processRunner: runner, + serialOf: () => SERIAL, + serverPath: SERVER, + tcp: server, + ...(options.scid === null ? {} : { scid: options.scid ?? (() => SCID) }), + }); + return { capture, clock, runner, server }; +} + +async function drive(clock: FakeClock, promise: Promise, stepMs = 50): Promise { + let settled = false; + promise.then( + () => (settled = true), + () => (settled = true), + ); + for (let spent = 0; spent <= 120_000; spent += stepMs) { + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + if (settled) return await promise; + clock.advance(stepMs); + } + throw new Error("did not settle within two fake minutes"); +} + +function streamStart(): Uint8Array { + return concatBytes(codecId(), sessionPacket(96, 64)); +} + +async function hasEnded(handle: { wait(): Promise }): Promise { + let ended = false; + void handle.wait().then(() => (ended = true)); + for (let flush = 0; flush < 5; flush += 1) await new Promise((resolve) => setImmediate(resolve)); + return ended; +} + +function frameTag(frame: Uint8Array | undefined): number | undefined { + return frame?.[4]; +} + +async function opened(setupResult: Awaited>) { + const { capture, clock, server } = setupResult; + const opening = capture.open(DEVICE, SETTINGS, new AbortController().signal); + const feed = await drive(clock, waitThenStart(server, opening)); + return feed; +} + +async function waitThenStart(server: AdbServer, opening: Promise): Promise { + for (let tries = 0; tries < 100 && server.peers.length === 0; tries += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + server.last.send(streamStart()); + return await opening; +} + +describe("Android screen capture: open", () => { + it("pushes the server to a per-session path and starts it with that session's id and the settings", async () => { + const world = await setup(); + await opened(world); + expect(world.runner.calls[0]).toMatchObject({ + args: ["-s", SERIAL, "push", SERVER, DEVICE_JAR], + command: ADB, + options: { env: ENV }, + }); + expect(world.runner.calls[1]).toMatchObject({ command: ADB, options: { env: ENV } }); + expect(world.runner.calls[1]!.args).toEqual([ + "-s", + SERIAL, + "shell", + `CLASSPATH=${DEVICE_JAR}`, + "app_process", + "/", + "com.genymobile.scrcpy.Server", + "5.0.1", + "scid=1234abcd", + "log_level=info", + "tunnel_forward=true", + "video=true", + "audio=false", + "control=false", + "video_codec=h264", + "send_device_meta=false", + "send_dummy_byte=true", + "send_stream_meta=true", + "send_frame_meta=true", + "cleanup=true", + "max_size=720", + "max_fps=30", + "video_bit_rate=1500000", + ]); + }); + + it("reaches the server's abstract socket through Simlock's adb server, named by the session id", async () => { + const world = await setup(); + await opened(world); + expect(world.server.ports).toEqual([PORT]); + expect(world.server.last.written).toEqual([ + "001chost:transport:emulator-5586", + "001dlocalabstract:scrcpy_1234abcd", + ]); + }); + + it("hands out a body that starts with the session's meta frame", async () => { + const feed = await opened(await setup()); + const first = await feed.body.getReader().read(); + expect(frameTag(first.value)).toBe(SCREEN_FRAME_TAG.meta); + expect(JSON.parse(new TextDecoder().decode(first.value!.subarray(5)))).toEqual({ + codec: "h264", + fps: 30, + height: 64, + width: 96, + }); + }); + + it("retries every 100 ms while the server is not listening yet, then streams", async () => { + const world = await setup({ + server: new AdbServer(["connection-refused", "refuse", "refuse"]), + }); + const opening = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const started = world.clock.now(); + const watcher = (async () => { + while (world.server.peers.length < 3) await new Promise((resolve) => setImmediate(resolve)); + world.server.last.send(streamStart()); + })(); + const feed = await drive(world.clock, opening, 10); + await watcher; + expect(world.server.ports).toHaveLength(4); + expect(world.clock.now() - started).toBeGreaterThanOrEqual(3 * CONNECT_RETRY_MS); + expect(frameTag((await feed.body.getReader().read()).value)).toBe(SCREEN_FRAME_TAG.meta); + }); + + it("retries a connection that ends before the server's first byte", async () => { + const world = await setup({ server: new AdbServer(["silent"]) }); + const opening = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const watcher = (async () => { + while (world.server.peers.length < 1) await new Promise((resolve) => setImmediate(resolve)); + world.server.peers[0]!.send("OKAY", "OKAY"); + world.server.peers[0]!.close(); + while (world.server.peers.length < 2) await new Promise((resolve) => setImmediate(resolve)); + world.server.last.send(streamStart()); + })(); + await drive(world.clock, opening, 10); + await watcher; + expect(world.server.peers).toHaveLength(2); + }); + + it("gives up at the start deadline, reporting the server's last output, and cleans up", async () => { + const world = await setup({ + expectations: [push(), shell({ stderrLines: ["[server] ERROR: no encoder"] }), remove()], + server: new AdbServer([], "refuse"), + }); + const started = world.clock.now(); + const opening = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const error = await drive(world.clock, opening).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message).toMatch(/within 15 s/); + expect((error as Error).message).toMatch(/closed/); + expect((error as Error).message).toMatch(/no encoder/); + const failedAt = world.clock.now() - started; + expect(failedAt).toBeGreaterThanOrEqual(START_DEADLINE_MS); + expect(failedAt).toBeLessThan(START_DEADLINE_MS + CLOSE_DEADLINE_MS); + expect(world.runner.calls.map((call) => call.args.slice(2, 4))).toEqual([ + ["push", SERVER], + ["shell", `CLASSPATH=${DEVICE_JAR}`], + ["shell", "rm"], + ]); + expect(world.server.peers.every((peer) => peer.closed)).toBe(true); + }); + + it("fails at once when the server's shell exits before it listens", async () => { + const world = await setup({ + expectations: [ + push(), + shell({ + hangs: false, + result: { code: 1, stderr: "", stdout: "" }, + stderrLines: ["Aborted"], + }), + remove(), + ], + server: new AdbServer([], "refuse"), + }); + const started = world.clock.now(); + const error = await drive( + world.clock, + world.capture.open(DEVICE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + expect((error as Error).message).toMatch(/exited before it streamed/); + expect((error as Error).message).toMatch(/Aborted/); + expect(world.clock.now() - started).toBeLessThan(START_DEADLINE_MS); + }); + + it("refuses at once when adb has no transport for the serial", async () => { + const world = await setup({ server: new AdbServer(["no-transport"]) }); + const error = await drive( + world.clock, + world.capture.open(DEVICE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message).toMatch(/not found/); + expect(world.server.peers).toHaveLength(1); + }); + + it("refuses with the server's reason when it cannot configure its encoder", async () => { + const world = await setup(); + const opening = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const watcher = (async () => { + while (world.server.peers.length < 1) await new Promise((resolve) => setImmediate(resolve)); + world.server.last.send(codecId(1)); + })(); + const error = await drive(world.clock, opening).catch((caught: unknown) => caught); + await watcher; + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message).toMatch(/could not configure/); + }); + + it("reports adb's push failure without starting a server", async () => { + const world = await setup({ + expectations: [push({ code: 1, stderr: "device offline" }), remove()], + }); + const error = await drive( + world.clock, + world.capture.open(DEVICE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + expect((error as Error).message).toMatch(/device offline/); + expect(world.runner.calls.map((call) => call.args[2])).toEqual(["push", "shell"]); + expect(world.runner.calls[1]!.args.slice(3)).toEqual(["rm", "-f", DEVICE_JAR]); + }); + + it("refuses a physical device as not supported, before running anything", async () => { + const world = await setup({ expectations: [] }); + await expect( + world.capture.open( + { ...DEVICE, enrolledApps: [], physical: true } as PhysicalDriverDevice, + SETTINGS, + new AbortController().signal, + ), + ).rejects.toBeInstanceOf(ScreenNotSupportedError); + expect(world.runner.calls).toEqual([]); + }); + + it("starts no server when it is stopped while the jar is being copied, and removes the jar", async () => { + const stop = new AbortController(); + const world = await setup({ + expectations: [{ ...push(), onSpawn: () => stop.abort() }, remove()], + }); + const opening = world.capture.open(DEVICE, SETTINGS, stop.signal); + await expect(drive(world.clock, opening)).rejects.toBeInstanceOf(ScreenUnavailableError); + expect(world.runner.calls.map((call) => call.args.join(" "))).not.toContainEqual( + expect.stringContaining("app_process"), + ); + expect(world.runner.calls.at(-1)?.args.join(" ")).toContain("rm -f"); + }); + + it("refuses an open that was stopped before it began, running nothing", async () => { + const world = await setup({ expectations: [] }); + const stop = new AbortController(); + stop.abort(); + await expect(world.capture.open(DEVICE, SETTINGS, stop.signal)).rejects.toBeInstanceOf( + ScreenUnavailableError, + ); + expect(world.runner.calls).toEqual([]); + }); + + it("is unavailable, with the reason, while the server jar is missing", async () => { + expect(await (await setup()).capture.unavailable()).toBeUndefined(); + const world = await setup({ expectations: [], jar: false }); + expect(await world.capture.unavailable()).toMatch( + /scrcpy-server v5\.0\.1.*\/simlock\/scrcpy-server-v5\.0\.1/, + ); + await expect( + world.capture.open(DEVICE, SETTINGS, new AbortController().signal), + ).rejects.toBeInstanceOf(ScreenUnavailableError); + expect(world.runner.calls).toEqual([]); + }); + + it("gives two concurrent opens two session ids, two sockets and two jar paths", async () => { + const world = await setup({ + expectations: [push(), push(), shell(), shell()], + scid: null, + }); + const first = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const second = world.capture.open(DEVICE, SETTINGS, new AbortController().signal); + const watcher = (async () => { + while (world.server.peers.length < 2) await new Promise((resolve) => setImmediate(resolve)); + for (const peer of world.server.peers) peer.send(streamStart()); + })(); + await drive(world.clock, Promise.all([first, second])); + await watcher; + const sockets = world.server.peers.map((peer) => peer.written[1]); + expect(new Set(sockets).size).toBe(2); + const jars = world.runner.calls + .filter((call) => call.args[2] === "push") + .map((call) => call.args[4]); + expect(new Set(jars).size).toBe(2); + for (const socket of sockets) { + const hex = /scrcpy_([0-9a-f]{8})$/.exec(socket ?? "")?.[1]; + expect(hex).toBeDefined(); + expect(jars).toContain(`/data/local/tmp/simlock-scrcpy-${hex!}.jar`); + } + }); +}); + +describe("Android screen capture: the body", () => { + it("reads the socket only when its reader asks for a frame", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + const before = peer.pulls; + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + expect(peer.pulls).toBe(before); + const reader = feed.body.getReader(); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.meta); + expect(peer.pulls).toBe(before); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.description); + expect(peer.pulls).toBeGreaterThan(before); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.keyframe); + }); + + it("delivers every delta, in order, however slowly it is read", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + for (let index = 0; index < 50; index += 1) peer.send(mediaPacket({ pts: index }, SAMPLE.p1)); + const reader = feed.body.getReader(); + const tags: (number | undefined)[] = []; + for (let index = 0; index < 53; index += 1) { + for (let flush = 0; flush < 3; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + tags.push(frameTag((await reader.read()).value)); + } + expect(tags.filter((tag) => tag === SCREEN_FRAME_TAG.delta)).toHaveLength(50); + }); + + it("says a still screen is alive every few seconds, and keeps the frames that follow in order", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + const reader = feed.body.getReader(); + for (let index = 0; index < 3; index += 1) await reader.read(); + const quiet = reader.read(); + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + world.clock.advance(ALIVE_AFTER_MS); + const alive = (await quiet).value; + expect(frameTag(alive)).toBe(SCREEN_FRAME_TAG.alive); + expect(alive?.length).toBe(5); + const next = reader.read(); + peer.send(mediaPacket({ pts: 1 }, SAMPLE.p1)); + expect(frameTag((await next).value)).toBe(SCREEN_FRAME_TAG.delta); + }); + + it("says nothing is alive while the encoder has not painted a first picture", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config)); + const reader = feed.body.getReader(); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.meta); + const quiet = reader.read(); + let settled = false; + void quiet.then(() => (settled = true)); + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + world.clock.advance(ALIVE_AFTER_MS * 3); + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + }); + + it("errors with the server's reason when the device side ends, and cleans up", async () => { + const world = await setup({ + expectations: [push(), shell({ stderrLines: ["[server] ERROR: Encoding error"] }), remove()], + }); + const feed = await opened(world); + const reader = feed.body.getReader(); + await reader.read(); + world.server.last.close(); + const error = await drive(world.clock, reader.read()).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message).toMatch(/Encoding error/); + await drive(world.clock, feed.close()); + expect(world.runner.calls.at(-1)?.args.slice(2)).toEqual(["shell", "rm", "-f", DEVICE_JAR]); + }); +}); + +describe("Android screen capture: close", () => { + it("closes the socket, kills a server that outlives it, and removes the jar, once however often it is called", async () => { + const world = await setup(); + const feed = await opened(world); + const closes = Promise.all([feed.close(), feed.close()]); + await drive(world.clock, closes); + await drive(world.clock, feed.close()); + expect(world.server.last.closed).toBe(true); + expect(await hasEnded(world.runner.handles[1]!)).toBe(true); + expect(world.runner.calls.filter((call) => call.args.includes("rm"))).toHaveLength(1); + }); + + it("kills a server still running a second after its socket closed, with SIGKILL", async () => { + const world = await setup({ + expectations: [push(), shell({ ignoresSigterm: true }), remove()], + }); + const feed = await opened(world); + const closing = feed.close(); + const server = world.runner.handles[1]!; + await hasEnded(server); + world.clock.advance(999); + expect(await hasEnded(server)).toBe(false); + world.clock.advance(1); + expect(await hasEnded(server)).toBe(true); + await drive(world.clock, closing); + }); + + it("resolves within its deadline even when removing the jar never returns", async () => { + const world = await setup({ expectations: [push(), shell(), { ...remove(), hangs: true }] }); + const feed = await opened(world); + const started = world.clock.now(); + await drive(world.clock, feed.close()); + expect(world.clock.now() - started).toBeGreaterThanOrEqual(CLOSE_DEADLINE_MS); + expect(world.clock.now() - started).toBeLessThanOrEqual(CLOSE_DEADLINE_MS + 50); + }); + + it("cancelling the body closes the capture", async () => { + const world = await setup(); + const feed = await opened(world); + await drive(world.clock, feed.body.cancel()); + expect(world.server.last.closed).toBe(true); + expect(world.runner.calls.filter((call) => call.args.includes("rm"))).toHaveLength(1); + }); +}); diff --git a/src/drivers/android/screen/session.ts b/src/drivers/android/screen/session.ts new file mode 100644 index 00000000..5a6115f4 --- /dev/null +++ b/src/drivers/android/screen/session.ts @@ -0,0 +1,461 @@ +import { randomInt } from "node:crypto"; + +import { + encodeScreenFrame, + isPhysicalDriverDevice, + type PhysicalDriverDevice, + type ScreenCapture, + type ScreenFeed, + ScreenNotSupportedError, + type ScreenSettings, + ScreenUnavailableError, + SCREEN_FRAME_TAG, + type VirtualDriverDevice, +} from "../../../core/index.js"; +import type { + Clock, + Filesystem, + ProcessHandle, + ProcessRunner, + TcpStream, + TcpStreamConnector, +} from "../../../ports/index.js"; +import { ChunkReader, openDeviceSocket } from "./adb-socket.js"; +import { EnvelopeMapper } from "./envelope.js"; +import { ScrcpyVideoParser } from "./scrcpy-video.js"; + +// why: the server refuses to start unless its first argument equals the version it was built as, so +// this must match the vendored jar byte for byte. +const SCRCPY_SERVER_VERSION = "5.0.1"; +const DEVICE_DIRECTORY = "/data/local/tmp"; +const PUSH_TIMEOUT_MS = 30_000; +// why: measured from the moment the push returned, through every connect retry, to the first session +// packet; the budget never restarts, so a server that keeps refusing cannot hold a viewer longer. +export const START_DEADLINE_MS = 15_000; +// why: scrcpy's own client retries its connection every 100 ms while the server starts. +export const CONNECT_RETRY_MS = 100; +// why: scrcpy's own client gives the server one second to exit after its sockets close, then kills +// the adb process that runs it. +const SHELL_EXIT_GRACE_MS = 1_000; +const REMOVE_TIMEOUT_MS = 3_000; +export const CLOSE_DEADLINE_MS = 5_000; +// why: scrcpy's encoder repeats a still picture once and then sends nothing until the screen changes, +// so a still screen would look exactly like a dead capture to every silence bound downstream. While +// the socket is open the session says so with an empty `alive` frame, well inside those bounds. +export const ALIVE_AFTER_MS = 5_000; +const ALIVE_FRAME = encodeScreenFrame(SCREEN_FRAME_TAG.alive, new Uint8Array(0)); +const QUIET = Symbol("quiet"); +const LOG_TAIL_LINES = 20; +const SCID_LIMIT = 2 ** 31; + +export interface AndroidScreenCaptureOptions { + readonly adb: string; + readonly adbServerPort: number; + readonly clock: Clock; + readonly env: () => NodeJS.ProcessEnv; + readonly filesystem: Filesystem; + readonly processRunner: ProcessRunner; + readonly serialOf: (device: VirtualDriverDevice) => string; + readonly serverPath: string; + readonly tcp: TcpStreamConnector; + readonly scid?: () => number; +} + +export function createAndroidScreenCapture(options: AndroidScreenCaptureOptions): ScreenCapture { + const unavailable = async (): Promise => + (await options.filesystem.exists(options.serverPath)) + ? undefined + : `Android screen streams need scrcpy-server v${SCRCPY_SERVER_VERSION}, and it is not at ${options.serverPath}`; + return { + open: async ( + device: VirtualDriverDevice | PhysicalDriverDevice, + settings: ScreenSettings, + signal: AbortSignal, + ): Promise => { + if (isPhysicalDriverDevice(device)) { + throw new ScreenNotSupportedError( + "Screen streams are offered for Android emulators only, not for physical Android devices", + ); + } + const reason = await unavailable(); + if (reason !== undefined) throw new ScreenUnavailableError(reason); + const serial = options.serialOf(device); + if (serial === "") { + throw new ScreenUnavailableError( + `${device.deviceId} is not running, so it has no screen to stream`, + ); + } + const scid = options.scid?.() ?? randomInt(0, SCID_LIMIT); + return await new ScrcpySession(options, serial, scid, settings).start(signal); + }, + unavailable, + }; +} + +class ScrcpySession { + readonly #options: AndroidScreenCaptureOptions; + readonly #serial: string; + readonly #scidHex: string; + readonly #devicePath: string; + readonly #settings: ScreenSettings; + readonly #tail: string[] = []; + readonly #abort = new AbortController(); + #pushed = false; + #shell: ProcessHandle | undefined; + #shellExited = false; + #stream: TcpStream | undefined; + #closing: Promise | undefined; + + constructor( + options: AndroidScreenCaptureOptions, + serial: string, + scid: number, + settings: ScreenSettings, + ) { + if (!Number.isInteger(scid) || scid < 0 || scid >= SCID_LIMIT) { + throw new RangeError(`A scrcpy session id is a 31-bit number, not ${String(scid)}`); + } + this.#options = options; + this.#serial = serial; + // why: the server parses `scid` as hex and names its socket `scrcpy_%08x`, exactly as scrcpy's + // own client formats it, so one formatting serves the argument, the socket and the file name. + this.#scidHex = scid.toString(16).padStart(8, "0"); + this.#devicePath = `${DEVICE_DIRECTORY}/simlock-scrcpy-${this.#scidHex}.jar`; + this.#settings = settings; + } + + async start(stopped: AbortSignal): Promise { + let timer: ReturnType | undefined; + const stop = () => this.#abort.abort(); + stopped.addEventListener("abort", stop, { once: true }); + try { + if (stopped.aborted) throw this.#startFailure("was stopped before it started"); + await this.#push(); + // why: the push is the one step that cannot be interrupted; a stop that arrived during it + // starts no server on a device that is being handed back. + if (this.#abort.signal.aborted) + throw this.#startFailure("was stopped while it was being copied"); + this.#spawnServer(); + const deadlineAt = this.#options.clock.now() + START_DEADLINE_MS; + timer = this.#options.clock.setTimer(START_DEADLINE_MS, () => { + this.#abort.abort(); + }); + const reader = await this.#connect(deadlineAt); + const parser = new ScrcpyVideoParser(); + const mapper = new EnvelopeMapper(this.#settings.fps); + const pending = await this.#untilSized(reader, parser, mapper); + return { body: this.#body(reader, parser, mapper, pending), close: () => this.close() }; + } catch (error) { + await this.close(); + throw error instanceof ScreenUnavailableError + ? error + : new ScreenUnavailableError(this.#withTail(errorMessage(error))); + } finally { + stopped.removeEventListener("abort", stop); + if (timer !== undefined) this.#options.clock.cancel(timer); + } + } + + async #untilSized( + reader: ChunkReader, + parser: ScrcpyVideoParser, + mapper: EnvelopeMapper, + ): Promise { + const pending: Uint8Array[] = []; + let sized = false; + while (!sized) { + const chunk = await reader.read(); + if (chunk === undefined) throw this.#startFailure("ended before it named the picture's size"); + for (const event of parser.push(chunk)) { + if (event.kind === "session") sized = true; + pending.push(...mapper.map(event)); + } + } + return pending; + } + + close(): Promise { + this.#closing ??= this.#close(); + return this.#closing; + } + + async #push(): Promise { + const result = await this.#options.processRunner.run( + this.#options.adb, + ["-s", this.#serial, "push", this.#options.serverPath, this.#devicePath], + { env: this.#options.env(), timeoutMs: PUSH_TIMEOUT_MS }, + ); + // why: a push that timed out may still have written part of the file, so cleanup removes it too. + this.#pushed = true; + if (result.code !== 0) { + throw new ScreenUnavailableError( + `adb could not copy the screen server to ${this.#serial}: ${(result.stderr || result.stdout).trim()}`, + ); + } + } + + #spawnServer(): void { + const { fps, maxDimension, bitrateKbps } = this.#settings; + const shell = this.#options.processRunner.spawn( + this.#options.adb, + [ + "-s", + this.#serial, + "shell", + `CLASSPATH=${this.#devicePath}`, + "app_process", + "/", + "com.genymobile.scrcpy.Server", + SCRCPY_SERVER_VERSION, + `scid=${this.#scidHex}`, + "log_level=info", + "tunnel_forward=true", + "video=true", + "audio=false", + "control=false", + "video_codec=h264", + "send_device_meta=false", + "send_dummy_byte=true", + "send_stream_meta=true", + "send_frame_meta=true", + "cleanup=true", + `max_size=${String(maxDimension)}`, + `max_fps=${String(fps)}`, + `video_bit_rate=${String(bitrateKbps * 1000)}`, + ], + { env: this.#options.env() }, + ); + this.#shell = shell; + void this.#collect(shell.stdout); + void this.#collect(shell.stderr); + void shell.wait().then( + () => this.#onShellExit(), + () => this.#onShellExit(), + ); + } + + #onShellExit(): void { + this.#shellExited = true; + this.#abort.abort(); + } + + async #collect(lines: AsyncIterable): Promise { + try { + for await (const line of lines) { + if (line.trim() === "") continue; + this.#tail.push(line); + if (this.#tail.length > LOG_TAIL_LINES) this.#tail.shift(); + } + } catch { + // why: the output is only the reason a failure is reported with; losing it loses no stream. + } + } + + async #connect(deadlineAt: number): Promise { + const signal = this.#abort.signal; + let last = "no attempt finished"; + for (;;) { + if (signal.aborted) throw this.#startFailure(`did not accept a connection (${last})`); + const attempt = await this.#attempt(deadlineAt, signal); + if (typeof attempt !== "string") return attempt; + last = attempt; + await this.#sleep(CONNECT_RETRY_MS, signal); + } + } + + async #attempt(deadlineAt: number, signal: AbortSignal): Promise { + try { + const outcome = await openDeviceSocket({ + adbServerPort: this.#options.adbServerPort, + serial: this.#serial, + signal, + socketName: `scrcpy_${this.#scidHex}`, + tcp: this.#options.tcp, + timeoutMs: Math.max(1, deadlineAt - this.#options.clock.now()), + }); + if (outcome.kind === "no-transport") { + throw new ScreenUnavailableError(`adb cannot reach ${this.#serial}: ${outcome.reason}`); + } + if (outcome.kind !== "open") return outcome.reason; + this.#stream = outcome.stream; + if (signal.aborted) outcome.stream.close(); + else signal.addEventListener("abort", () => outcome.stream.close(), { once: true }); + // why: the server writes one zero byte the moment it accepts, so a connection that ends + // before it is one that reached nothing and is retried. + const dummy = await outcome.reader.readExactly(1); + if (dummy?.[0] === 0) return outcome.reader; + outcome.stream.close(); + return dummy === undefined + ? "the socket closed before the server spoke" + : "the socket is not scrcpy's"; + } catch (error) { + if (error instanceof ScreenUnavailableError) throw error; + return errorMessage(error); + } + } + + #sleep(delayMs: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) { + resolve(); + return; + } + const timer = this.#options.clock.setTimer(delayMs, () => { + signal.removeEventListener("abort", wake); + resolve(); + }); + const wake = () => { + this.#options.clock.cancel(timer); + resolve(); + }; + signal.addEventListener("abort", wake, { once: true }); + }); + } + + #startFailure(what: string): ScreenUnavailableError { + const server = `The screen server on ${this.#serial}`; + const reason = this.#shellExited + ? `${server} exited before it streamed` + : this.#abort.signal.aborted + ? `${server} did not start streaming within ${String(START_DEADLINE_MS / 1000)} s: it ${what}` + : `${server} ${what}`; + return new ScreenUnavailableError(this.#withTail(reason)); + } + + #withTail(message: string): string { + return this.#tail.length === 0 + ? message + : `${message}. Its last output:\n${this.#tail.join("\n")}`; + } + + #body( + reader: ChunkReader, + parser: ScrcpyVideoParser, + mapper: EnvelopeMapper, + pending: Uint8Array[], + ): ReadableStream { + // why: a read the quiet timer outran is still owed its chunk; the next pull awaits that read + // rather than starting a second one, which would deliver the stream's bytes out of order. + let inflight: Promise | undefined; + // why: alive means "the picture you have is still the picture", so nothing is said to be alive + // before a first picture exists; an encoder that never paints one meets the silence bound. + let started = false; + return new ReadableStream( + { + // why: the socket is read only when the body's reader asks for a frame, so a viewer that + // falls behind stops this read, TCP stops the adb transport, and the device's encoder + // waits; nothing is dropped and nothing queues here beyond one socket read. + // fallow-ignore-next-line complexity -- one read: a frame, the quiet signal, or the end. + pull: async (controller) => { + try { + while (pending.length === 0) { + inflight ??= reader.read(); + const chunk = started ? await this.#readOrQuiet(inflight) : await inflight; + if (chunk === QUIET) { + controller.enqueue(ALIVE_FRAME); + return; + } + inflight = undefined; + if (chunk === undefined) { + if (this.#closing !== undefined) controller.close(); + else + controller.error( + new ScreenUnavailableError( + this.#withTail("The screen server stopped streaming"), + ), + ); + void this.close(); + return; + } + for (const event of parser.push(chunk)) pending.push(...mapper.map(event)); + } + const frame = pending.shift(); + if (frame !== undefined) { + if (frame[4] === SCREEN_FRAME_TAG.keyframe) started = true; + controller.enqueue(frame); + } + } catch (error) { + controller.error(error); + void this.close(); + } + }, + cancel: () => this.close(), + }, + { highWaterMark: 0 }, + ); + } + + #readOrQuiet( + read: Promise, + ): Promise { + return new Promise((resolve, reject) => { + const timer = this.#options.clock.setTimer(ALIVE_AFTER_MS, () => resolve(QUIET)); + read.then( + (chunk) => { + this.#options.clock.cancel(timer); + resolve(chunk); + }, + (error: unknown) => { + this.#options.clock.cancel(timer); + reject(error instanceof Error ? error : new Error(String(error))); + }, + ); + }); + } + + async #close(): Promise { + let timer: ReturnType | undefined; + const bound = new Promise((resolve) => { + timer = this.#options.clock.setTimer(CLOSE_DEADLINE_MS, resolve); + }); + try { + await Promise.race([this.#teardown(), bound]); + } finally { + if (timer !== undefined) this.#options.clock.cancel(timer); + } + } + + async #teardown(): Promise { + this.#abort.abort(); + // why: the server exits when a write to its video socket fails, which is the stop it was built + // for; the kill below only covers a screen so still that the server has nothing to write. + this.#stream?.close(); + const shell = this.#shell; + if (shell !== undefined && !(await this.#exitsWithin(shell, SHELL_EXIT_GRACE_MS))) { + shell.kill("SIGKILL"); + await this.#exitsWithin(shell, SHELL_EXIT_GRACE_MS); + } + if (this.#pushed) { + // why: with cleanup=true the server unlinks its own jar as it starts, so this only matters for + // a server that never got that far; it runs anyway because that case is not visible from here. + await this.#options.processRunner + .run(this.#options.adb, ["-s", this.#serial, "shell", "rm", "-f", this.#devicePath], { + env: this.#options.env(), + timeoutMs: REMOVE_TIMEOUT_MS, + }) + .catch(() => undefined); + } + } + + async #exitsWithin(shell: ProcessHandle, delayMs: number): Promise { + if (this.#shellExited) return true; + let timer: ReturnType | undefined; + try { + return await Promise.race([ + shell.wait().then( + () => true, + () => true, + ), + new Promise((resolve) => { + timer = this.#options.clock.setTimer(delayMs, () => resolve(false)); + }), + ]); + } finally { + if (timer !== undefined) this.#options.clock.cancel(timer); + } + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/drivers/android/screen/test-fixture.ts b/src/drivers/android/screen/test-fixture.ts new file mode 100644 index 00000000..3cc6ece5 --- /dev/null +++ b/src/drivers/android/screen/test-fixture.ts @@ -0,0 +1,44 @@ +// why: real encoder output rather than hand-made bytes, so the parser and the avcC record are checked +// against what an H.264 encoder writes. Made with +// `ffmpeg -f lavfi -i testsrc=size=96x64:rate=10 -frames:v 5 -c:v libx264 -profile:v baseline +// -level 3.0 -pix_fmt yuv420p -g 3 -bsf:v filter_units=remove_types=6 -f h264 sample.h264`: an IDR, two +// P frames, then SPS, PPS and a second IDR in-band, then a P frame (https://ffmpeg.org). Base64 because a +// raw H.264 file holds NUL bytes, which the repo keeps out of tracked text files. +export const SAMPLE_H264 = Buffer.from( + [ + "AAAAAWdCwB7ZBibARAAAAwAEAAADAFA8WLkgAAAAAWjLg8sgAAABZYiEN/8P7R0UAAQI/igACAJ4AF6W/0Cxzk/aDiibv27/", + "yCC7OEAATDHnQdAmIAPs7syCKf+wcAIAViAB4NBLraKLFwakOXK0oeNwAWvPIsQ7zD9dACz7qA/1yAAcY1QAHsH/Pv7/9//9", + "3hwgAEGBoNBAACAgAAIBoAD0wEMcTXDv/XW/f//8D8Ag0YiBzeWDhD9dd9/f7/7GpVKoPw//grxQABAH4gACAGg4AAgFAACC", + "TEgbGDwNiIACAEYwcAEAIxGYDgACAMAOHAACEImCAAIAAiNxAABAKNg8DYg8DYg4AIARiDgAgBGP4H8SwWQALanIFdcGW/fg", + "AD8AWNu8D/XIQAJgAocEAAIAwA4AAgBgNYAZWEtoR/66/gAORUqVwcjJywLOQ+H0onw3igACAN+AC0uImUKc/7bwhM66/e+A", + "hMhpitaixGCAgC4QEAAdAAEDZ4PEQJjAAwA1L2KEUMQAIAVjAAfNgE68rMIf4tAP7Ewe0LDhAAGMhdKhyiVsTYPb6wwmcg2h", + "/3gW/SAzjRxdPv6KiBfI/cq7kgBZLkOAslxO6d0/5Q+kMF0ACyNjsM9BNUWtn/+AoXwcv2sY125UeARMjJDbqZWO/iYzJdnK", + "yq2UIACO9hvC7L4HIRJekPj+wv/hmMWaiOarJJj/TAACIGC5pgZYAB2Hr//SlghgAI2M/0kIUiK5g/EZBe7l7MnHUGKfFzab", + "ebRT35t5tinmTjstbL/6b+BbDm/ZgIEznZr8YAAIAQO8QmAaQnQxzyKxuS//+7gVM3F9v1qU5v/3h3CBYZoQABgCh5iUEUtb", + "LbsOCMbL//tb2C3AAoyC6VBUo3Ob/98aGfaykIRUcWX/Q+IfhgEODoAAgDcHQvxEFwdC+Ac4RgFpwK7VH4K11ALo6BKPCgAC", + "AGBgCQgADQIPBwUwoZAOs5YIszCyCQ8SHA5E0sC8gkA6xUQHHq4GQaycnxA8QPwCFaTRNt/UYPeCFXAlyCRNQ9D7+EAEA4k0", + "GAAIBCgACAeW3gGHkfGQH+eIEbywqBUwIWQEMyGg5Ge48O/PGMN6f/gEADXXUX3AeIgPLfDGW4QCMeAgIAQBCAFngLAXAaB8", + "DqBOl4cm4EIh+YBLpESFL/8IgMHg98Hh/B4Pg8P4HYx0NQCy7o8WeLBxgxYvjCQABBAUAAQKggADYDhJcqnINWqwgCI6WQAR", + "iJYUBkDgYAFsBxDLLAonGaxUfhgAciWfif2xJ+8kI34AD6X98LK8/4OQfhT6h+kCAAKA6+CAAIQADgCuTvwAuZpYuFWp4sBg", + "2HR1+AQMa1pqTViCUocA3AgAPwfcV6inqoBxDODS377d4UIlCAAQMPFhQMUDpQWTrH5A3hYmEYN+0QQiS4AAAAABQZo4I4TL", + "4yOkj+Xx0ZG8Ec6t5Qe+hkbXLL46O4+KJe4fZb9DOrmgpzxQ2Jw/gAAAAAFBmlQI4TJ4hr4bhwgYyxhl9frPsnl+5N704bn5", + "ZqjEtm/J56iouCfe7j+/c0vhvEGMt/ny1kNaxnJm83gAAAABZ0LAHtkGJsBEAAADAAQAAAMAUDxYuSAAAAABaMuDyyAAAAFl", + "iIIN/8P7R0UAAQI/igACAJ4AF6W/0Cxzk/aDiibv27/yCC7OEAATDHnQdAmIAPs7syCKf+wcAIAViAB4NBLraKLFwakOXK0o", + "eNwAWvPIsQ7zD9dACz7qA/1yAAcY1QAHsH/Pv7/9//93hwgAEGBoNBAACAgAAIBoAD0wEMcTXDv/XW/f//8D8Ag0YiBzeWDh", + "D9dd9/f7/7GpVKoPw//grxQABAH4gACAGg4AAgFAACCTEgbGDwNiIACAEYwcAEAIxGYDgACAMAOHAACEImCAAIAAiNxAABAK", + "Ng8DYg8DYg4AIARiDgAgBGP4H8SwWQALanIFdcGW/fgAD8AWNu8D/XIQAJgAocEAAIAwA4AAgBgNYAZWEtoR/66/gAORUqVw", + "cjJywLOQ+H0onw3igACAN+AC0uImUKc/7bwhM66/e+AhMhpitaixGCAgC4QEAAdAAEDZ4PEQJjAAwA1L2KEUMQAIAVjAAfNg", + "E68rMIf4tAP7Ewe0LDhAAGMhdKhyiVsTYPb6wwmcg2h/3gW/SAzjRxdPv6KiBfI/cq7kgBZLkOAslxO5mV6p/yh9BDBdAAcQ", + "2PQYWo+qPeRP//4BKgOel7nPM3sj3kRidkKR3E9Cf8JSideYlOf6M8IAAkHLKN4NXLtwOExpekPj+wuT+GYxZqI5qskmP9MA", + "AIgYLmmBlgAHYev/9KWC6AAjYz/SQhSIrmD8RkF7uXsycdQa+bfinrm028alimXyrNvRNGKeZOOy1svv9N/IisOb9mAICcor", + "tfjAABAIAc8cmAaIZFOMLMzn9L//7uAVGRZKSU1605//3hvCAEgzwgADwBQNFoeYpa2W3YcBDCZZd//a3sFpIAFGQXSoKlG5", + "zf/vjQz7WUhCKjilv+h8Qw4YBDg6AAIDHB0F+IguIfgOBEIBMsCstUfiuiwdMGBJbwMAAIBYIAJCAANIA4AoQeojQ6Y5YDx3", + "jcloSHigYeRSwDfkEn4H+1EMOsDINaaeIH8AhWpyN7+o6/AOVcCXIDyIBy38IAICixYIAAQBoUAAQD7LGGGR8ZAf5zIjliQP", + "FJVAIZkNByM9xod+eITHL/4BAAgsXXFweP5MAchGB5b4R9eEAhjwGBAAgBnAWWAxQZhB5A8il8hVwAw2H9gE+0QREl/+EAGD", + "weHweD+DwfE/gcGHFkIoBZd0eCz3QDgnHyW8DgACCRQABA6CAANjhaZVOQNdhZgcAjESx4BgxFOcDw4GABbAdMcsBIxnTu8/", + "4B5EsSeJ/PP4ppZR7zCdAA/S74JFc/W+sA/T7j9IGAAWAVLDAAJAAUA45Jjh5oAuZpYlCrU8TgMIJhwnr4CxjWtdSasQSlEc", + "A3AgAA9QfFYOi+ikymqDglKDS3fa4QeIWEACAg8SOAsChloJgnZdYfNQG8LEwjBv2iCIkuAAAAABQZo4O4TL4yM42iPod9Dv", + "oZ1rQzr2h3FZvN5L4A==", + ].join(""), + "base64", +); diff --git a/src/drivers/android/screen/test-scrcpy.ts b/src/drivers/android/screen/test-scrcpy.ts new file mode 100644 index 00000000..7b06a949 --- /dev/null +++ b/src/drivers/android/screen/test-scrcpy.ts @@ -0,0 +1,69 @@ +import { SCRCPY_CODEC_H264 } from "./scrcpy-video.js"; +import { SAMPLE_H264 } from "./test-fixture.js"; + +export function concatBytes(...parts: Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +export function codecId(id = SCRCPY_CODEC_H264): Uint8Array { + const out = new Uint8Array(4); + new DataView(out.buffer).setUint32(0, id); + return out; +} + +export function sessionPacket(width: number, height: number, clientResized = false): Uint8Array { + const out = new Uint8Array(12); + const view = new DataView(out.buffer); + view.setUint32(0, 0x80000000 | (clientResized ? 1 : 0)); + view.setUint32(4, width); + view.setUint32(8, height); + return out; +} + +export function mediaPacket( + flags: { readonly config?: boolean; readonly key?: boolean; readonly pts?: number }, + payload: Uint8Array, + claimedSize = payload.length, +): Uint8Array { + const out = new Uint8Array(12 + payload.length); + const view = new DataView(out.buffer); + const pts = flags.pts ?? 0; + let high = Math.floor(pts / 2 ** 32); + if (flags.config === true) high = 0x40000000; + else if (flags.key === true) high |= 0x20000000; + view.setUint32(0, high); + view.setUint32(4, flags.config === true ? 0 : pts % 2 ** 32); + view.setUint32(8, claimedSize); + out.set(payload, 12); + return out; +} + +// why: the fixture cut where an encoder hands its output over -- parameter sets, then one access unit +// per packet -- with the offsets read from the file's own start codes. +export const SAMPLE = { + config: SAMPLE_H264.subarray(0, 36), + idr: SAMPLE_H264.subarray(36, 1070), + p1: SAMPLE_H264.subarray(1070, 1120), + p2: SAMPLE_H264.subarray(1120, 1184), + idrWithParameterSets: SAMPLE_H264.subarray(1184, 2285), + p3: SAMPLE_H264.subarray(2285), +}; + +export function sampleStream(): Uint8Array { + return concatBytes( + codecId(), + sessionPacket(96, 64), + mediaPacket({ config: true }, SAMPLE.config), + mediaPacket({ key: true, pts: 0 }, SAMPLE.idr), + mediaPacket({ pts: 100_000 }, SAMPLE.p1), + mediaPacket({ pts: 200_000 }, SAMPLE.p2), + mediaPacket({ key: true, pts: 300_000 }, SAMPLE.idrWithParameterSets), + mediaPacket({ pts: 400_000 }, SAMPLE.p3), + ); +} diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 5c6648df..4dde270e 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -14,6 +14,7 @@ Parent: [drivers](../AGENTS.md). Flows: [device selection](../../../docs/interna | `devicectl-exec.ts` | `devicectlExecCommand`: the reviewed `devicectl` list | | `slim-labels.ts` | `SLIM_CATEGORIES`, `resolveSlimCategories`, `labelsFor`, `slimSignature` | | `prerequisites.ts` | `iosPrerequisites`: Xcode selected, license accepted, first launch done | +| `screen-capture.ts` | `createIosScreenCapture`: runs `vendor/screen-capture/` on a simulator | ## How it works @@ -78,6 +79,12 @@ Passthrough — Lease environment: virtual `SIMLOCK_IOS_DEVICE_SET=`; physical `SIMLOCK_DEVICE_UDID=`. +Screen — `screenCapture` (only when `create` is given a launcher): `simlock-screen-capture +--codec avcc --fps --bitrate --max-dimension` with `SIMLOCK_IOS_DEVICE_SET=`; stdout is +the frames, stdin its liveness. Close: stdin, 2 s, SIGKILL. Physical → `ScreenNotSupportedError`; +program missing → `ScreenUnavailableError`. Default program: dist/bin, built by `pnpm build`; +`drivers.ios.screenCaptureCommand` names another. + Advisories — `runtime-cache-unreclaimable` (asset bundles no installed runtime mounts); `slim-runtime-unsupported` (installed runtimes below 18.5, only when `ios.defaultMode` is `slim`). @@ -115,4 +122,8 @@ record of a device's mode; `driverData` holds the slim signature. - **Slim drift is invisible.** `launchctl disable` accepts labels that do not exist. Narrowing `ios.slim.categories` never re-enables a daemon: reclaim or destroy old devices first. - **A recovered slim device runs full but its record says `slim` (#174)** until its next `makeReady`. +- **A screen stream can freeze after a reboot under its lease.** The capture program checks the + simulator is booted only when it starts, then repaints its last surface every 200 ms, so after + the health monitor reboots a crashed simulator the viewer can see the old picture until they + press Stop and Show screen again. A lease ending stops it as usual. - **A cold slim lease takes ~160 s.** MCP clients need `resetTimeoutOnProgress` or a longer timeout. diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 54e00a44..3d16b560 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -43,8 +43,10 @@ import { type DeviceMode, type DeviceSpec, type VirtualSpec, + type ScreenCapture, } from "../../core/index.js"; import type { + BytePipeLauncher, Clock, Filesystem, IdGenerator, @@ -54,6 +56,7 @@ import type { import { runBoundedProcess, runInstallerProcess } from "../installer-process.js"; import { devicectlExecCommand } from "./devicectl-exec.js"; import { IosPhysicalHandler } from "./physical.js"; +import { createIosScreenCapture } from "./screen-capture.js"; import { labelsFor, resolveSlimCategories, slimSignature } from "./slim-labels.js"; const COMMAND_TIMEOUT_MS = 30_000; @@ -206,6 +209,14 @@ export interface IosSimctlDriverOptions { readonly bootTimeoutMs: number; }; readonly slimByDefault?: boolean; + // why: absent when this host has no way to run the capture program; the driver then offers no + // screen at all rather than one that fails on every open. + readonly screenCapture?: { + readonly command: string; + readonly launcher: BytePipeLauncher; + readonly baseEnvironment: NodeJS.ProcessEnv; + readonly onLog?: (line: string) => void; + }; } type SlimOptions = NonNullable; @@ -306,7 +317,23 @@ export class IosSimctlDriver implements Driver { this.#rootOptions = rootOptions; this.#slim = options.slim; this.#slimByDefault = options.slimByDefault === true; - } + this.screenCapture = + options.screenCapture === undefined + ? undefined + : createIosScreenCapture({ + baseEnvironment: options.screenCapture.baseEnvironment, + clock: options.clock, + command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, + deviceSet: deviceRoot, + filesystem: options.filesystem, + launcher: options.screenCapture.launcher, + ...(options.screenCapture.onLog === undefined + ? {} + : { onLog: options.screenCapture.onLog }), + }); + } + + readonly screenCapture: ScreenCapture | undefined; /** * why: Establishes the device set this driver owns before it can be asked to do anything, @@ -1633,6 +1660,11 @@ class IosRuntimeUnavailableForDownloadError extends RuntimeMissingError { } } +function configuredScreenCaptureCommand(options: IosSimctlDriverOptions): string | undefined { + const configured = options.driverConfig["screenCaptureCommand"]; + return typeof configured === "string" && configured !== "" ? configured : undefined; +} + function configuredDeviceRoot(options: IosSimctlDriverOptions): string { const configured = options.driverConfig["deviceRoot"]; diff --git a/src/drivers/ios/screen-capture.test.ts b/src/drivers/ios/screen-capture.test.ts new file mode 100644 index 00000000..540d4e64 --- /dev/null +++ b/src/drivers/ios/screen-capture.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; + +import { ScreenNotSupportedError, ScreenUnavailableError } from "../../core/index.js"; +import { FakeBytePipeLauncher, FakeClock, MemoryFilesystem } from "../../ports/index.js"; +import { createIosScreenCapture } from "./screen-capture.js"; + +const settings = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; +const simulator = { + address: "UDID-1", + deviceId: "UDID-1", + driverData: {}, + physical: false, +} as const; +const COMMAND = "/opt/simlock/simlock-screen-capture"; + +async function setup() { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/opt/simlock"); + await filesystem.writeFileAtomic(COMMAND, ""); + const launcher = new FakeBytePipeLauncher(); + const clock = new FakeClock(); + const capture = createIosScreenCapture({ + baseEnvironment: { PATH: "/usr/bin" }, + clock, + command: COMMAND, + deviceSet: "/home/.simlock/devices/ios", + filesystem, + launcher, + }); + return { capture, clock, filesystem, launcher }; +} + +async function readAll(stream: ReadableStream): Promise { + const bytes: number[] = []; + for await (const chunk of stream) bytes.push(...chunk); + return bytes; +} + +describe("iOS screen capture", () => { + it("an open whose signal was already stopped starts nothing", async () => { + const { capture, launcher } = await setup(); + const stopped = new AbortController(); + stopped.abort(); + await expect(capture.open(simulator, settings, stopped.signal)).rejects.toBeInstanceOf( + ScreenUnavailableError, + ); + expect(launcher.starts).toEqual([]); + }); + + it("runs the capture program on the device's UDID, inside Simlock's device set, with the stream's settings", async () => { + const { capture, launcher } = await setup(); + await capture.open(simulator, settings, new AbortController().signal); + expect(launcher.starts).toEqual([ + { + args: [ + "UDID-1", + "--codec", + "avcc", + "--fps", + "30", + "--bitrate", + "1500000", + "--max-dimension", + "720", + ], + command: COMMAND, + env: { PATH: "/usr/bin", SIMLOCK_IOS_DEVICE_SET: "/home/.simlock/devices/ios" }, + }, + ]); + }); + + it("refuses a physical iPhone as not supported, without starting anything", async () => { + const { capture, launcher } = await setup(); + await expect( + capture.open( + { ...simulator, enrolledApps: [], physical: true }, + settings, + new AbortController().signal, + ), + ).rejects.toBeInstanceOf(ScreenNotSupportedError); + expect(launcher.starts).toEqual([]); + }); + + it("refuses as unavailable, and says why, when the capture program is not installed", async () => { + const { capture, filesystem, launcher } = await setup(); + await filesystem.rm(COMMAND); + expect(await capture.unavailable()).toBe(`No screen capture program at ${COMMAND}`); + await expect( + capture.open(simulator, settings, new AbortController().signal), + ).rejects.toBeInstanceOf(ScreenUnavailableError); + expect(launcher.starts).toEqual([]); + }); + + it("hands out the program's stdout byte for byte and ends cleanly when it exits 0", async () => { + const { capture, launcher } = await setup(); + const feed = await capture.open(simulator, settings, new AbortController().signal); + launcher.last.write(new Uint8Array([1, 2, 3])); + launcher.last.exit({ code: 0, signal: null }); + expect(await readAll(feed.body)).toEqual([1, 2, 3]); + }); + + it("fails the body with the program's last log line when it dies mid-picture", async () => { + const { capture, launcher } = await setup(); + const feed = await capture.open(simulator, settings, new AbortController().signal); + launcher.last.stderr("Device UDID-1 shut down"); + launcher.last.exit({ code: 3, signal: null }); + await expect(readAll(feed.body)).rejects.toThrow( + "The screen capture program exited with code 3: Device UDID-1 shut down", + ); + }); + + it("close stops the program through stdin and resolves once it has exited", async () => { + const { capture, launcher } = await setup(); + const feed = await capture.open(simulator, settings, new AbortController().signal); + launcher.last.exitOnStdinClose = false; + let closed = false; + const closing = feed.close().then(() => (closed = true)); + for (let turn = 0; turn < 10; turn += 1) await new Promise((resolve) => setImmediate(resolve)); + expect(launcher.last.stdinClosed).toBe(true); + expect(closed).toBe(false); + launcher.last.exit({ code: 0, signal: null }); + await closing; + expect(launcher.last.kills).toEqual([]); + }); + + it("close kills a program that ignores its closed stdin once the grace runs out", async () => { + const { capture, clock, launcher } = await setup(); + const feed = await capture.open(simulator, settings, new AbortController().signal); + launcher.last.exitOnStdinClose = false; + const closed = feed.close(); + await Promise.resolve(); + clock.advance(1_999); + await Promise.resolve(); + expect(launcher.last.kills).toEqual([]); + clock.advance(1); + await closed; + expect(launcher.last.kills).toEqual(["SIGKILL"]); + }); + + it("cancelling the body stops the program, and a second close does not stop it again", async () => { + const { capture, launcher } = await setup(); + const feed = await capture.open(simulator, settings, new AbortController().signal); + await feed.body.cancel(); + await feed.close(); + expect(launcher.last.stdinCloses).toBe(1); + expect(launcher.last.kills).toEqual([]); + expect(launcher.processes).toHaveLength(1); + }); +}); diff --git a/src/drivers/ios/screen-capture.ts b/src/drivers/ios/screen-capture.ts new file mode 100644 index 00000000..3681f6e2 --- /dev/null +++ b/src/drivers/ios/screen-capture.ts @@ -0,0 +1,150 @@ +import { + isPhysicalDriverDevice, + type ScreenCapture, + type ScreenFeed, + ScreenNotSupportedError, + type ScreenSettings, + ScreenUnavailableError, +} from "../../core/index.js"; +import type { BytePipeLauncher, BytePipeProcess, Clock, Filesystem } from "../../ports/index.js"; + +export interface IosScreenCaptureOptions { + readonly command: string; + readonly launcher: BytePipeLauncher; + readonly clock: Clock; + readonly filesystem: Filesystem; + readonly baseEnvironment: NodeJS.ProcessEnv; + readonly deviceSet: string; + readonly onLog?: (line: string) => void; +} + +// why: closing stdin is the stop the capture program was built for, and it needs a moment to flush +// its last log line; a program that ignores its liveness handle is killed after this, so a stuck +// encoder cannot pin a simulator's framebuffer. +const STOP_GRACE_MS = 2_000; +// why: a SIGKILLed child is reaped by the kernel at once; the bound exists only so a close cannot +// wait forever on an exit that the OS never reports. +const KILL_WAIT_MS = 2_000; +const LOG_TAIL = 20; + +export function createIosScreenCapture(options: IosScreenCaptureOptions): ScreenCapture { + // why: only a command that spells a path can be checked; a bare program name is resolved by the + // OS at spawn, and a failure there is reported with the spawn's own words. + const missing = async (): Promise => + options.command.includes("/") && !(await options.filesystem.exists(options.command)) + ? `No screen capture program at ${options.command}` + : undefined; + + return { + open: async (device, settings, signal) => { + if (isPhysicalDriverDevice(device)) { + throw new ScreenNotSupportedError( + "A physical iPhone's screen cannot be streamed; only a simulator's can", + ); + } + const reason = await missing(); + if (reason !== undefined) throw new ScreenUnavailableError(reason); + if (signal.aborted) + throw new ScreenUnavailableError("The capture was stopped before it started"); + return startCapture(options, device.deviceId, settings); + }, + unavailable: missing, + }; +} + +function startCapture( + options: IosScreenCaptureOptions, + udid: string, + settings: ScreenSettings, +): ScreenFeed { + const tail: string[] = []; + const child = options.launcher.start( + options.command, + [ + udid, + "--codec", + "avcc", + "--fps", + String(settings.fps), + "--bitrate", + String(settings.bitrateKbps * 1_000), + "--max-dimension", + String(settings.maxDimension), + ], + { + env: { ...options.baseEnvironment, SIMLOCK_IOS_DEVICE_SET: options.deviceSet }, + onStderrLine: (line) => { + tail.push(line); + if (tail.length > LOG_TAIL) tail.shift(); + options.onLog?.(line); + }, + }, + ); + const close = once(() => stop(child, options.clock)); + return { body: bodyOf(child, tail, close), close }; +} + +// why: a program that dies mid-picture ends its stdout like one that finished, so the body asks how +// it exited and fails with its last log line rather than ending cleanly -- a viewer must be able to +// tell a simulator that shut down from a stream that was asked to stop. +function bodyOf( + child: BytePipeProcess, + tail: readonly string[], + close: () => Promise, +): ReadableStream { + const reader = child.stdout.getReader(); + return new ReadableStream({ + cancel: async (reason) => { + await reader.cancel(reason).catch(() => undefined); + await close(); + }, + pull: async (controller) => { + const step = await reader.read().catch((error: unknown) => ({ error })); + if ("error" in step) { + await close(); + controller.error(step.error); + return; + } + if (!step.done) { + controller.enqueue(step.value); + return; + } + const exit = await child.exited; + await close(); + if (exit.code === 0) controller.close(); + else { + const how = + exit.error ?? + (exit.signal === null + ? `exited with code ${String(exit.code)}` + : `was killed by ${exit.signal}`); + const last = tail.at(-1); + controller.error( + new Error(`The screen capture program ${how}${last === undefined ? "" : `: ${last}`}`), + ); + } + }, + }); +} + +async function stop(child: BytePipeProcess, clock: Clock): Promise { + child.closeStdin(); + if (await exitsWithin(child, clock, STOP_GRACE_MS)) return; + child.kill("SIGKILL"); + await exitsWithin(child, clock, KILL_WAIT_MS); +} + +function exitsWithin(child: BytePipeProcess, clock: Clock, ms: number): Promise { + return new Promise((resolve) => { + const timer = clock.setTimer(ms, () => resolve(false)); + void child.exited.then(() => { + clock.cancel(timer); + resolve(true); + }); + }); +} + +function once(fn: () => Promise): () => Promise { + let running: Promise | undefined; + return () => (running ??= fn()); +} diff --git a/src/gateway/AGENTS.md b/src/gateway/AGENTS.md index 4f1761ee..270d1e46 100644 --- a/src/gateway/AGENTS.md +++ b/src/gateway/AGENTS.md @@ -89,12 +89,12 @@ Exec forwards the namespaced requester; `gateway.execTimeoutMs` is a backstop ov The lease index is in memory, rebuilt from each view's leases whose requester starts `gw::`; missing from 2 snapshots in a row → dropped; kept across a disconnect, forgotten with the view. -**HTTP relay** (`GatewayRelay.relay`, used by `../agent-device/front/relay-host-link.ts`): +**HTTP relay** (`GatewayRelay.relay`, used by `../agent-device/front/relay-host-link.ts` and `../stream/`): -1. `openChannel` on the link, then `relay.request` (`relayId`, `target`, `method`, `url`, headers - minus credentials, `tenant`, `leaseId`, `hasBody`, `channel`). +1. `openChannel`, then `relay.request` (ids, target, method, url, headers minus credentials, `tenant`, `channel`). 2. Request body up and response body down on that byte channel; head from the `relay-head` push. -3. Caller abort or cancel → `relay.cancel`. Deadline 330 s for the whole exchange → `WORKER_UNREACHABLE`. +3. Caller abort or cancel → `relay.cancel`. Deadline for the channel and head, then an idle bound on + the body (a live body runs while bytes come), both on the Clock → `WORKER_UNREACHABLE`, `relay.cancel`. 4. Worker: `relay.*` only on the uplink session (`FORBIDDEN` elsewhere; a gateway refuses them from everyone). Target from `relayTargets`; unknown → `RELAY_TARGET_MISSING`. @@ -113,7 +113,7 @@ views. `worker.install-component` fans out. `nuke.run`, `cleanup.run`, `doctor.r | Forwarded `lease.request` | 5 min | `gateway.leaseRequestTimeoutMs` | | Forwarded `device.exec` | 11 min | `gateway.execTimeoutMs` | | Disconnected view kept | 24 h, held while a gateway lease is live | `gateway.disconnectedRetentionMs` | -| Relay exchange | 330 s | `GatewayRelayOptions.deadlineMs` | +| Relay: channel + head / silent body | 330 s / 330 s | `GatewayRelayOptions.deadlineMs` / `.idleMs` | | Link: channels per side, accept, stall | 64, 30 s, 60 s | `../link/` constants | | Request records | 10 000, 10 min | `lease.maxRequestRecords`, `lease.requestRetentionMs` | diff --git a/src/gateway/relay.test.ts b/src/gateway/relay.test.ts index 9164ffd2..d3862b10 100644 --- a/src/gateway/relay.test.ts +++ b/src/gateway/relay.test.ts @@ -108,7 +108,13 @@ class FakeLink extends ScriptedConnection implements LinkConnection { } } -async function setup(options: { readonly deadlineMs?: number; readonly holdOpen?: boolean } = {}) { +async function setup( + options: { + readonly deadlineMs?: number; + readonly holdOpen?: boolean; + readonly idleMs?: number; + } = {}, +) { const connection = new FakeLink(); connection.holdOpen = options.holdOpen ?? false; const connecting = connectSimlockClient({ connection, credential: "secret" }, true); @@ -126,6 +132,7 @@ async function setup(options: { readonly deadlineMs?: number; readonly holdOpen? relay: new GatewayRelay({ directory, ...(options.deadlineMs === undefined ? {} : { deadlineMs: options.deadlineMs }), + ...(options.idleMs === undefined ? {} : { idleMs: options.idleMs }), }), }; } @@ -169,6 +176,26 @@ async function readFailure(response: Response) { return outcome(response.body?.getReader().read() ?? Promise.reject(new Error("no body"))); } +async function readAll(response: Response) { + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("no body"); + let bytes = 0; + try { + for (;;) { + const step = await reader.read(); + if (step.done) return { bytes, state: "ended" as const }; + bytes += step.value.length; + } + } catch (error: unknown) { + return { + bytes, + code: (error as { code?: unknown } | null)?.code, + message: error instanceof Error ? error.message : String(error), + state: "failed" as const, + }; + } +} + async function openedChannel(connection: FakeLink): Promise { await vi.waitFor(() => expect(connection.channels).toHaveLength(1)); return connection.channels[0] as FakeChannel; @@ -240,9 +267,11 @@ describe("GatewayRelay on a byte channel", () => { expect(last?.done).toBe(true); }); - it("cancels the caller's request body and clears its deadline timer when the channel ends", async () => { + it("cancels the caller's request body, clears its deadline timer at the head and its idle timer when the channel ends", async () => { const deadlineMs = 123_456; + const idleMs = 234_567; const handles: unknown[] = []; + const idleHandles: unknown[] = []; const realSetTimeout = globalThis.setTimeout; const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((( handler: () => void, @@ -250,6 +279,7 @@ describe("GatewayRelay on a byte channel", () => { ) => { const handle = realSetTimeout(handler, delay); if (delay === deadlineMs) handles.push(handle); + if (delay === idleMs) idleHandles.push(handle); return handle; }) as unknown as typeof setTimeout); const clearSpy = vi.spyOn(globalThis, "clearTimeout"); @@ -261,12 +291,16 @@ describe("GatewayRelay on a byte channel", () => { pull: () => new Promise(() => undefined), }); try { - const { connection, relay } = await setup({ deadlineMs }); + const { connection, relay } = await setup({ deadlineMs, idleMs }); const pending = relay.relay(WORKER_ID, call(body)); const request = await sentFrame(connection, "relay.request"); const channel = await openedChannel(connection); connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); const response = await pending; + const deadlineClearedAtHead = clearSpy.mock.calls.some(([handle]) => handle === handles[0]); + const idleClearedAtHead = clearSpy.mock.calls.some( + ([handle]) => handle === idleHandles.at(-1), + ); const reader = response.body?.getReader(); channel.end(); @@ -275,7 +309,10 @@ describe("GatewayRelay on a byte channel", () => { expect(last?.done).toBe(true); await vi.waitFor(() => expect(cancelled).toBe(true)); expect(handles).toHaveLength(1); - expect(clearSpy).toHaveBeenCalledWith(handles[0]); + expect(deadlineClearedAtHead).toBe(true); + expect(idleHandles.length).toBeGreaterThan(0); + expect(idleClearedAtHead).toBe(false); + expect(clearSpy).toHaveBeenCalledWith(idleHandles.at(-1)); } finally { setTimeoutSpy.mockRestore(); clearSpy.mockRestore(); @@ -302,8 +339,8 @@ describe("GatewayRelay on a byte channel", () => { }); }); - it("errors the caller's body with the deadline message when the worker answers {} and never ends the channel", async () => { - const { connection, relay } = await setup({ deadlineMs: 100 }); + it("errors the caller's body with the idle message when the worker answers {} and never ends the channel", async () => { + const { connection, relay } = await setup({ deadlineMs: 2_000, idleMs: 100 }); const pending = relay.relay(WORKER_ID, call()); const request = await sentFrame(connection, "relay.request"); await openedChannel(connection); @@ -314,7 +351,7 @@ describe("GatewayRelay on a byte channel", () => { const read = await readFailure(response); expect(read).toMatchObject({ - message: `worker ${WORKER_ID} did not finish the relayed request within 100 ms`, + message: `worker ${WORKER_ID} sent nothing on the relayed response for 100 ms`, state: "rejected", }); }); @@ -427,6 +464,75 @@ describe("GatewayRelay on a byte channel", () => { }); }); +describe("GatewayRelay bounding the head and the body", () => { + it.each([ + { bound: "the deadline", deadlineMs: 100, idleMs: 10_000 }, + { bound: "the idle bound", deadlineMs: 10_000, idleMs: 100 }, + ])( + "keeps relaying a body whose bytes keep coming past $bound, and ends it at the channel's end", + async ({ deadlineMs, idleMs }) => { + const { connection, relay } = await setup({ deadlineMs, idleMs }); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + connection.reply(request.id, {}); + + const reading = readAll(response); + for (let piece = 0; piece < 12 && !channel.aborted; piece += 1) { + await sleep(30); + if (!channel.aborted) channel.feed(new Uint8Array(KIB)); + } + if (!channel.aborted) channel.end(); + + expect(await reading).toEqual({ bytes: 12 * KIB, state: "ended" }); + }, + ); + + it("ends a body that sends nothing for the idle bound with WORKER_UNREACHABLE, sends relay.cancel and aborts its channel", async () => { + const { connection, relay } = await setup({ deadlineMs: 10_000, idleMs: 100 }); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + connection.reply(request.id, {}); + + channel.feed(new Uint8Array(KIB)); + const result = await Promise.race([ + readAll(response), + sleep(2_000).then(() => ({ state: "still open" as const })), + ]); + + expect(result).toEqual({ + bytes: KIB, + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} sent nothing on the relayed response for 100 ms`, + state: "failed", + }); + const cancel = await sentFrame(connection, "relay.cancel"); + expect(relayIdOf(cancel)).toBe(relayIdOf(request)); + expect(channel.aborted).toBe(true); + }); + + it("fails a relay whose head never arrives at the deadline, not at the shorter idle bound", async () => { + const { connection, relay } = await setup({ deadlineMs: 150, idleMs: 50 }); + const pending = outcome(relay.relay(WORKER_ID, call())); + await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + const failure = await pending; + + expect(failure).toEqual({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} did not finish the relayed request within 150 ms`, + state: "rejected", + }); + expect(channel.aborted).toBe(true); + }); +}); + describe("GatewayRelay mapping a channel failure", () => { const before: readonly { readonly reason: LinkChannelErrorReason; diff --git a/src/gateway/relay.ts b/src/gateway/relay.ts index daf80e2f..618f31b1 100644 --- a/src/gateway/relay.ts +++ b/src/gateway/relay.ts @@ -1,15 +1,19 @@ // why: every wait of a relayed exchange is bounded (architecture rule on cross-process waits): one deadline covers the -// channel wait, the exchange and the response body; the link bounds the channels open to a worker (ADR 0028). +// channel wait and the wait for the head, and an idle bound covers the response body, which may be a live stream that +// runs as long as someone reads it; the link bounds the channels open to a worker (ADR 0028). import { isSimlockError, relayHeaderPairs, type AnySimlockError } from "../contract/index.js"; import { DispatchError } from "../daemon/dispatch.js"; import { LinkChannelError, type LinkChannel } from "../link/index.js"; +import { type Clock, SystemClock, type TimerHandle } from "../ports/index.js"; import { isUplinkLost, liveClient, type WorkerDirectory } from "./fleet-ports.js"; import type { GatewayWorkerClient } from "./worker-client.js"; export interface GatewayRelayOptions { readonly directory: WorkerDirectory; readonly deadlineMs?: number; + readonly idleMs?: number; + readonly clock?: Clock; } export interface GatewayRelayCall { @@ -21,6 +25,10 @@ export interface GatewayRelayCall { // why: agent-device's proxy allows 300 s per exchange; the rest is a margin. const DEFAULT_DEADLINE_MS = 330_000; +// why: agent-device's proxy ends an exchange 300 s after it starts, and its RPC body may sit silent between progress +// lines for most of that; a bound at or under 300 s would cut exchanges the worker still finishes. A live stream's +// worker side already ends a capture silent for 30 s, so for a stream this only backs up a worker that hangs. +const DEFAULT_IDLE_MS = 330_000; type RelayMethod = Parameters[0]["method"]; const NO_BODY_STATUSES: ReadonlySet = new Set([204, 205, 304]); @@ -33,6 +41,8 @@ export class GatewayRelay { call, client: () => liveClient(this.options.directory.target(workerId)), deadlineMs: this.options.deadlineMs ?? DEFAULT_DEADLINE_MS, + clock: this.options.clock ?? new SystemClock(), + idleMs: this.options.idleMs ?? DEFAULT_IDLE_MS, reject, resolve, workerId, @@ -46,6 +56,8 @@ interface ExchangeInit { readonly call: GatewayRelayCall; readonly client: () => GatewayWorkerClient | undefined; readonly deadlineMs: number; + readonly idleMs: number; + readonly clock: Clock; readonly resolve: (response: Response) => void; readonly reject: (error: unknown) => void; readonly workerId: string; @@ -58,7 +70,8 @@ class Exchange { #channel: LinkChannel | undefined; #reader: ReadableStreamDefaultReader | undefined; #ahead: Promise> | undefined; - #timer: ReturnType | undefined; + #timer: TimerHandle | undefined; + #idleTimer: TimerHandle | undefined; #callerBody: ReadableStreamDefaultReader | undefined; #requestSent = false; #cancelSent = false; @@ -79,7 +92,7 @@ class Exchange { return; } signal.addEventListener("abort", this.#abortListener, { once: true }); - this.#timer = setTimeout(() => { + this.#timer = this.init.clock.setTimer(deadlineMs, () => { this.#fail( new DispatchError( "WORKER_UNREACHABLE", @@ -87,7 +100,7 @@ class Exchange { { workerId }, ), ); - }, deadlineMs); + }); void this.#run(); } @@ -172,7 +185,12 @@ class Exchange { #look(): Promise> { if (this.#ahead === undefined) { const read = (this.#reader as ReadableStreamDefaultReader).read(); - read.catch((error: unknown) => this.#onChannelError(error)); + read.then( + (step) => { + if (!step.done) this.#armIdle(); + }, + (error: unknown) => this.#onChannelError(error), + ); this.#ahead = read; } return this.#ahead; @@ -193,6 +211,9 @@ class Exchange { return; } this.#headed = true; + // why: once the head is in, the body is bounded by silence, not by the deadline, so a live stream is not cut. + if (this.#timer !== undefined) this.init.clock.cancel(this.#timer); + this.#armIdle(); const method = this.init.call.request.method; if (method === "HEAD" || NO_BODY_STATUSES.has(status)) { this.init.resolve(new Response(null, { headers: responseHeaders, status })); @@ -229,7 +250,7 @@ class Exchange { controller.enqueue(step.value); } - // why: a response with no body still ends with the channel's end, which frees the channel and the deadline. + // why: a response with no body still ends with the channel's end, which frees the channel and its timers. async #drain(): Promise { try { for (;;) { @@ -246,6 +267,21 @@ class Exchange { } } + #armIdle(): void { + if (!this.#headed || this.#done) return; + if (this.#idleTimer !== undefined) this.init.clock.cancel(this.#idleTimer); + const { clock, idleMs, workerId } = this.init; + this.#idleTimer = clock.setTimer(idleMs, () => { + this.#fail( + new DispatchError( + "WORKER_UNREACHABLE", + `worker ${workerId} sent nothing on the relayed response for ${String(idleMs)} ms`, + { workerId }, + ), + ); + }); + } + #invalid(): void { this.#fail( new DispatchError("INTERNAL", `worker ${this.init.workerId} sent an invalid relay response`), @@ -271,7 +307,8 @@ class Exchange { } switch (error.reason) { case "aborted": - // why: the worker's `relay.request` answer says why it aborted; the deadline ends the wait when none comes. + // why: the worker's `relay.request` answer says why it aborted; when none comes, the deadline ends the wait, + // or after the head the idle bound does. this.#peerAborted = true; if (this.#answered) this.#fail(this.#endedEarly()); return; @@ -368,7 +405,8 @@ class Exchange { #finish(): void { if (this.#done) return; this.#done = true; - if (this.#timer !== undefined) clearTimeout(this.#timer); + if (this.#timer !== undefined) this.init.clock.cancel(this.#timer); + if (this.#idleTimer !== undefined) this.init.clock.cancel(this.#idleTimer); this.init.call.request.signal.removeEventListener("abort", this.#abortListener); this.#callerBody?.cancel().catch(() => undefined); } diff --git a/src/http/AGENTS.md b/src/http/AGENTS.md index 2cb4f0d0..b087752e 100644 --- a/src/http/AGENTS.md +++ b/src/http/AGENTS.md @@ -40,7 +40,7 @@ Surface: [HTTP-API.md](../../docs/HTTP-API.md), [CONSOLE.md](../../docs/CONSOLE. adds `Retry-After: 5`. An unknown error answers `INTERNAL`, "Internal error", and is logged. `HttpMount` adds a prefix (`/v1/`, `/v1//*`) behind the same auth. Today: the -agent-device front. +agent-device front and the stream (`/v1/stream`). ## Lease requests diff --git a/src/http/errors.ts b/src/http/errors.ts index 2a23175e..ab0353d2 100644 --- a/src/http/errors.ts +++ b/src/http/errors.ts @@ -9,14 +9,31 @@ import { type SimlockErrorCode, } from "../contract/index.js"; -/** Every status this frontend ever answers with; keeps `mapError` exhaustive by construction. - * 501 is `UNSUPPORTED_IN_GATEWAY_MODE`'s (ADR 0005 §34) and `UNSUPPORTED_IN_WORKER_MODE`'s (ADR - * 0012 §2): the request was well-formed and the daemon is healthy, this daemon's mode simply - * does not implement that operation. 504 is - * `EXEC_TIMEOUT`'s (ADR 0005 §19e): a `device.exec` command the daemon killed for outrunning - * `exec.timeoutMs` is the one case where what this frontend was waiting on never finished, - * which is exactly what a gateway timeout says. */ -export type HttpStatus = 400 | 401 | 403 | 404 | 409 | 422 | 500 | 501 | 503 | 504; +// why: every status this frontend ever answers with, which keeps `mapError` exhaustive by +// construction. 501 is `UNSUPPORTED_IN_GATEWAY_MODE`'s (ADR 0005 §34) and +// `UNSUPPORTED_IN_WORKER_MODE`'s (ADR 0012 §2): the request was well-formed and the daemon is +// healthy, this daemon's mode simply does not implement that operation. 504 is `EXEC_TIMEOUT`'s +// (ADR 0005 §19e): a `device.exec` command the daemon killed for outrunning `exec.timeoutMs` is the +// one case where what this frontend was waiting on never finished, which is exactly what a gateway +// timeout says. 405, 415 and 502 are answered only by the screen stream mount, which returns its own +// response rather than raising an error for `errorResponse` -- a stream cannot refuse one once it has +// begun, so it refuses with a status while the answer is still a status: 405 for a stream asked to be +// written, 415 for a device that can never produce a picture, 502 for a worker that answered "200" to +// its gateway with no picture at all. +export type HttpStatus = + | 400 + | 401 + | 403 + | 404 + | 405 + | 409 + | 415 + | 422 + | 500 + | 501 + | 502 + | 503 + | 504; /** * Uniform `{status, code}` pair a route or middleware raises directly (auth, ownership, diff --git a/src/leasing/AGENTS.md b/src/leasing/AGENTS.md index a8a44f4e..be217fdb 100644 --- a/src/leasing/AGENTS.md +++ b/src/leasing/AGENTS.md @@ -55,7 +55,7 @@ and registry writers. Read-decide-commit runs under it; driver I/O never does. emits `lease.released` or `lease.expired`. 8. **Lease-end participants** — called after that commit, outside the lock, with an `AbortSignal` bounded by `lease.endWaitMs`. Today: agent-device's - `AgentDeviceLeaseEnd`. Reclaim waits for them. + `AgentDeviceLeaseEnd` and the stream's `createStreamLeaseEnd`. Reclaim waits for them. 9. **Reclaim** (background, `reclaim` claim) — core's `ReclaimCoordinator` wipes the device and commits the state the driver returns (iOS `shutdown`; Android `ready`, or `shutdown` after a full clean, a changed config or a diff --git a/src/link/AGENTS.md b/src/link/AGENTS.md index 4064404d..6e325f7d 100644 --- a/src/link/AGENTS.md +++ b/src/link/AGENTS.md @@ -83,4 +83,5 @@ None of these is configurable. ## Known gaps - **The link bounds only its own waits.** A wait for bytes the peer has not - sent yet is the user's to bound; the gateway relay does it with its 330 s deadline. + sent yet is the user's to bound; the gateway relay does it with its 330 s deadline for the + head and its 330 s idle bound for the body. diff --git a/src/module-registry/index.ts b/src/module-registry/index.ts index fd4e9868..25e78241 100644 --- a/src/module-registry/index.ts +++ b/src/module-registry/index.ts @@ -1,6 +1,8 @@ import { agentDeviceModule } from "../agent-device/index.js"; import type { OptionalModuleDefinition } from "../modules/index.js"; +import { streamModule } from "../stream/index.js"; export const optionalModules: readonly OptionalModuleDefinition[] = [ agentDeviceModule as OptionalModuleDefinition, + streamModule as OptionalModuleDefinition, ]; diff --git a/src/modules/testing.ts b/src/modules/testing.ts index a86cf8ea..7b4d0a8a 100644 --- a/src/modules/testing.ts +++ b/src/modules/testing.ts @@ -156,6 +156,11 @@ export async function fakeWorkerContext(name: string, config: C): Promise Promise.reject(new Error("no screen in a fake context")), + screenAvailability: () => Promise.resolve([]), + }, tcpProbe: new FakeTcpProbe(), }, processRunner, diff --git a/src/modules/types.ts b/src/modules/types.ts index 70cbbd76..1664f27e 100644 --- a/src/modules/types.ts +++ b/src/modules/types.ts @@ -8,7 +8,17 @@ import type { grantedDeviceSchema, workerViewSchema, } from "../contract/index.js"; -import type { Config, Platform } from "../core/index.js"; +import type { + Config, + DeviceRecord, + LeaseRecord, + PhysicalDriverDevice, + Platform, + ScreenAvailability, + ScreenFeed, + ScreenSettings, + VirtualDriverDevice, +} from "../core/index.js"; import type { Clock, Filesystem, @@ -96,6 +106,26 @@ export interface WorkerModule { readonly hooks?: WorkerHooks; } +// why: a read of the worker's registry as it stands, so a module that serves a lease checks the lease +// against the worker's own record rather than against what a caller claims. +export interface ModuleRegistryView { + readonly snapshot: { + readonly devices: readonly DeviceRecord[]; + readonly leases: readonly LeaseRecord[]; + }; +} + +// why: the drivers' screen capture, routed by platform, without handing a module the drivers. +export interface ModuleScreens { + openScreen( + platform: Platform, + device: VirtualDriverDevice | PhysicalDriverDevice, + settings: ScreenSettings, + signal: AbortSignal, + ): Promise; + screenAvailability(): Promise; +} + export interface WorkerContext { readonly name: string; readonly config: C; @@ -112,6 +142,8 @@ export interface WorkerContext { readonly nodeVersion: string; readonly platforms: readonly Platform[]; readonly leaseEnvironment: Readonly>; + readonly registry: ModuleRegistryView; + readonly screens: ModuleScreens; cancelPending(requesterId: string): Promise; readonly events: { subscribe(event: Event, handler: EventHandler): () => void; diff --git a/src/ports/AGENTS.md b/src/ports/AGENTS.md index 1f66d2d0..009769b1 100644 --- a/src/ports/AGENTS.md +++ b/src/ports/AGENTS.md @@ -23,6 +23,8 @@ Uses: nothing from other components. Imported by: every component that touches t | `HostInfo` | `host-info.ts` | `NodeHostInfo` | `FakeHostInfo` | OS, version (`sw_vers` on macOS), arch | | `HostLoadProbe` | `host-load.ts` | `DarwinHostLoadProbe`, `LinuxHostLoadProbe` | `FakeHostLoadProbe` | CPU, memory, disk counters | | `TcpProbe` | `tcp-probe.ts` | `NodeTcpProbe` | `FakeTcpProbe` | Is a loopback port listening; send bytes | +| `TcpStreamConnector` | `tcp-stream.ts` | `NodeTcpStreamConnector` | `FakeTcpStreamConnector` | A long-lived loopback connection, bytes both ways | +| `BytePipeLauncher` | `byte-pipe.ts` | `NodeBytePipeLauncher` | `FakeBytePipeLauncher` | A child with binary stdout and stdin as its liveness | | `IdGenerator` | `id-generator.ts` | `CryptoIdGenerator` | none; tests inline one | Random ids | | `TokenSecrets` | `token-secrets.ts` | `CryptoTokenSecrets` | none; tests inline one | New secret, its hash | | Paths | `paths.ts` | `resolveSimlockHome`, `resolveDaemonSocketPath` | — | `SIMLOCK_HOME` (default `~/.simlock`), socket path length | @@ -33,7 +35,7 @@ Uses: nothing from other components. Imported by: every component that touches t ## Rules - Application code never imports `fs` or `child_process`, or calls `Date.now()` or - `setTimeout`: it takes a port. Lint fails core, leasing, agent-device and host-load on an + `setTimeout`: it takes a port. Lint fails core, leasing, agent-device, host-load and stream on an `fs` or `child_process` import (`no-fs-or-process` in `.dependency-cruiser.mjs`). - The real adapters are built once, in the composition root (`src/daemon/main.ts`, or the CLI and MCP environment builders), and injected. diff --git a/src/ports/byte-pipe.test.ts b/src/ports/byte-pipe.test.ts new file mode 100644 index 00000000..382109fb --- /dev/null +++ b/src/ports/byte-pipe.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { NodeBytePipeLauncher } from "./byte-pipe.js"; + +async function readAll(stream: ReadableStream): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of stream) chunks.push(chunk); + return Buffer.concat(chunks); +} + +describe("NodeBytePipeLauncher", () => { + it("hands stdout out as the exact bytes the child wrote, including ones that are not text", async () => { + const child = new NodeBytePipeLauncher().start( + process.execPath, + ["-e", "process.stdout.write(Buffer.from([0, 255, 10, 128, 1]))"], + { env: process.env }, + ); + expect([...(await readAll(child.stdout))]).toEqual([0, 255, 10, 128, 1]); + expect(await child.exited).toEqual({ code: 0, signal: null }); + }); + + it("closing stdin ends a child that waits on it", async () => { + const child = new NodeBytePipeLauncher().start( + process.execPath, + ["-e", "process.stdin.resume(); process.stdin.on('end', () => process.exit(7))"], + { env: process.env }, + ); + child.closeStdin(); + expect(await child.exited).toEqual({ code: 7, signal: null }); + }); + + it("reports each stderr line once it is complete", async () => { + const lines: string[] = []; + const child = new NodeBytePipeLauncher().start( + process.execPath, + [ + "-e", + "process.stderr.write('one\\ntw'); setTimeout(() => process.stderr.write('o\\n'), 20)", + ], + { env: process.env, onStderrLine: (line) => lines.push(line) }, + ); + await child.exited; + expect(lines).toEqual(["one", "two"]); + }); + + it("resolves a program that cannot be started with the spawn error instead of rejecting", async () => { + const child = new NodeBytePipeLauncher().start("/nonexistent/simlock-capture", [], { + env: process.env, + }); + const exit = await child.exited; + expect(exit.code).toBeNull(); + expect(exit.error).toMatch(/ENOENT/); + }); + + it("kill ends a child that ignores stdin", async () => { + const child = new NodeBytePipeLauncher().start( + process.execPath, + ["-e", "setInterval(() => undefined, 1000)"], + { env: process.env }, + ); + child.kill("SIGKILL"); + expect(await child.exited).toEqual({ code: null, signal: "SIGKILL" }); + }); +}); diff --git a/src/ports/byte-pipe.ts b/src/ports/byte-pipe.ts new file mode 100644 index 00000000..4b3c7e74 --- /dev/null +++ b/src/ports/byte-pipe.ts @@ -0,0 +1,141 @@ +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; + +// why: `ProcessRunner` hands output out as text, which a video stream cannot survive, and closes or +// never offers the stdin a capture program treats as its liveness handle. +export interface BytePipeLauncher { + start(command: string, args: readonly string[], options: BytePipeOptions): BytePipeProcess; +} + +export interface BytePipeOptions { + readonly env: NodeJS.ProcessEnv; + readonly onStderrLine?: (line: string) => void; +} + +export interface BytePipeProcess { + readonly stdout: ReadableStream; + // why: never rejects -- a program the OS refused to start resolves with `error` set, so a caller + // has one place to learn how it ended. + readonly exited: Promise; + closeStdin(): void; + kill(signal: "SIGTERM" | "SIGKILL"): void; +} + +export interface BytePipeExit { + readonly code: number | null; + readonly signal: string | null; + readonly error?: string; +} + +export class NodeBytePipeLauncher implements BytePipeLauncher { + start(command: string, args: readonly string[], options: BytePipeOptions): BytePipeProcess { + const child = spawn(command, [...args], { env: options.env, stdio: ["pipe", "pipe", "pipe"] }); + // why: a child that exits while stdin is being closed raises EPIPE on the pipe, which is the + // stop having worked, not a failure anyone can act on. + child.stdin.on("error", () => undefined); + + let partial = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + const lines = (partial + chunk).split("\n"); + partial = lines.pop() ?? ""; + for (const line of lines) if (line.trim() !== "") options.onStderrLine?.(line); + }); + + const exited = new Promise((resolve) => { + child.once("error", (error) => resolve({ code: null, error: error.message, signal: null })); + child.once("close", (code, signal) => resolve({ code, signal })); + }); + + return { + closeStdin: () => { + child.stdin.end(); + }, + exited, + kill: (signal) => { + if (child.exitCode === null && child.signalCode === null) child.kill(signal); + }, + stdout: Readable.toWeb(child.stdout) as ReadableStream, + }; + } +} + +export interface FakeBytePipeStart { + readonly command: string; + readonly args: readonly string[]; + readonly env: NodeJS.ProcessEnv; +} + +// fallow-ignore-next-line unused-export -- the public shape of FakeBytePipeLauncher.last and .processes. +export class FakeBytePipeProcess implements BytePipeProcess { + readonly stdout: ReadableStream; + readonly exited: Promise; + stdinClosed = false; + stdinCloses = 0; + readonly kills: string[] = []; + exitOnStdinClose = true; + readonly #onStderrLine: ((line: string) => void) | undefined; + #controller!: ReadableStreamDefaultController; + #resolveExit!: (exit: BytePipeExit) => void; + #ended = false; + + constructor(onStderrLine?: (line: string) => void) { + this.#onStderrLine = onStderrLine; + this.stdout = new ReadableStream({ + start: (controller) => { + this.#controller = controller; + }, + }); + this.exited = new Promise((resolve) => { + this.#resolveExit = resolve; + }); + } + + write(bytes: Uint8Array): void { + if (!this.#ended) this.#controller.enqueue(bytes); + } + + stderr(line: string): void { + this.#onStderrLine?.(line); + } + + exit(exit: BytePipeExit = { code: 0, signal: null }): void { + if (this.#ended) return; + this.#ended = true; + try { + this.#controller.close(); + } catch { + // why: a reader that already cancelled the stream has closed it; the exit still settles. + } + this.#resolveExit(exit); + } + + closeStdin(): void { + this.stdinClosed = true; + this.stdinCloses += 1; + if (this.exitOnStdinClose) this.exit(); + } + + kill(signal: "SIGTERM" | "SIGKILL"): void { + this.kills.push(signal); + this.exit({ code: null, signal }); + } +} + +export class FakeBytePipeLauncher implements BytePipeLauncher { + readonly starts: FakeBytePipeStart[] = []; + readonly processes: FakeBytePipeProcess[] = []; + + start(command: string, args: readonly string[], options: BytePipeOptions): FakeBytePipeProcess { + this.starts.push({ args: [...args], command, env: options.env }); + const process = new FakeBytePipeProcess(options.onStderrLine); + this.processes.push(process); + return process; + } + + get last(): FakeBytePipeProcess { + const process = this.processes.at(-1); + if (process === undefined) throw new Error("No byte-pipe process was started"); + return process; + } +} diff --git a/src/ports/index.ts b/src/ports/index.ts index 9b5d926a..1c3b9a0d 100644 --- a/src/ports/index.ts +++ b/src/ports/index.ts @@ -66,6 +66,18 @@ export type { ProcessSupervisor } from "./process-supervisor.js"; // fallow-ignore-next-line unused-type -- public shape of FakeProcessSupervisor.signals. export type { SignalRecord } from "./process-supervisor.js"; export { FakeTcpProbe, NodeTcpProbe } from "./tcp-probe.js"; +export { + FakeTcpStreamConnector, + NodeTcpStreamConnector, + type TcpStream, + type TcpStreamConnector, +} from "./tcp-stream.js"; +export { + type BytePipeLauncher, + type BytePipeProcess, + FakeBytePipeLauncher, + NodeBytePipeLauncher, +} from "./byte-pipe.js"; // fallow-ignore-next-line unused-type -- public shape of FakeTcpProbe.sends. export type { FakeSend } from "./tcp-probe.js"; export type { TcpProbe } from "./tcp-probe.js"; diff --git a/src/ports/tcp-stream.test.ts b/src/ports/tcp-stream.test.ts new file mode 100644 index 00000000..ec00af0a --- /dev/null +++ b/src/ports/tcp-stream.test.ts @@ -0,0 +1,54 @@ +import { createServer, type Server, type Socket } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NodeTcpStreamConnector } from "./tcp-stream.js"; + +let server: Server | undefined; + +afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = undefined; +}); + +async function listen(onConnection: (socket: Socket) => void): Promise { + server = createServer(onConnection); + await new Promise((resolve) => server?.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (address === null || typeof address === "string") throw new Error("no port"); + return address.port; +} + +describe("NodeTcpStreamConnector", () => { + it("carries bytes both ways until the peer hangs up", async () => { + const port = await listen((socket) => { + socket.once("data", (request: Buffer) => { + socket.end(Buffer.concat([Buffer.from("echo:"), request])); + }); + }); + const stream = await new NodeTcpStreamConnector().connect(port, 1000); + await stream.write("ping"); + const chunks: Uint8Array[] = []; + for await (const chunk of stream.readable) chunks.push(chunk); + expect(Buffer.concat(chunks).toString()).toBe("echo:ping"); + }); + + it("rejects a port nothing listens on", async () => { + const port = await listen(() => undefined); + await new Promise((resolve) => server?.close(() => resolve())); + server = undefined; + await expect(new NodeTcpStreamConnector().connect(port, 1000)).rejects.toThrow(/ECONNREFUSED/); + }); + + it("close ends the connection the peer sees", async () => { + const peerClosed = new Promise((resolve) => { + void listen((socket) => { + socket.on("close", () => resolve()); + }).then(async (port) => { + const stream = await new NodeTcpStreamConnector().connect(port, 1000); + stream.close(); + }); + }); + await expect(peerClosed).resolves.toBeUndefined(); + }); +}); diff --git a/src/ports/tcp-stream.ts b/src/ports/tcp-stream.ts new file mode 100644 index 00000000..5211e56e --- /dev/null +++ b/src/ports/tcp-stream.ts @@ -0,0 +1,110 @@ +import { createConnection, type Socket } from "node:net"; +import { Readable } from "node:stream"; + +// why: `TcpProbe` answers one question and hangs up; a device's video socket reached through adb +// keeps talking for as long as someone watches. +export interface TcpStreamConnector { + connect(port: number, timeoutMs: number): Promise; +} + +export interface TcpStream { + readonly readable: ReadableStream; + write(bytes: Uint8Array | string): Promise; + close(): void; +} + +export class NodeTcpStreamConnector implements TcpStreamConnector { + connect(port: number, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + let socket: Socket; + try { + socket = createConnection({ host: "127.0.0.1", port }); + } catch (error: unknown) { + reject(error instanceof Error ? error : new Error(String(error))); + return; + } + const fail = (error: Error) => { + socket.destroy(); + reject(error); + }; + socket.setTimeout(timeoutMs, () => + fail(new Error(`No connection to 127.0.0.1:${String(port)} within ${String(timeoutMs)}ms`)), + ); + socket.once("error", fail); + socket.once("connect", () => { + socket.setTimeout(0); + socket.off("error", fail); + // why: after connect an error ends the readable through `Readable.toWeb`; a listener must + // still exist or Node raises it as an uncaught exception and takes the daemon down. + socket.on("error", () => undefined); + resolve({ + close: () => { + socket.destroy(); + }, + readable: Readable.toWeb(socket) as ReadableStream, + write: (bytes) => + new Promise((done, failed) => { + socket.write(bytes, (error) => (error ? failed(error) : done())); + }), + }); + }); + }); + } +} + +// fallow-ignore-next-line unused-export -- the public shape of FakeTcpStreamConnector.connections. +export class FakeTcpPeer { + readonly written: Uint8Array[] = []; + closed = false; + #controller!: ReadableStreamDefaultController; + readonly stream: TcpStream; + + constructor() { + const readable = new ReadableStream({ + cancel: () => { + this.closed = true; + }, + start: (controller) => { + this.#controller = controller; + }, + }); + this.stream = { + close: () => { + if (this.closed) return; + this.closed = true; + try { + this.#controller.close(); + } catch { + // why: a cancelled readable is already closed; closing the connection is still done. + } + }, + readable, + write: async (bytes) => { + this.written.push(typeof bytes === "string" ? new TextEncoder().encode(bytes) : bytes); + }, + }; + } + + writtenText(): string { + return this.written.map((chunk) => new TextDecoder().decode(chunk)).join(""); + } + + send(bytes: Uint8Array | string): void { + if (this.closed) return; + this.#controller.enqueue(typeof bytes === "string" ? new TextEncoder().encode(bytes) : bytes); + } + + end(): void { + this.stream.close(); + } +} + +export class FakeTcpStreamConnector implements TcpStreamConnector { + readonly connections: { readonly port: number; readonly peer: FakeTcpPeer }[] = []; + + async connect(port: number): Promise { + const peer = new FakeTcpPeer(); + this.connections.push({ peer, port }); + return peer.stream; + } +} diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index 13befc4d..72ff7219 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -1,6 +1,7 @@ import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; @@ -9,8 +10,56 @@ const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); const NUL = 0x00; -/** why: built binaries committed for tests are never searched as text. */ -const isCommittedBinary = (name: string): boolean => name.startsWith("e2e/fixtures/"); +const lsFiles = (args: string[]): string[] => + execFileSync("git", ["ls-files", "-z", ...args], { + cwd: REPO_ROOT, + encoding: "buffer", + maxBuffer: 32 * 1024 * 1024, + }) + .toString("utf8") + .split("\0") + .filter((name) => name.length > 0); + +/** + * why: a vendored artifact lands untracked -- the directory is complete, at the right checksum, + * long before anybody stages it, and a check that read only the index would wave a half-vendored + * directory through the very run meant to catch it. So `vendor/` is enumerated from the index + * *and* the working tree. Ignored files stay out: `vendor/`'s own build scratch (`.build/`) is not + * part of what anyone clones. + */ +const vendoredFiles = lsFiles(["--cached", "--others", "--exclude-standard", "vendor/"]); + +/** + * why: pinning an artifact's digest is the statement that its bytes are someone else's and are not + * to be read as text, so the same list says which vendored files the text scan must skip -- and the + * digest test below proves the list means what it says. It also keeps the skip from growing on its + * own: a new vendored binary joins it by being pinned, not by being named here. + */ +const pinnedVendoredArtifacts = new Set( + vendoredFiles + .filter((name) => name.endsWith("/SHA256SUMS.txt")) + .flatMap((sums) => { + const dir = sums.slice(0, -"SHA256SUMS.txt".length); + return sha256Sums(sums).map(([, file]) => `${dir}${file}`); + }), +); + +/** why: the digest and file name a `SHA256SUMS.txt` pins, read in the format `shasum -c` reads. */ +function sha256Sums(name: string): [string, string][] { + return readFileSync(join(REPO_ROOT, name), "utf8") + .split("\n") + .map((line) => line.trim().split(/\s+/)) + .filter((pair): pair is [string, string] => pair.length === 2 && pair[1]!.length > 0); +} + +/** + * why: a committed binary is never searched as text -- built fixtures the e2e lane installs, and + * the vendored third-party artifacts a `SHA256SUMS.txt` pins bytes for (a packed server APK). Both kinds hold raw NUL bytes by nature, which is what makes them unsearchable: that + * is a claim about their contents, so for `vendor/` the test that compares this list against the + * files in `vendor/` that actually carry a NUL is what makes the skip mean something. + */ +const isCommittedBinary = (name: string): boolean => + name.startsWith("e2e/fixtures/") || pinnedVendoredArtifacts.has(name); /** * why: a raw NUL byte in a text file makes `grep` and `ripgrep` classify the whole file as @@ -20,19 +69,13 @@ const isCommittedBinary = (name: string): boolean => name.startsWith("e2e/fixtur * * Enumerated from `git ls-files` rather than a directory walk so the check sees every * tracked file in the repository (testing rule 4), not just the directories this test - * happened to name. The exception is `e2e/fixtures/`: built binaries committed for tests, - * never searched as text, so they are skipped. Untracked and ignored files are out of scope: they are not part of - * what anyone clones or searches. + * happened to name. The exceptions are the committed binaries `isCommittedBinary` names: + * built binaries committed for tests, and the vendored third-party artifacts, none of + * which is ever searched as text. Untracked and ignored files are out of scope: + * they are not part of what anyone clones or searches. */ describe("tracked files", () => { - const trackedFiles = execFileSync("git", ["ls-files", "-z"], { - cwd: REPO_ROOT, - encoding: "buffer", - maxBuffer: 32 * 1024 * 1024, - }) - .toString("utf8") - .split("\0") - .filter((name) => name.length > 0) + const trackedFiles = lsFiles([]) .filter((name) => !isCommittedBinary(name)) // A submodule or a path deleted from the working tree is still listed by // `git ls-files`; only regular files can be read. @@ -68,3 +111,166 @@ describe("tracked files", () => { expect(offenders).toEqual([]); }); }); + +/** + * why: `vendor/` is the claim that what is in there is someone else's code, so the check walks + * every first-level directory (testing rule 4) rather than the two it happens to know about. Built + * from `vendoredFiles`, which sees the working tree as well as the index. + */ +const vendoredDirs = new Map(); +for (const name of vendoredFiles) { + const [, dir = "", rest = ""] = name.split("/"); + // why: a file at `vendor/`'s root belongs to no directory, and has its own section below. + if (rest.length === 0) continue; + const paths = vendoredDirs.get(dir); + if (paths === undefined) vendoredDirs.set(dir, [name]); + else paths.push(name); +} + +/** why: a vendored directory's license texts are exactly what its `LICENSES/` holds (rule 3). */ +const licenseTextsOf = (dir: string): string[] => + (vendoredDirs.get(dir) ?? []).filter((name) => name.startsWith(`vendor/${dir}/LICENSES/`)); + +/** + * why: a license text named for a license is all a reader has to trust that it *is* + * that license, so the name and the contents have to agree; the expected words are + * read out of the file name (`serve-sim.APACHE-2.0.txt` asks the text to say + * `APACHE`) so no list here can go stale. + */ +const licensesClaimedOf = (name: string): string[] => { + const base = name.slice(name.lastIndexOf("/") + 1).replace(/\.txt$/i, ""); + const separator = base.indexOf("."); + if (separator <= 0) return []; + return base + .slice(separator + 1) + .split(/[.\-_]/) + .filter((token) => /^[a-z]{2,}$/i.test(token)); +}; + +/** + * why: some licenses' verbatim text never names the license -- BSD-3-Clause is a copyright line + * and three conditions -- so a name-word check would reject the verbatim text rule 3 demands. For + * those, the name is checked against the clause that makes the text that license instead: the + * third condition is what separates BSD-3-Clause from BSD-2-Clause. + */ +const clauseNamingLicense: Readonly> = { + "BSD-3-Clause": /Neither the name of[\s\S]+?may\s+be\s+used\s+to\s+endorse\s+or\s+promote/, +}; + +/** why: what in `text` contradicts the license `name` claims to hold, one line per mismatch. */ +const licenseMismatchesOf = (name: string, text: string): string[] => { + const base = name.slice(name.lastIndexOf("/") + 1).replace(/\.txt$/i, ""); + const clause = clauseNamingLicense[base.slice(base.indexOf(".") + 1)]; + if (clause !== undefined) { + return clause.test(text) + ? [] + : [`${name}: its text lacks the clause that makes it that license`]; + } + const claimed = licensesClaimedOf(name); + if (claimed.length === 0) return [`${name}: its file name names no license`]; + return claimed + .filter((word) => !new RegExp(`\\b${word}\\b`, "i").test(text)) + .map((word) => `${name}: its text never says ${word}`); +}; + +/** + * why: the rule has to be enforced everywhere it applies or it only guards the directories someone + * remembered (testing rule 4). Vendor and third-party code rule 2 + * (`docs/internal/agent-rules/vendor-and-third-party-code.md`): a first-level directory under + * `vendor/` holds the artifact(s), the license text(s) they require, and a `NOTICE.md`. + */ +describe("a first-level directory under vendor/", () => { + it("was enumerated, and holds the vendored code this repo carries", () => { + // why: without this, an empty enumeration would leave every claim below with nothing to check + // and the suite would pass while enforcing the rule on no directory at all. + expect([...vendoredDirs.keys()].toSorted()).toEqual( + expect.arrayContaining(["scrcpy", "screen-capture"]), + ); + expect(vendoredFiles.length).toBeGreaterThan(10); + }); + + it("holds a NOTICE.md", () => { + const withoutNotice = [...vendoredDirs.keys()].filter( + (dir) => !vendoredDirs.get(dir)?.includes(`vendor/${dir}/NOTICE.md`), + ); + expect(withoutNotice).toEqual([]); + }); + + it("holds a license text under LICENSES/ whose text matches its name", () => { + const missing = [...vendoredDirs.keys()].filter((dir) => licenseTextsOf(dir).length === 0); + const mismatched = [...vendoredDirs.keys()].flatMap((dir) => + licenseTextsOf(dir).flatMap((name) => + licenseMismatchesOf(name, readFileSync(join(REPO_ROOT, name), "utf8")), + ), + ); + expect({ missing, mismatched }).toEqual({ missing: [], mismatched: [] }); + }); + + it("holds no file named LICENSE, which would claim to license the whole directory", () => { + // why: rule 3 -- our own code often sits beside theirs, so a license text names what it covers + // instead of taking the name that states it covers everything nearby. + const isBareLicense = (name: string): boolean => /^LICENSE(\.|$)/.test(basename(name)); + const bare = vendoredFiles.filter((name) => name.split("/").length > 2 && isBareLicense(name)); + expect(bare).toEqual([]); + }); +}); + +/** + * why: the digest in a `SHA256SUMS.txt` is the claim that a vendored binary is byte-identical to + * what upstream published, and a claim nobody recomputes is a claim that rots. Computed here rather + * than read back out of `NOTICE.md`, so both the notice and the text-scan exclusion this list feeds + * rest on bytes on disk (vendor and third-party code rule 7). + */ +describe("a vendored artifact pinned by a SHA256SUMS.txt", () => { + const pinned = [...pinnedVendoredArtifacts].toSorted(); + + it("was enumerated, and holds the artifacts that are here", () => { + // why: without this, a `SHA256SUMS.txt` that stopped being listed would empty the two checks + // below and the suite would pass having verified no digest at all. + expect(pinned).toEqual(expect.arrayContaining(["vendor/scrcpy/scrcpy-server-v5.0.1"])); + }); + + it("matches the digest its SHA256SUMS.txt gives", () => { + const wrong = pinned.flatMap((name) => { + const separator = name.lastIndexOf("/"); + const sums = sha256Sums(`${name.slice(0, separator + 1)}SHA256SUMS.txt`); + const expected = sums.find(([, file]) => file === name.slice(separator + 1))?.[0]; + if (expected === undefined) return [`${name}: its SHA256SUMS.txt does not list it`]; + const actual = createHash("sha256") + .update(readFileSync(join(REPO_ROOT, name))) + .digest("hex"); + return actual === expected ? [] : [`${name}: ${actual} != ${expected}`]; + }); + expect(wrong).toEqual([]); + }); + + it("is exactly the vendored file that a raw NUL byte would make unsearchable", () => { + // why: this proves the vendored clause of `isCommittedBinary` skips what it says it skips -- an + // artifact pinned but not a binary is skipped for no reason, which costs every reader a + // searchable file, and a binary left off the list turns the NUL check red instead of being + // excluded from it. + const withNul = vendoredFiles + .filter((name) => { + try { + return statSync(join(REPO_ROOT, name)).isFile(); + } catch { + return false; + } + }) + .filter((name) => readFileSync(join(REPO_ROOT, name)).includes(NUL)) + .toSorted(); + expect(withNul).toEqual(["vendor/scrcpy/scrcpy-server-v5.0.1"]); + expect(pinned).toEqual(withNul); + }); +}); + +/** + * why: a loose file at `vendor/`'s root would sit outside the per-directory rule entirely, so the + * index is the only one allowed there. + */ +describe("a file at vendor/'s root", () => { + it("is only the index, never third-party code outside a directory", () => { + const rootFiles = vendoredFiles.filter((name) => !name.slice("vendor/".length).includes("/")); + expect(rootFiles).toEqual(["vendor/README.md"]); + }); +}); diff --git a/src/stream/AGENTS.md b/src/stream/AGENTS.md new file mode 100644 index 00000000..1179688d --- /dev/null +++ b/src/stream/AGENTS.md @@ -0,0 +1,81 @@ +# Stream + +Does: serves a leased device's live screen at `GET /v1/stream/leases/{leaseId}`, on a worker and through a gateway. +Entry: `index.ts` → `streamModule`, an optional module (ADR 0030): `worker.ts` `createWorker`, +`gateway.ts` `createGateway`, loaded by the module host only when turned on. +Uses: core (`ScreenFeed`, frame tags, registry records), modules (the context and hook types), ports. +Never imports: http, gateway, drivers, daemon, leasing, `fs`, `child_process`. Imported by: `module-registry` only. +Off unless `capabilities.stream.enabled` (false). Off: nothing loaded, and the `stream` capability reads `off`. + +## Files + +| File | Role | +| ------------------------- | ------------------------------------------------------------------------------- | +| `definition.ts` | `streamModule`: name `stream`, config section, the two loaders | +| `worker.ts`, `gateway.ts` | The module's two sides, built from the host's context | +| `create.ts` | The two factories; the `stream` capability read from each driver's availability | +| `worker-link.ts` | Worker: lease check, open through the driver, register, re-check after frame 1 | +| `gateway-link.ts` | Gateway: fleet ownership, relay to the holding worker, refusal and header check | +| `feed.ts` | `readScreenFeed`: meta frame first, start and silence bounds, close on any end | +| `framing.ts` | `MetaFrameScanner`, `checkedMeta`: codec and size a picture may claim | +| `captures.ts` | Captures per lease; the lease-end participant that stops them | +| `service.ts` | One request either door: method, refusal body, response headers, log line | +| `mounts.ts` | The HTTP mount and the relay target | +| `refusal.ts` | `StreamRefusalError`, each refusal's status and code, a relayed refusal checked | +| `caller.ts` | Who asks | +| `test-fixtures.ts` | `TestRegistry`, `ScriptedFeed` for tests | + +## How it works + +Worker — `worker-link.ts` `createWorkerStreamLink`: + +1. Lease from the registry snapshot. Agent: its requester (relayed) or principal must hold it. + Unknown and not yours both answer 404 `UNKNOWN_LEASE`; nothing is opened first. +2. `captures.begin(leaseId)` — registered before the capture starts. +3. `ctx.screens.openScreen(platform, device, settings)` (the daemon passes its `DriverCatalog`) → the driver's `ScreenFeed`. + `ScreenNotSupportedError` → 415 `STREAM_NOT_SUPPORTED`; anything else → 503 `STREAM_UNAVAILABLE`. +4. `readScreenFeed`: the meta frame within the start bound, or 503 and close. +5. Lease re-read: gone, re-granted, or its slot stopped → close, 404. +6. 200; headers `simlock-stream-{codec,fps,width,height}`; body = the frames, meta first. + +Ends — each closes the capture once (`closedOnce`): the viewer cancels or aborts; the capture +ends or errors; no byte within the silence bound; the lease ends (the `beforeDeviceReclaim` hook, `createStreamLeaseEnd` → +`captures.stop`, which waits for every close, bounded by the driver and by leasing's signal). + +Gateway — `gateway-link.ts`: the fleet lease index names owner, worker and the worker's lease +id. Not the owner (agent) → 404. Relays `GET /stream/leases/` as target `stream` +with tenant `` from the index, never from the caller. A refusal +body is read to 16 KiB; its code and message are forwarded only if the code matches +`^[A-Z][A-Z0-9_]{0,63}$`, else the code becomes `STREAM_UNAVAILABLE`. The worker's 4xx or 5xx +status is kept either way; a 2xx without a picture → 502. Headers go through `checkedMeta`. +The body is relayed unread; the gateway's relay bounds it by inactivity, not by a deadline. + +Capability `stream`: `available`, with no reason, when any driver can capture (a platform that +cannot says why in its 503); `unavailable` when none can, with each platform's reason. + +## State and timers + +Memory only: running captures per lease (`captures.ts`). + +| What | Default | Key | +| ---------------------------- | -------------------- | ------------------------------------------------------------------------------------------------ | +| First frame | 30 s | `DEFAULT_START_MS` | +| Silence between frames | 30 s | `DEFAULT_SILENCE_MS` | +| Frame ceiling (envelope) | 8 MiB | `MAX_FRAME_BYTES` | +| fps / longest side / bitrate | 30 / 720 / 1500 kbps | `capabilities.stream.fps`, `capabilities.stream.maxDimension`, `capabilities.stream.bitrateKbps` | + +## Rules + +- No platform name here: a driver offers `screenCapture` or not (`core/screen.ts`). +- Ownership before any work; unknown and not-yours answer the same. +- A capture is registered before it starts and the lease is re-read after its first frame, + so a capture never outlives its lease (`worker-link.test.ts`). +- A gateway never parses a video byte and never takes a caller's word for the requester. +- No business events: watching changes no lease or device. Each open and refusal is logged. + +## Known gaps + +- **View only.** No touch or key input. +- **Each viewer is its own capture.** Two tabs on one device run two captures. +- **A frame skipped by a slow viewer is not dropped, it is waited for.** A viewer that stops + reading slows the capture behind it to its pace. diff --git a/src/stream/CLAUDE.md b/src/stream/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/src/stream/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/src/stream/caller.ts b/src/stream/caller.ts new file mode 100644 index 00000000..c5f78a84 --- /dev/null +++ b/src/stream/caller.ts @@ -0,0 +1,7 @@ +export type StreamRole = "agent" | "operator"; + +export interface StreamCaller { + readonly principal: string; + readonly requesterId?: string; + readonly role: StreamRole; +} diff --git a/src/stream/captures.test.ts b/src/stream/captures.test.ts new file mode 100644 index 00000000..2622fa27 --- /dev/null +++ b/src/stream/captures.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; + +import { createCaptureRegistry, createStreamLeaseEnd } from "./captures.js"; + +async function flush(): Promise { + for (let turn = 0; turn < 10; turn += 1) await new Promise((resolve) => setImmediate(resolve)); +} + +function ended(leaseId: string) { + return { leaseId } as Parameters["leaseEnded"]>[0]; +} + +describe("the capture registry", () => { + it("a lease that ends while its capture is starting aborts the slot, waits for it to attach, and closes it", async () => { + const captures = createCaptureRegistry(); + const slot = captures.begin("lse_1"); + let stopped = false; + const stopping = captures.stop("lse_1").then(() => (stopped = true)); + expect(slot.ended.aborted).toBe(true); + await Promise.resolve(); + expect(stopped).toBe(false); + let closes = 0; + slot.attach(async () => { + closes += 1; + }); + await stopping; + expect(closes).toBe(1); + }); + + it("stops every capture of the lease that ended and leaves another lease's running", async () => { + const captures = createCaptureRegistry(); + const closed: string[] = []; + captures.begin("lse_1").attach(async () => void closed.push("a")); + captures.begin("lse_1").attach(async () => void closed.push("b")); + captures.begin("lse_2").attach(async () => void closed.push("c")); + expect(await captures.stop("lse_1")).toBe(2); + expect(closed).toEqual(["a", "b"]); + expect(captures.leases()).toEqual(["lse_2"]); + }); + + it("closes a capture once however many doors close it", async () => { + const captures = createCaptureRegistry(); + let closes = 0; + const slot = captures.begin("lse_1"); + slot.attach(async () => { + closes += 1; + }); + await slot.release(); + await slot.release(); + await captures.stop("lse_1"); + expect(closes).toBe(1); + }); + + it("an abandoned open no longer counts as its lease's", async () => { + const captures = createCaptureRegistry(); + captures.begin("lse_1").abandon(); + expect(captures.leases()).toEqual([]); + expect(await captures.stop("lse_1")).toBe(0); + }); + + it("a released capture counts as its lease's until it is closed", async () => { + const captures = createCaptureRegistry(); + const slot = captures.begin("lse_1"); + let finish!: () => void; + slot.attach(() => new Promise((resolve) => (finish = resolve))); + const releasing = slot.release(); + await flush(); + expect(captures.leases()).toEqual(["lse_1"]); + finish(); + await releasing; + expect(captures.leases()).toEqual([]); + }); + + it("a slot released before its capture attached closes that capture once it attaches", async () => { + const captures = createCaptureRegistry(); + const slot = captures.begin("lse_1"); + const releasing = slot.release(); + let closes = 0; + slot.attach(async () => { + closes += 1; + }); + await releasing; + expect(closes).toBe(1); + expect(captures.leases()).toEqual([]); + }); +}); + +describe("the stream's lease-end participant", () => { + it("returns only once the ended lease's captures have closed", async () => { + const captures = createCaptureRegistry(); + let finish!: () => void; + captures.begin("lse_1").attach(() => new Promise((resolve) => (finish = resolve))); + let returned = false; + const done = createStreamLeaseEnd(captures) + .leaseEnded(ended("lse_1"), new AbortController().signal) + .then(() => { + returned = true; + }); + await flush(); + expect(returned).toBe(false); + finish(); + await done; + expect(returned).toBe(true); + }); + + it("stops waiting when leasing gives up on it", async () => { + const captures = createCaptureRegistry(); + captures.begin("lse_1").attach(() => new Promise(() => undefined)); + const abort = new AbortController(); + const done = createStreamLeaseEnd(captures) + .leaseEnded(ended("lse_1"), abort.signal) + .then(() => "returned"); + abort.abort(); + const outcome = await Promise.race([ + done, + new Promise((resolve) => setTimeout(() => resolve("still waiting"), 50)), + ]); + expect(outcome).toBe("returned"); + }); +}); diff --git a/src/stream/captures.ts b/src/stream/captures.ts new file mode 100644 index 00000000..ddeeab3e --- /dev/null +++ b/src/stream/captures.ts @@ -0,0 +1,117 @@ +// why: a capture belongs to a lease, not to a connection -- the lease is the thing that says this +// machine still owns that device. A viewer going away stops a capture through its own request; this +// covers what that cannot, a lease that ends somewhere else (released elsewhere, expired, the device +// lost) while somebody is still watching, or while a capture is still starting. +import type { ModuleLeaseEndParticipant } from "../modules/index.js"; + +export interface CaptureSlot { + // why: aborted when the lease ends, so a capture still being opened learns it has no lease to + // film before it hands a picture out. + readonly ended: AbortSignal; + attach(close: () => Promise): void; + // why: an open that failed has already cleaned up after itself; this settles and forgets the slot + // with nothing to close. + abandon(): void; + // why: the viewer is gone. The slot stays registered until its open settles and what it attached + // is closed, so a lease that ends meanwhile still waits for the device to be left alone. + release(): Promise; +} + +export interface CaptureRegistry { + begin(leaseId: string): CaptureSlot; + stop(leaseId: string): Promise; + leases(): string[]; +} + +interface Slot { + readonly controller: AbortController; + // why: resolves once the open in progress has either attached a capture or given up, so a lease + // that ends mid-open waits for the device to be left alone before its reclaim starts. + readonly settled: Promise; + settle: () => void; + close: (() => Promise) | undefined; + closing: Promise | undefined; +} + +export function createCaptureRegistry(): CaptureRegistry { + const running = new Map>(); + + const closeSlot = async (slot: Slot): Promise => { + await slot.settled; + // why: an open that gave up attached nothing and cleaned up after itself; one that attaches + // after this slot was let go closes from `attach`, so the latch is set only once there is a + // close to run. + if (slot.close === undefined) return; + slot.closing ??= slot.close(); + await slot.closing; + }; + + const forget = (leaseId: string, slot: Slot) => { + const forLease = running.get(leaseId); + forLease?.delete(slot); + if (forLease?.size === 0) running.delete(leaseId); + }; + + return { + // why: registered before the capture starts, not after its first frame -- a lease that ends in + // between must find something to stop, or the capture would come up filming a device that is + // already on its way to the next holder. + begin: (leaseId) => { + let settle = () => undefined as void; + const settled = new Promise((resolve) => (settle = resolve)); + const slot: Slot = { + close: undefined, + closing: undefined, + controller: new AbortController(), + settle, + settled, + }; + const forLease = running.get(leaseId) ?? new Set(); + forLease.add(slot); + running.set(leaseId, forLease); + return { + // why: a capture can finish opening after its viewer left or its lease ended; whoever let + // go of the slot is waiting on `settled`, and closes it the moment it attaches. + attach: (close) => { + slot.close = close; + slot.settle(); + }, + ended: slot.controller.signal, + abandon: () => { + slot.settle(); + forget(leaseId, slot); + }, + release: async () => { + await closeSlot(slot); + forget(leaseId, slot); + }, + }; + }, + leases: () => [...running.keys()], + stop: async (leaseId) => { + const forLease = running.get(leaseId); + if (forLease === undefined) return 0; + running.delete(leaseId); + for (const slot of forLease) slot.controller.abort(); + await Promise.allSettled([...forLease].map(closeSlot)); + return forLease.size; + }, + }; +} + +export function createStreamLeaseEnd(captures: CaptureRegistry): ModuleLeaseEndParticipant { + return { + // why: waits for each capture's close, which a driver bounds, so a reclaim does not wipe or + // reboot a device while its capture is still being torn down on it; leasing's own signal ends + // the wait if leasing gives up first. + leaseEnded: async (ended, signal) => { + await Promise.race([ + captures.stop(ended.leaseId), + new Promise((resolve) => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }), + ]); + }, + }; +} diff --git a/src/stream/create.test.ts b/src/stream/create.test.ts new file mode 100644 index 00000000..775de6db --- /dev/null +++ b/src/stream/create.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; + +import type { ScreenAvailability } from "../core/index.js"; +import { FakeClock, NoopLogger } from "../ports/index.js"; +import { createWorkerStream } from "./create.js"; +import { ScriptedFeed, TestRegistry } from "./test-fixtures.js"; + +function worker(availability: readonly ScreenAvailability[] = [], sendsMeta = true) { + const clock = new FakeClock(); + const registry = new TestRegistry( + [{ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent", requesterId: "gw:inst:agent-1" }], + [{ id: "dev_1", platform: "ios" }], + ); + const feeds: ScriptedFeed[] = []; + const stream = createWorkerStream({ + clock, + logger: new NoopLogger(), + name: "stream", + registry, + screens: { + openScreen: async () => { + const feed = new ScriptedFeed(); + if (sendsMeta) feed.meta(); + feeds.push(feed); + return feed; + }, + screenAvailability: async () => availability, + }, + settings: { bitrateKbps: 1500, fps: 30, maxDimension: 720 }, + }); + return { clock, feeds, stream }; +} + +const relayed = () => new Request("http://simlock-worker/stream/leases/lse_1"); + +async function settledWithin(promise: Promise): Promise { + for (let flush = 0; flush < 10; flush += 1) await new Promise((resolve) => setImmediate(resolve)); + return await Promise.race([promise, Promise.resolve("pending" as const)]); +} + +describe("createWorkerStream", () => { + it("reports the stream available, with no reason, when any platform can capture", async () => { + for (const availability of [ + [{ platform: "android" }, { platform: "ios", reason: "No capture program at /x" }], + [{ platform: "ios", reason: "No capture program at /x" }, { platform: "android" }], + ] as const) { + const { stream } = worker(availability); + expect(stream.capabilitySource.name).toBe("stream"); + expect(await stream.capabilitySource.read()).toEqual({ state: "available" }); + } + }); + + it("gives a capture 30 s to send its first frame", async () => { + const { clock, feeds, stream } = worker([], false); + const served = stream.relayTarget(relayed(), "gw:inst:agent-1", "lse_1"); + await settledWithin(served); + clock.advance(29_999); + expect(await settledWithin(served)).toBe("pending"); + clock.advance(1); + const answer = await settledWithin(served); + expect(answer === "pending" ? answer : answer.status).toBe(503); + expect(feeds[0]?.closes).toBe(1); + }); + + it("ends a stream whose capture sends nothing for 30 s", async () => { + const { clock, feeds, stream } = worker(); + const served = await stream.relayTarget(relayed(), "gw:inst:agent-1", "lse_1"); + const reader = served.body!.getReader(); + await reader.read(); + const next = reader.read().then( + () => "read", + () => "failed", + ); + clock.advance(29_999); + expect(await settledWithin(next)).toBe("pending"); + clock.advance(1); + expect(await settledWithin(next)).toBe("failed"); + expect(feeds[0]?.closes).toBe(1); + }); + + it("reports the stream unavailable with each platform's reason when none can capture", async () => { + const { stream } = worker([ + { platform: "android", reason: "no jar" }, + { platform: "ios", reason: "No capture program at /x" }, + ]); + expect(await stream.capabilitySource.read()).toEqual({ + reason: "android: no jar; ios: No capture program at /x", + state: "unavailable", + }); + }); + + it("reports the stream unavailable when no driver on the host can capture", async () => { + const { stream } = worker([]); + expect(await stream.capabilitySource.read()).toEqual({ + reason: "No driver on this host can capture a screen", + state: "unavailable", + }); + }); + + it("serves a relayed request for the requester the gateway names, checked against this worker's registry", async () => { + const { stream } = worker(); + const request = () => new Request("http://simlock-worker/stream/leases/lse_1"); + const served = await stream.relayTarget(request(), "gw:inst:agent-1", "lse_1"); + expect(served.status).toBe(200); + await served.body?.cancel(); + const refused = await stream.relayTarget(request(), "gw:inst:agent-2", "lse_1"); + expect(refused.status).toBe(404); + }); + + it("stops a relayed capture when its lease ends", async () => { + const { feeds, stream } = worker(); + const served = await stream.relayTarget( + new Request("http://simlock-worker/stream/leases/lse_1"), + "gw:inst:agent-1", + "lse_1", + ); + expect(served.status).toBe(200); + await stream.leaseEnd.leaseEnded( + { leaseId: "lse_1" } as Parameters[0], + new AbortController().signal, + ); + expect(feeds[0]?.closes).toBe(1); + }); +}); diff --git a/src/stream/create.ts b/src/stream/create.ts new file mode 100644 index 00000000..7e47a8f8 --- /dev/null +++ b/src/stream/create.ts @@ -0,0 +1,93 @@ +import type { ScreenAvailability, ScreenSettings } from "../core/index.js"; +import type { ModuleLeaseEndParticipant } from "../modules/index.js"; +import type { Clock, Logger } from "../ports/index.js"; +import { createCaptureRegistry, createStreamLeaseEnd } from "./captures.js"; +import { DEFAULT_SILENCE_MS, DEFAULT_START_MS } from "./feed.js"; +import { createGatewayStreamLink, type FleetPort } from "./gateway-link.js"; +import { + createStreamMount, + createStreamRelayTarget, + type StreamMount, + type StreamRelayTarget, +} from "./mounts.js"; +import type { StreamServiceOptions } from "./service.js"; +import { createWorkerStreamLink, type ScreenOpener, type StreamRegistry } from "./worker-link.js"; + +export interface WorkerStreamOptions { + readonly name: string; + readonly registry: StreamRegistry; + readonly screens: ScreenOpener & { + screenAvailability(): Promise; + }; + readonly clock: Clock; + readonly logger: Logger; + readonly settings: ScreenSettings; +} + +export interface StreamCapabilitySource { + readonly name: string; + read(): Promise<{ readonly state: "available" | "unavailable"; readonly reason?: string }>; +} + +export interface WorkerStream { + readonly mount: StreamMount; + readonly relayTarget: StreamRelayTarget; + readonly leaseEnd: ModuleLeaseEndParticipant; + readonly capabilitySource: StreamCapabilitySource; +} + +// why: the mount, the relay target and the lease-end participant sit on one link and one registry, so +// a stream asked for on this host and one relayed by a gateway go through the same lease check, and a +// lease that ends has one place to stop its captures whichever door the picture came in by. +export function createWorkerStream(options: WorkerStreamOptions): WorkerStream { + const captures = createCaptureRegistry(); + const service: StreamServiceOptions = { + link: createWorkerStreamLink({ + bounds: { clock: options.clock, silenceMs: DEFAULT_SILENCE_MS, startMs: DEFAULT_START_MS }, + captures, + registry: options.registry, + screens: options.screens, + settings: options.settings, + }), + logger: options.logger, + }; + return { + capabilitySource: { + name: options.name, + read: async () => describeAvailability(await options.screens.screenAvailability()), + }, + leaseEnd: createStreamLeaseEnd(captures), + mount: createStreamMount(service), + relayTarget: createStreamRelayTarget(service), + }; +} + +export interface GatewayStreamOptions { + readonly fleet: FleetPort; + readonly logger: Logger; +} + +export function createGatewayStream(options: GatewayStreamOptions): { + readonly mount: StreamMount; +} { + return { + mount: createStreamMount({ + link: createGatewayStreamLink({ fleet: options.fleet }), + logger: options.logger, + }), + }; +} + +function describeAvailability(platforms: readonly ScreenAvailability[]): { + readonly state: "available" | "unavailable"; + readonly reason?: string; +} { + // why: an `available` capability carries no reason on the wire, so a platform that cannot capture + // on a host where another can says why when a stream on it is asked for, as a 503 with its reason. + if (platforms.some((entry) => entry.reason === undefined)) return { state: "available" }; + const why = platforms.map((entry) => `${entry.platform}: ${entry.reason ?? ""}`).join("; "); + return { + reason: why === "" ? "No driver on this host can capture a screen" : why, + state: "unavailable", + }; +} diff --git a/src/stream/definition.ts b/src/stream/definition.ts new file mode 100644 index 00000000..38967fbd --- /dev/null +++ b/src/stream/definition.ts @@ -0,0 +1,26 @@ +import { integerInRange, objectValidator } from "../core/index.js"; +import type { OptionalModuleDefinition } from "../modules/index.js"; + +export interface StreamModuleConfig { + readonly fps: number; + readonly maxDimension: number; + readonly bitrateKbps: number; +} + +// why: the module is called `stream`, so its capability, its relay target and its route segment are +// one word; the picture settings are what a worker encodes with, and a gateway that only relays a +// stream reads none of them. +export const streamModule: OptionalModuleDefinition = { + config: { + defaults: { bitrateKbps: 1_500, fps: 30, maxDimension: 720 }, + gatewayKeys: [], + validator: objectValidator({ + bitrateKbps: integerInRange(100, 40_000), + fps: integerInRange(1, 60), + maxDimension: integerInRange(96, 4096), + }), + }, + loadGateway: () => import("./gateway.js"), + loadWorker: () => import("./worker.js"), + name: "stream", +}; diff --git a/src/stream/feed.test.ts b/src/stream/feed.test.ts new file mode 100644 index 00000000..8365240d --- /dev/null +++ b/src/stream/feed.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; + +import { encodeScreenMeta } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import { readScreenFeed } from "./feed.js"; +import { ScriptedFeed } from "./test-fixtures.js"; + +const BOUNDS = { silenceMs: 1_000, startMs: 5_000 }; + +async function flush(): Promise { + for (let turn = 0; turn < 10; turn += 1) await Promise.resolve(); +} + +describe("readScreenFeed", () => { + it("answers with the meta frame and replays it, and the bytes that came with it, at the head of the body", async () => { + const feed = new ScriptedFeed(); + const meta = encodeScreenMeta({ codec: "h264", fps: 30, height: 720, width: 330 }); + feed.write(Uint8Array.from([...meta, 9, 9])); + const stream = await readScreenFeed(feed, { clock: new FakeClock(), ...BOUNDS }); + const reader = stream.body.getReader(); + expect([...((await reader.read()).value ?? [])]).toEqual([...meta, 9, 9]); + expect(stream.meta.width).toBe(330); + }); + + it("fails before the first frame only at the start bound, not the shorter silence bound", async () => { + const clock = new FakeClock(); + const feed = new ScriptedFeed(); + const outcome = readScreenFeed(feed, { clock, ...BOUNDS }).then( + () => "answered", + (error: unknown) => (error as Error).message, + ); + clock.advance(BOUNDS.silenceMs); + await flush(); + feed.meta(); + expect(await outcome).toBe("answered"); + }); + + it("refuses a capture that paints nothing within the start bound, and closes it", async () => { + const clock = new FakeClock(); + const feed = new ScriptedFeed(); + const outcome = readScreenFeed(feed, { clock, ...BOUNDS }); + clock.advance(BOUNDS.startMs); + await expect(outcome).rejects.toThrow( + /wrote nothing for 5000ms while waiting for its first frame/u, + ); + expect(feed.closes).toBe(1); + }); + + it("refuses a capture that ends before it writes a frame", async () => { + const feed = new ScriptedFeed(); + const outcome = readScreenFeed(feed, { clock: new FakeClock(), ...BOUNDS }); + feed.end(); + await expect(outcome).rejects.toThrow(/ended before it wrote a frame/u); + }); + + it("errors the body, and closes the capture, when it goes quiet mid-picture past the silence bound", async () => { + const clock = new FakeClock(); + const feed = new ScriptedFeed(); + feed.meta(); + const stream = await readScreenFeed(feed, { clock, ...BOUNDS }); + const reader = stream.body.getReader(); + await reader.read(); + const next = reader.read(); + await flush(); + clock.advance(BOUNDS.silenceMs); + await expect(next).rejects.toThrow( + /wrote nothing for 1000ms while waiting for the next frame/u, + ); + expect(feed.closes).toBe(1); + }); + + it("keeps a picture that keeps arriving within the silence bound", async () => { + const clock = new FakeClock(); + const feed = new ScriptedFeed(); + feed.meta(); + const stream = await readScreenFeed(feed, { clock, ...BOUNDS }); + const reader = stream.body.getReader(); + await reader.read(); + for (let frame = 0; frame < 3; frame += 1) { + const next = reader.read(); + await flush(); + clock.advance(BOUNDS.silenceMs - 1); + feed.keyframe(); + expect((await next).done).toBe(false); + } + expect(feed.closes).toBe(0); + }); + + it("closes the capture when the viewer cancels the body", async () => { + const feed = new ScriptedFeed(); + feed.meta(); + const stream = await readScreenFeed(feed, { clock: new FakeClock(), ...BOUNDS }); + await stream.body.cancel(); + expect(feed.closes).toBe(1); + }); +}); diff --git a/src/stream/feed.ts b/src/stream/feed.ts new file mode 100644 index 00000000..14df533c --- /dev/null +++ b/src/stream/feed.ts @@ -0,0 +1,134 @@ +import type { ScreenFeed, ScreenMeta } from "../core/index.js"; +import type { Clock, TimerHandle } from "../ports/index.js"; +import { MetaFrameScanner } from "./framing.js"; + +export interface CapturedStream { + readonly meta: ScreenMeta; + readonly body: ReadableStream; + close(): Promise; +} + +export interface FeedBounds { + readonly clock: Clock; + readonly startMs: number; + readonly silenceMs: number; +} + +// why: a cold simulator or an emulator whose encoder is warming up can take seconds to paint its +// first frame, and a capture that never paints must still end; generous, but a bound. +export const DEFAULT_START_MS = 30_000; +// why: a screen that does not change still yields frames -- the iOS capture repaints at an idle floor +// and the Android driver writes an `alive` frame when scrcpy has nothing new -- so this long without a byte is a capture that died without +// saying so, and a viewer must not be left looking at a frozen picture it cannot tell from a live one. +export const DEFAULT_SILENCE_MS = 30_000; + +export async function readScreenFeed( + feed: ScreenFeed, + bounds: FeedBounds, +): Promise { + const reader = feed.body.getReader(); + const scanner = new MetaFrameScanner(); + const buffered: Uint8Array[] = []; + const read = (ms: number, what: string) => readWithin(reader, bounds.clock, ms, what); + + try { + for (;;) { + const step = await read(bounds.startMs, "its first frame"); + if (step.done) throw new Error("The screen capture ended before it wrote a frame"); + buffered.push(step.value); + const found = scanner.push(step.value); + if (found !== undefined) { + // why: everything read so far goes back on the front in order. The meta frame is what the + // response head was answered from but a decoder needs it in the body too, and bytes that + // arrived in the same read as it are the stream's first frames. + return { + body: followedBy( + concat(buffered), + () => read(bounds.silenceMs, "the next frame"), + reader, + feed, + ), + close: () => feed.close(), + meta: found.meta, + }; + } + } + } catch (error) { + void reader.cancel().catch(() => undefined); + await feed.close(); + throw error; + } +} + +function readWithin( + reader: ReadableStreamDefaultReader, + clock: Clock, + ms: number, + what: string, +): Promise> { + return new Promise((resolve, reject) => { + let timer: TimerHandle | undefined = clock.setTimer(ms, () => { + timer = undefined; + reject( + new Error(`The screen capture wrote nothing for ${String(ms)}ms while waiting for ${what}`), + ); + }); + const settle = (outcome: () => void) => { + if (timer === undefined) return; + clock.cancel(timer); + timer = undefined; + outcome(); + }; + reader.read().then( + (step) => settle(() => resolve(step)), + (error: unknown) => + settle(() => reject(error instanceof Error ? error : new Error(String(error)))), + ); + }); +} + +function followedBy( + prefix: Uint8Array, + read: () => Promise>, + source: ReadableStreamDefaultReader, + feed: ScreenFeed, +): ReadableStream { + let handed = false; + return new ReadableStream({ + cancel: async (reason) => { + await source.cancel(reason).catch(() => undefined); + await feed.close(); + }, + async pull(controller) { + if (!handed) { + handed = true; + controller.enqueue(prefix); + return; + } + try { + const step = await read(); + if (step.done) { + await feed.close(); + controller.close(); + } else controller.enqueue(step.value); + } catch (error) { + // why: a read that failed is the capture's end as much as a clean close is, and the device + // must not go on being filmed for a viewer whose stream already errored. + void source.cancel().catch(() => undefined); + await feed.close(); + controller.error(error); + } + }, + }); +} + +function concat(chunks: readonly Uint8Array[]): Uint8Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} diff --git a/src/stream/framing.test.ts b/src/stream/framing.test.ts new file mode 100644 index 00000000..5a21f5a5 --- /dev/null +++ b/src/stream/framing.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; + +import { SCREEN_FRAME_TAG as CAPTURE_TAG } from "../core/index.js"; +import { checkedMeta, MAX_FRAME_BYTES, MetaFrameScanner } from "./framing.js"; + +const META = { codec: "h264", deviceSet: "/set", fps: 30, height: 720, udid: "UDID", width: 330 }; + +function frame(tag: number, payload: Uint8Array): Uint8Array { + const head = new Uint8Array(5); + new DataView(head.buffer).setUint32(0, payload.length + 1); + head[4] = tag; + return Uint8Array.from([...head, ...payload]); +} + +function metaFrame(payload: unknown = META): Uint8Array { + return frame(CAPTURE_TAG.meta, new TextEncoder().encode(JSON.stringify(payload))); +} + +describe("MetaFrameScanner", () => { + it("finds a meta frame only once its last byte has arrived", () => { + const bytes = metaFrame(); + const scanner = new MetaFrameScanner(); + const found = []; + // why: one byte at a time, because a header split across reads is what a pipe actually does, and a + // scanner that reads a length it has not fully received acts on bytes that are not there yet. + for (const byte of bytes) { + found.push(scanner.push(Uint8Array.from([byte]))); + } + expect(found.slice(0, -1).every((step) => step === undefined)).toBe(true); + expect(found.at(-1)?.meta).toEqual({ codec: "h264", fps: 30, height: 720, width: 330 }); + expect(found.at(-1)?.bytes).toEqual(bytes); + }); + + it("reports the meta frame it returned as exactly the bytes that frame occupies", () => { + const bytes = metaFrame(); + const scanner = new MetaFrameScanner(); + const found = scanner.push(bytes); + expect(found?.bytes.byteLength).toBe(bytes.byteLength); + }); + + it("refuses a stream that puts a video frame before the meta frame", () => { + const scanner = new MetaFrameScanner(); + // why: the meta frame is the one promise the envelope makes, and a decoder configured from a header + // that never arrived would fail later and further away from the cause. + expect(() => scanner.push(frame(CAPTURE_TAG.keyframe, new Uint8Array([1, 2, 3])))).toThrow( + /does not begin with a meta frame/u, + ); + expect(() => scanner.push(frame(CAPTURE_TAG.description, new Uint8Array([1])))).toThrow( + /does not begin with a meta frame/u, + ); + }); + + it("refuses a header whose length is not a frame", () => { + for (const length of [0, MAX_FRAME_BYTES + 10, 0xffff_ffff]) { + const head = new Uint8Array(5); + new DataView(head.buffer).setUint32(0, length); + head[4] = CAPTURE_TAG.meta; + expect(() => new MetaFrameScanner().push(head)).toThrow(/not a frame envelope/u); + } + }); + + it("refuses a meta frame whose payload is not JSON rather than reporting a stream with no size", () => { + const scanner = new MetaFrameScanner(); + // why: a size of zero would answer a response with headers saying so, and the viewer would sit on a + // stream it can never decode; the tool's own log is where that failure is explained. + expect(() => scanner.push(frame(CAPTURE_TAG.meta, new TextEncoder().encode("")))).toThrow( + /meta frame is not JSON/u, + ); + }); + + it("reads an mjpeg meta as mjpeg", () => { + const scanner = new MetaFrameScanner(); + const found = scanner.push(metaFrame({ codec: "mjpeg", fps: 5, height: 720, width: 330 })); + expect(found?.meta).toEqual({ codec: "mjpeg", fps: 5, height: 720, width: 330 }); + }); + + it("refuses a meta frame that leaves out the size rather than reporting a stream with no size", () => { + expect(() => new MetaFrameScanner().push(metaFrame({ codec: "h264", fps: 30 }))).toThrow( + /is not a size a screen has/u, + ); + }); +}); + +describe("checkedMeta", () => { + it("refuses a codec the stream cannot carry instead of calling it h264", () => { + expect(() => checkedMeta("vp9", 330, 720, 30)).toThrow(/codec this stream cannot carry: vp9/u); + }); + + it("refuses a size no screen has", () => { + expect(() => checkedMeta("h264", 330, 1_000_000, 30)).toThrow(/height/u); + expect(() => checkedMeta("h264", 0, 720, 30)).toThrow(/width/u); + expect(() => checkedMeta("h264", 330.5, 720, 30)).toThrow(/width/u); + expect(() => checkedMeta("h264", 330, 720, -1)).toThrow(/fps/u); + }); +}); diff --git a/src/stream/framing.ts b/src/stream/framing.ts new file mode 100644 index 00000000..c8f99ba4 --- /dev/null +++ b/src/stream/framing.ts @@ -0,0 +1,80 @@ +import { SCREEN_FRAME_HEADER_BYTES, SCREEN_FRAME_TAG, type ScreenMeta } from "../core/index.js"; + +export const MAX_FRAME_BYTES = 8 * 1024 * 1024; + +// why: a size is a claim from a program or a worker, and both end up in a browser's decoder +// configuration and a response header; anything past this is not a screen any device has. +const MAX_SIDE = 16_384; +const MAX_FPS = 240; + +export interface MetaFrame { + readonly meta: ScreenMeta; + readonly bytes: Uint8Array; +} + +export class MetaFrameScanner { + #buffer = new Uint8Array(0); + + push(chunk: Uint8Array): MetaFrame | undefined { + if (this.#buffer.length > MAX_FRAME_BYTES) { + throw new Error("The screen capture wrote no meta frame before an oversized frame"); + } + const merged = new Uint8Array(this.#buffer.length + chunk.length); + merged.set(this.#buffer, 0); + merged.set(chunk, this.#buffer.length); + this.#buffer = merged; + + if (this.#buffer.length < SCREEN_FRAME_HEADER_BYTES) return undefined; + const length = new DataView(this.#buffer.buffer, this.#buffer.byteOffset).getUint32(0); + const total = SCREEN_FRAME_HEADER_BYTES + length - 1; + if (length < 1 || total > MAX_FRAME_BYTES) { + throw new Error("The screen capture's output is not a frame envelope"); + } + // why: a frame ahead of the meta frame means a capture that broke the one promise the envelope + // makes -- meta first -- so the stream is refused rather than decoded blind. + if (this.#buffer[4] !== SCREEN_FRAME_TAG.meta) { + throw new Error("The screen capture's stream does not begin with a meta frame"); + } + if (this.#buffer.length < total) return undefined; + const meta = readScreenMeta(this.#buffer.subarray(SCREEN_FRAME_HEADER_BYTES, total)); + return { bytes: this.#buffer.slice(0, total), meta }; + } +} + +function readScreenMeta(payload: Uint8Array): ScreenMeta { + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(payload)); + } catch { + throw new Error("The screen capture's meta frame is not JSON"); + } + const fields = + typeof parsed === "object" && parsed !== null ? (parsed as Record) : {}; + return checkedMeta(fields["codec"], fields["width"], fields["height"], fields["fps"]); +} + +// why: one check for a meta frame read from a capture and for the headers a gateway reads from a +// worker, so the two doors cannot disagree about what a picture may claim to be. +export function checkedMeta( + codec: unknown, + width: unknown, + height: unknown, + fps: unknown, +): ScreenMeta { + if (codec !== "h264" && codec !== "mjpeg") { + throw new Error(`The screen capture names a codec this stream cannot carry: ${String(codec)}`); + } + return { + codec, + fps: bounded(fps, 0, MAX_FPS, "fps"), + height: bounded(height, 1, MAX_SIDE, "height"), + width: bounded(width, 1, MAX_SIDE, "width"), + }; +} + +function bounded(value: unknown, min: number, max: number, name: string): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < min || value > max) { + throw new Error(`The screen capture's ${name} is not a size a screen has: ${String(value)}`); + } + return value; +} diff --git a/src/stream/gateway-link.test.ts b/src/stream/gateway-link.test.ts new file mode 100644 index 00000000..9652572a --- /dev/null +++ b/src/stream/gateway-link.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamCaller } from "./caller.js"; +import { createGatewayStreamLink, type FleetPort } from "./gateway-link.js"; +import { StreamRefusalError } from "./refusal.js"; + +const AGENT: StreamCaller = { principal: "tok_agent", role: "agent" }; +const STRANGER: StreamCaller = { principal: "tok_other", role: "agent" }; + +interface Relayed { + readonly workerId: string; + readonly url: string; + readonly tenant: string; + readonly leaseId: string; +} + +class CodedError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + } +} + +function fleet(answer: () => Response): { port: FleetPort; relayed: Relayed[] } { + const relayed: Relayed[] = []; + return { + port: { + holder: (leaseId) => + leaseId === "w1.lse_9" + ? { ownerId: "tok_agent", requesterId: "agent-1", workerId: "w1", workerLeaseId: "lse_9" } + : undefined, + relay: async (workerId, call) => { + relayed.push({ + leaseId: call.leaseId, + tenant: call.tenant, + url: call.request.url, + workerId, + }); + return answer(); + }, + requesterPrefix: "gw:inst:", + }, + relayed, + }; +} + +function streamHeaders(overrides: Record = {}): Record { + return { + "simlock-stream-codec": "h264", + "simlock-stream-fps": "30", + "simlock-stream-height": "720", + "simlock-stream-width": "330", + ...overrides, + }; +} + +async function refusalOf(served: Promise): Promise { + const outcome = await served.then( + () => undefined, + (error: unknown) => error, + ); + expect(outcome, "the stream was expected to be refused").toBeInstanceOf(StreamRefusalError); + return outcome as StreamRefusalError; +} + +const signal = () => new AbortController().signal; + +describe("the gateway's stream link", () => { + it("relays to the worker holding the lease, by the worker's own lease id, as the lease's namespaced requester", async () => { + const { port, relayed } = fleet(() => new Response("bytes", { headers: streamHeaders() })); + const stream = await createGatewayStreamLink({ fleet: port }).serve( + "w1.lse_9", + AGENT, + signal(), + ); + expect(relayed).toEqual([ + { + leaseId: "lse_9", + tenant: "gw:inst:agent-1", + url: "http://simlock-worker/stream/leases/lse_9", + workerId: "w1", + }, + ]); + expect(stream.meta).toEqual({ codec: "h264", fps: 30, height: 720, width: 330 }); + }); + + it("answers a lease the fleet does not hold with 404 UNKNOWN_LEASE, relaying nothing", async () => { + const { port, relayed } = fleet(() => new Response("bytes", { headers: streamHeaders() })); + const refusal = await refusalOf( + createGatewayStreamLink({ fleet: port }).serve("w1.lse_ended", AGENT, signal()), + ); + expect([refusal.status, refusal.code]).toEqual([404, "UNKNOWN_LEASE"]); + expect(relayed).toEqual([]); + }); + + it("answers a caller who does not own the lease exactly as an unknown lease, relaying nothing", async () => { + const { port, relayed } = fleet(() => new Response("bytes", { headers: streamHeaders() })); + const refusal = await refusalOf( + createGatewayStreamLink({ fleet: port }).serve("w1.lse_9", STRANGER, signal()), + ); + expect([refusal.status, refusal.code]).toEqual([404, "UNKNOWN_LEASE"]); + expect(relayed).toEqual([]); + }); + + it("forwards the worker's own refusal code and status", async () => { + const { port } = fleet(() => + Response.json( + { error: { code: "STREAM_NOT_SUPPORTED", message: "A physical phone cannot" } }, + { status: 415 }, + ), + ); + const refusal = await refusalOf( + createGatewayStreamLink({ fleet: port }).serve("w1.lse_9", AGENT, signal()), + ); + expect([refusal.status, refusal.code, refusal.message]).toEqual([ + 415, + "STREAM_NOT_SUPPORTED", + "A physical phone cannot", + ]); + }); + + it("does not forward a refusal code that is not shaped like one of ours", async () => { + const { port } = fleet(() => + Response.json({ error: { code: "