From 3e9307f3c18ebd0f6041fd024a525e4c76d948fc Mon Sep 17 00:00:00 2001 From: YONGJAE LEE Date: Thu, 3 Sep 2026 01:00:28 +0900 Subject: [PATCH] [ZEPPELIN-6666] Add notebook transport fixtures --- zeppelin-web-angular/angular.json | 2 +- zeppelin-web-angular/e2e/AGENTS.md | 1 + .../e2e/core-contract/README.md | 347 +++ .../e2e/core-contract/capture-server.sh | 389 +++ .../e2e/core-contract/capture-server.test.mjs | 527 ++++ .../core-contract/capture-stub-zeppelin.mjs | 60 + .../e2e/core-contract/fixture-doubles.d.mts | 41 + .../e2e/core-contract/fixture-doubles.mjs | 45 + .../notebook-transport-fixture.d.mts | 124 + .../notebook-transport-fixture.mjs | 1109 +++++++ .../notebook-transport-fixture.test.mjs | 2629 +++++++++++++++++ .../core-contract/playwright-runner.test.mjs | 141 + zeppelin-web-angular/e2e/global.setup.ts | 9 +- .../e2e/models/login-page.util.ts | 12 +- .../e2e/models/notebook-keyboard-page.ts | 10 +- .../core-contract/capture-fixtures.spec.ts | 400 +++ .../tests/notebook/inline-completion.spec.ts | 29 +- .../notebook-keyboard-shortcuts.spec.ts | 33 +- .../paragraph/paragraph-functionality.spec.ts | 17 +- .../eslint-rules/core-contract-config.test.js | 31 + zeppelin-web-angular/eslint.config.js | 24 + zeppelin-web-angular/package-lock.json | 8 +- zeppelin-web-angular/package.json | 17 +- zeppelin-web-angular/playwright.config.js | 6 +- .../playwright.core-contract.config.js | 64 + zeppelin-web-angular/pom.xml | 26 + .../src/components/login/login.controller.js | 8 +- .../components/login/login.controller.test.js | 75 + 28 files changed, 6116 insertions(+), 68 deletions(-) create mode 100644 zeppelin-web-angular/e2e/core-contract/README.md create mode 100755 zeppelin-web-angular/e2e/core-contract/capture-server.sh create mode 100644 zeppelin-web-angular/e2e/core-contract/capture-server.test.mjs create mode 100755 zeppelin-web-angular/e2e/core-contract/capture-stub-zeppelin.mjs create mode 100644 zeppelin-web-angular/e2e/core-contract/fixture-doubles.d.mts create mode 100644 zeppelin-web-angular/e2e/core-contract/fixture-doubles.mjs create mode 100644 zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.d.mts create mode 100644 zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.mjs create mode 100644 zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.test.mjs create mode 100644 zeppelin-web-angular/e2e/core-contract/playwright-runner.test.mjs create mode 100644 zeppelin-web-angular/e2e/tests/notebook/core-contract/capture-fixtures.spec.ts create mode 100644 zeppelin-web-angular/eslint-rules/core-contract-config.test.js create mode 100644 zeppelin-web-angular/playwright.core-contract.config.js create mode 100644 zeppelin-web/src/components/login/login.controller.test.js diff --git a/zeppelin-web-angular/angular.json b/zeppelin-web-angular/angular.json index 0561cd77b29..2dc47a4154f 100644 --- a/zeppelin-web-angular/angular.json +++ b/zeppelin-web-angular/angular.json @@ -151,7 +151,7 @@ "lint": { "builder": "@angular-eslint/builder:lint", "options": { - "lintFilePatterns": ["src/**/*.ts", "src/**/*.html", "test/**/*.ts", "e2e/**/*.ts"] + "lintFilePatterns": ["src/**/*.ts", "src/**/*.html", "test/**/*.ts", "e2e/**/*.ts", "e2e/**/*.mjs"] } } } diff --git a/zeppelin-web-angular/e2e/AGENTS.md b/zeppelin-web-angular/e2e/AGENTS.md index 9b76c1f3874..5558eab30d6 100644 --- a/zeppelin-web-angular/e2e/AGENTS.md +++ b/zeppelin-web-angular/e2e/AGENTS.md @@ -141,6 +141,7 @@ Pages are moving from Angular to React fragments incrementally. Today this is na ### Suite Shape - Keep the composed suite focused on real cross-seam user flows. Behavior that lives entirely inside one fragment belongs in that fragment's own tests; do not grow the composed suite into a per-fragment unit suite. +- The capture suite in `tests/notebook/core-contract/` tags its live-capture test `@live`, because it needs an isolated Zeppelin server. `playwright.config.js` excludes `@live`; `e2e:core-contract:live` selects it through `playwright.core-contract.config.js` and requires an explicit server URL. The live test deletes its own note in `finally`. The dedicated non-live runner has no auth setup, global hooks or backend cleanup. Tag live tests rather than excluding a whole file and hiding its synthetic tests. ## Classic UI Tests (`e2e/tests/classic/`) diff --git a/zeppelin-web-angular/e2e/core-contract/README.md b/zeppelin-web-angular/e2e/core-contract/README.md new file mode 100644 index 00000000000..cfb94293b78 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/README.md @@ -0,0 +1,347 @@ + + +# Notebook transport contract fixtures + +This directory defines the versioned REST and WebSocket fixture format used by +Notebook adapter tests. It is an in-repository contract test, not a Pact +consumer/provider contract and not a replacement for live-server E2E tests. + +## Fixture ownership + +Every committed fixture includes these required fields. +`createNotebookTransportRecorder` rejects a capture without them and +`validateFixture` reports them as errors. The replay adapter does not: it calls +`validateReplayFixture`, which checks record shape and ordering only, so a fixture +replayed without being validated first is never checked for metadata. + +```json +{ + "version": 1, + "metadata": { + "scenario": "Open a notebook", + "owner": "zeppelin-web-angular", + "coveredOperations": ["GET_NOTE"], + "knownExclusions": ["Live interpreter execution is covered by a separate E2E scenario"] + }, + "records": [ + { + "kind": "websocket", + "sequence": 1, + "websocket": { "direction": "send", "payloadText": "{\"op\":\"GET_NOTE\"}" } + } + ] +} +``` + +- `scenario` describes the user-visible flow. +- `owner` identifies the component that maintains the fixture. +- `coveredOperations` lists the REST or WebSocket operations represented by the + fixture. +- `knownExclusions` records intentionally uncovered behavior. An empty array + is valid when there are no exclusions. + +Add a fixture when a Notebook operation is moved into the shared adapter +contract. If that operation cannot yet be represented, add its explicit reason +to the scenario's `knownExclusions`; do not silently rely on another fixture. + +## What version 1 does not model + +A fixture records one observed interleaving of REST and WebSocket traffic and delivers +it in exactly that order. The migration plan this format serves does not assume the +observed order of an HTTP response and its related WebSocket events is guaranteed, +and the capture scenarios that follow this issue (ZEPPELIN-6671, ZEPPELIN-6672) have +to reproduce duplicate, reordered, late and dropped events. Version 1 cannot express +"either order is acceptable here": it can only pin the order that was captured. + +A REST response occupies the position of Playwright's `requestfinished` event, +when its body has finished downloading. The earlier `response` event only supplies +headers. Reading the body with `response.text()` is asynchronous and must not move +that position past later WebSocket frames. `stop()` and `write()` wait for those +body reads; `stop()` also allows already captured responses to finish after it +stops accepting new traffic. This format replays complete response bodies, not +header-only availability, intermediate HTTP chunks or application callback timing. + +For example, a client that awaits `fetch()` headers and sends another request or a +WebSocket acknowledgement before reading the body needs a separate header event. +Version 1 cannot reproduce that dependency and replay can stall. Such a scenario +must list this limitation in `knownExclusions`; capture validation checks record +structure, not whether application-level dependencies are replayable. Scenarios +using complete JSON responses should consume and assert those bodies during both +capture and replay. + +What is pinned is the order the fixture *answers* in, not the order the page happens to +ask in, because a browser issues requests when it wants and a fixture cannot dictate +that. Two tolerances follow from it, and nothing beyond them: + +- Consecutive REST request records form a request batch, ending at the next response + or WebSocket record. Requests in that batch are matched by shape regardless of + arrival order; responses retain their recorded order. For example, request A, + request B, response A, response B replays when B arrives before A. +- A request that arrives while the fixture is expecting a WebSocket frame waits, as long + as a later record answers it. + +A request outside the current batch is rejected as a request mismatch. Response-only +fixtures contain no request batches, so they reject a different first request as out +of order. A request no remaining record can answer and a WebSocket frame that does +not match the next recorded frame are also rejected. + +Two identical requests in flight at once cannot be told apart either. A record is +matched to a route by request shape - method, path, headers and body - so if a page +issues the same request twice concurrently and the capture recorded two different +responses, replay may hand each route the other's response and still report success. +Sequential requests are unaffected, because they are matched in arrival order. + +`assertComplete()` requires every REST delivery to have settled successfully. A failed +fulfillment rejects its route and prevents successful completion, while independent +routes can still receive their recorded responses. + +`principal` is redacted wherever it appears, and Zeppelin puts it on nearly every +WebSocket frame; `user`, `users` and `roles` go the same way. A fixture therefore names +nobody, which also means it cannot express a scenario that turns on who is acting. The +permission and authentication scenarios in ZEPPELIN-6673 will need that distinction. + +Only `accept` and `content-type` survive header filtering, so a fixture cannot carry +the `www-authenticate` of a 401 or the `location` of a redirect, both of which +ZEPPELIN-6673 needs. + +Two more limits follow from the same shape. A record carries no timing, so a +scenario that turns on delay - a server change applied before an HTTP response times +out - cannot be replayed. And a fixture models one WebSocket connection, so the +reconnection scenarios in ZEPPELIN-6672 need more than version 1 provides. +Capture rejects a second notebook WebSocket connection, including a reconnect, +instead of flattening connections into a fixture that cannot be replayed. + +That is deliberate. Widening the format before those scenarios exist would mean +designing for guesses. When a scenario needs it, the `version` field is the place to +introduce it, and a fixture written for an older version is rejected rather than +silently reinterpreted. + +## Test layers + +Run the format, redaction, ordering, and replay checks with: + +```bash +npm run check:core-contract-fixtures +``` + +That covers the format, the redaction rules and the replay adapter against records, +and takes a couple of seconds. The capture server has a suite of its own, which starts +`capture-server.sh` for real against a stub to cover start, stop and pid-file +behaviour. It spawns processes and binds a free local port, so it takes tens of +seconds and runs in Maven's integration-test phase rather than on every build: + +```bash +npm run check:core-contract-server +``` + +One thing neither layer reaches: the recorder's rejection of a binary WebSocket +frame. A frame can only be recorded from a real WebSocket server - a socket answered +by `route.fulfill` never opens, and one mocked with `routeWebSocket` does not raise +`page.on('websocket')` - so that path is covered only by the node test's event +emitter, and its behaviour under Playwright's own dispatch is unverified. + +The checks above need no browser. What a browser adds is the adapter's contact with +Playwright's own `Request`, `Response`, `Route` and `WebSocket` objects, which a hand +written double cannot stand in for - a `route.continue()` that should have been +`route.fallback()` looked correct against doubles for a long time. Browser tests cover +that contact, including both HTTP-body-first and WebSocket-first exchanges against +a local server. A capture/replay round trip also checks requests without Accept, because Playwright does not +describe a request identically to a `page.on('request')` listener and to a `page.route` +handler: + +```bash +npm run e2e:core-contract +``` + +The focused command uses `playwright.core-contract.config.js`. Its non-live mode +has no authentication setup, stored browser session, global setup/teardown or dev +server. It does not contact `PLAYWRIGHT_BASE_URL` or clean notebooks from another run. +The focused command runs Chromium. The ordinary E2E suite still includes the synthetic +browser tests in its Chromium, Firefox and WebKit projects and excludes `@live`. + +The capture server requires `lsof` to verify listener ownership and a built checkout +(`./mvnw clean install -DskipTests -pl zeppelin-web-angular -am`). A startup failure +reports the server log; a successful HTTP response alone does not establish ownership. +The capture root and repository paths must not contain whitespace, including in +physical paths reached through symlinks. The Zeppelin launcher splits JVM arguments +on whitespace; the capture script rejects these paths before creating files or +launching a process. Choose a root and checkout without spaces, tabs or newlines. +Start and stop hold an atomic `.capture-operation-lock` in the capture root until the +operation completes. A concurrent operation fails. If an operation is killed with +SIGKILL, inspect its processes before removing a leftover lock or `starting` claim; +the script does not guess that another operation's lock is stale. + +Live capture uses the same dedicated config with an explicit `PLAYWRIGHT_BASE_URL`, +authentication setup and zero retries. The test deletes only the note it created, +in a `finally` block; neither mode invokes the shared API cleanup. `CI=true` disables +screenshots and video. Point the login helper at the capture root even in anonymous +mode: its absent `shiro.ini` prevents fallback to unrelated repository credentials. +Authentication state and browser results use separate directories under a unique +temporary run directory. They do not overwrite the ordinary suite's auth snapshot or +test results. Set `ZEPPELIN_CORE_CONTRACT_RUN_DIR` to keep them in a chosen capture +directory; use a different directory for each concurrent run. The auth snapshot is +under `.auth/user.json` and results are under `results/`; remove the run directory +when its artifacts are no longer needed. + +The server discards inherited `ZEPPELIN_*` settings, JVM option variables +(`JAVA_OPTS`, `JAVA_TOOL_OPTIONS`, `_JAVA_OPTIONS`, `JDK_JAVA_OPTIONS`) and `CLASSPATH`. +It explicitly selects local `VFSNotebookRepo` storage and loopback binding, so a shell's +remote notebook configuration cannot redirect capture writes. `JAVA_HOME` and `PATH` +still select the installed toolchain. + +```bash +CAPTURE_ROOT="$(mktemp -d)" +e2e/core-contract/capture-server.sh start --root "${CAPTURE_ROOT}" --port 18080 +ZEPPELIN_E2E_SHIRO_INI="${CAPTURE_ROOT}/conf/shiro.ini" \ + ZEPPELIN_CORE_CONTRACT_RUN_DIR="${CAPTURE_ROOT}/browser" \ + CI=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:18080 npm run e2e:core-contract:live +e2e/core-contract/capture-server.sh stop --root "${CAPTURE_ROOT}" +``` + +For authenticated capture, add `--mode auth` to start. That installs +`shiro.ini.template` in the capture root; the same `ZEPPELIN_E2E_SHIRO_INI` setting +selects it. The helper wiring and a successful authenticated capture are separate +checks. Multi-user permission scenarios remain the responsibility of ZEPPELIN-6673. +`npm run check:core-contract-auth` runs a browser-backed anonymous setup regression +in a disposable directory and verifies that the ordinary auth snapshot is preserved. +It requires installed Chromium but no Zeppelin server; the Node-only fixture check +skips this browser regression. + +Replay is strict about what it answers: a request no remaining record can answer, a +WebSocket frame that does not match the next recorded one, and any record left +unconsumed all fail the test. The limits of that strictness are in "What version 1 +does not model". Maven runs the format checks in its test phase and the capture-server checks +in integration-test. The browser tests are part of the ordinary e2e suite, so they +run wherever it does. The live capture is the one layer nothing runs for you: it is +tagged `@live` and excluded until `npm run e2e:core-contract:live` asks for it. + +These checks prove fixture shape and adapter transport behavior. Separate E2E +scenarios must cover a running Zeppelin server, authorization, collaboration, +reconnection, interpreter execution, streaming output, performance, and +accessibility. + +### How a replay reports failure + +The Playwright adapter reports a broken fixture by rejecting the route handler, which +Playwright surfaces as an unhandled error and attributes to the running test. A route +whose key no remaining record can answer is rejected immediately. A route that is +merely waiting is not: the fixture cannot tell "the page has not sent that request +yet" from "the page will never send it", so that case is left to Playwright's own +test timeout. + +## Capturing safely + +`createNotebookTransportRecorder(metadata)` records only `/api/notebook` REST +traffic and `/ws` frames. It redacts configured sensitive and volatile fields +before it writes a fixture. JSON WebSocket frames are normalized and redacted; +binary frames are rejected during capture until a binary redaction policy is +implemented. Replay still supports deliberately authored binary fixtures for +protocol-level tests. +Each WebSocket record must contain exactly one of `payloadText` and `payloadBase64`; +validation rejects records with both representations. + +A notebook request still awaiting a response when `stop()` or `write()` is called +fails the capture, as does a failed request. Await scenario completion before stopping. +`validateFixture` also rejects request records without a matching response, so an +incomplete hand-authored fixture fails validation before replay. + +### Which identifiers are normalized, and which are kept + +Timestamp fields `dateCreated`, `dateStarted`, `dateFinished`, `dateUpdated`, +`lastUpdated` and `time` are replaced by placeholders. Timing-dependent scenarios +remain outside v1. + +WebSocket envelope `msgId` is a correlation key: Angular uses the echoed ID to focus +a locally inserted or cloned paragraph. Capture assigns distinct stable placeholders +such as `` and preserves repeated references to each ID. Replay binds these +to IDs actually sent by the client and substitutes the live ID in matching responses. +It rejects inconsistent or reused bindings. Legacy fixtures containing the erased +`` envelope value must be recaptured because their identity relationships +cannot be recovered. Non-envelope `msgId` fields retain the generic normalization rule. + +Names that carry a person are masked the same way, by field name only: `user`, which is +what Zeppelin calls the acting principal on a paragraph, the `owners`, `readers`, +`writers` and `runners` that `/api/notebook/{id}/permissions` answers with, the `users` a +`COLLABORATIVE_MODE_STATUS` frame lists while more than one session has the note open, +and `roles`. Without them +an authenticated capture would write whoever took it, and whoever else can reach the +note, into the fixture. A permission set is masked entry by entry, so the array keeps its +shape and its count. The text rules leave `user=` and `owners=` alone, because both are +ordinary in a url and in note text. + +`noteId` and `paragraphId` are kept as captured, preserving references across URLs, +bodies and frames. This is a v1 reproducibility tradeoff: recapturing the same flow +can produce a different file. A future bijective ID mapping could remove that churn +without losing references; it is not required for replaying one captured trace. + +They are stable within a fixture but not between captures, and a Zeppelin paragraph id +carries the creation time in it (`paragraph_1757...`), so re-capturing a scenario +produces a textually different fixture even when the contract has not changed. A future +version could map each id to a numbered placeholder consistently and get diff-clean +re-captures without losing the reference; version 1 does not, and a re-capture is +reviewed as a new recording rather than as a diff. + +### What redaction reaches, and what it does not + +Redaction acts on field names first. A name counts as sensitive when one of its words +is a sensitive word, or when it ends in one, or - for a name written in one uppercase +run, as environment variables are - when it merely contains one. Names are split on +separators and on camel case, so `accessToken`, `secretKey`, `aws_secret_access_key`, +`spark.hadoop.fs.s3a.secret.key`, `x-api-key`, `PGPASSWORD`, `SECRETKEY`, +`private_key` and `passphrase` all count, while `tokenizer`, `secretary`, `tokens`, +`max_tokens`, `privately` and `keyboard` do not. A name outside the list is not masked +- `pwd`, `bearer` and `sessionId` are not - so add the word rather than relying on the +shape of the value. A name in mixed case with no separator, such as `SECRETkey`, falls +between the rules and is missed. + +Two fields carry no inner structure for that to work on, and only those two are +scanned as text: `url`, whose credential sits in the query under no name of its own, +and `bodyRaw`, which arrives as one opaque string. There the rules look for +`name=value` and `name: value`, a url's `user:password@host`, and a header credential +that runs to the end of its line. + +A WebSocket frame picks its own treatment. A frame that parses as JSON is redacted by +key, like any other record; a frame that does not is text-scanned as one string, the +same way `bodyRaw` is. Scanning the JSON as text on top of the key pass would rewrite +the note inside it and leave a capture the fixture could no longer replay. + +**Everything else keeps its text.** A json field, and a string inside an array, are +covered by the name they sit under, so the value is left as written. This is deliberate: the text rules cannot tell a credential +from prose that mentions one, and a note is full of prose that does. Scanning note +text rewrote `ticketCount = df.count()`, `SELECT ... WHERE ticket_id = 42` and +`const cookieBanner = document.getElementById("x")`, which is a worse outcome for a +fixture than the narrow gap it closed. The capture helper starts an isolated, empty +server for the same reason: a fixture's note text is written by the test that captured +it, not by whoever owns the machine. + +What the text rules still do not reach, where they do scan: an unquoted +value containing a space keeps its tail; a `name: value` pair is only matched where the +name opens an unindented line, a quoted string, an object or a JSON array entry, so an +indented yaml key and a `- ` list item are left as written; a url's userinfo is only +matched in its `user:password@host` form, so `token@host` is left alone; a name and its +value split across two fields, as in `{"name": "password", "value": "hunter2"}`, has no +name beside the value to match on; `password==abc` is read as a comparison rather than +an assignment; and a credential that carries no name at all - a bare token, a base64 +blob - is invisible to every rule here. + +In text a purely numeric value is left alone for names ending in `principal`, because +that word is also an accounting term. A field named `principal` is masked whatever it +holds. + +Do not rely on any of this for a fixture captured from a server holding real +credentials. Capture from the isolated server this directory starts. diff --git a/zeppelin-web-angular/e2e/core-contract/capture-server.sh b/zeppelin-web-angular/e2e/core-contract/capture-server.sh new file mode 100755 index 00000000000..b9b8f975d6a --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/capture-server.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +usage() { + echo "usage: $0 start|stop --root [--mode anonymous|auth] [--port ]" >&2 +} + +command="${1:-}" +shift || true +capture_root="" +capture_mode="anonymous" +zeppelin_port="8080" +port_given="no" + +while [[ $# -gt 0 ]]; do + case "$1" in + --root) + capture_root="${2:-}" + shift 2 + ;; + --mode) + capture_mode="${2:-}" + shift 2 + ;; + --port) + zeppelin_port="${2:-}" + port_given="yes" + # Stop reuses this port from the marker, so reject invalid values now. + if [[ ! "${zeppelin_port}" =~ ^[0-9]+$ ]]; then + echo "--port must be a number, got '${zeppelin_port}'" >&2 + exit 2 + fi + shift 2 + ;; + *) + usage + exit 2 + ;; + esac +done + +if [[ -z "${command}" || -z "${capture_root}" ]]; then + usage + exit 2 +fi + +reject_whitespace_path() { + if [[ "$1" =~ [[:space:]] ]]; then + echo "$2 path must not contain whitespace: the Zeppelin launcher splits JVM arguments" >&2 + exit 2 + fi +} + +# Validate before creating directories or locking. +# Resolve an existing ancestor so symlinks cannot hide unsupported paths. +reject_whitespace_path "${capture_root}" "capture root" +existing_parent="${capture_root}" +while [[ ! -d "${existing_parent}" ]]; do + existing_parent="$(dirname "${existing_parent}")" +done +# The suffix preserves trailing newlines in directory names during substitution. +physical_parent="$(cd -P "${existing_parent}" && printf '%s/.' "$PWD")" +reject_whitespace_path "${physical_parent}" "canonical capture root" +repo_root="$(cd -P "$(dirname "$0")/../../.." && printf '%s/.' "$PWD")" +reject_whitespace_path "${repo_root}" "repository" +repo_root="${repo_root%/.}" +# Use the physical path so symlink and direct access produce the same marker. +capture_root="$(mkdir -p "${capture_root}" && cd "${capture_root}" && pwd -P)" +capture_marker="-Dzeppelin.capture.root=${capture_root}" +marker_file="${capture_root}/.zeppelin-capture-root" +zeppelin_pid_file="${capture_root}/zeppelin.pid" +operation_lock="${capture_root}/.capture-operation-lock" + +acquire_operation_lock() { + # Lock atomically before inspecting ownership; hold through readiness and cleanup. + if ! mkdir "${operation_lock}" 2>/dev/null; then + echo "another capture server operation is in progress for ${capture_root}" >&2 + echo "if its owner crashed, verify no operation is running before removing ${operation_lock}" >&2 + exit 1 + fi + trap 'rmdir "${operation_lock}"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM +} + +port_in_use() { + local port="$1" + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + return + fi + if command -v ss >/dev/null 2>&1; then + ss -ltn "sport = :${port}" 2>/dev/null | grep -q LISTEN + return + fi + (exec 3<>"/dev/tcp/127.0.0.1/${port}") >/dev/null 2>&1 +} + +write_marker() { + { + echo "root=${capture_root}" + echo "repo=${repo_root}" + echo "port=${zeppelin_port}" + } > "${marker_file}" +} + +verify_root_marker() { + # Match paths literally, including regex metacharacters. + [[ -f "${marker_file}" ]] && grep -qxF "root=${capture_root}" "${marker_file}" +} + +verify_pid_identity() { + local pid="$1" + local expected="$2" + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + local command + command="$(ps -p "${pid}" -o command= 2>/dev/null)" || return 1 + # Match a whole argument so /tmp/capture cannot claim /tmp/capture-other. + [[ " ${command} " == *" ${expected} "* ]] +} + +group_members() { + local pid="$1" + if command -v pgrep >/dev/null 2>&1; then + pgrep -g "${pid}" 2>/dev/null + else + ps -A -o pid=,pgid= 2>/dev/null | awk -v group="${pid}" '$2 == group { print $1 }' + fi +} + +# Check the group so surviving children still trigger KILL escalation or failure. +process_group_alive() { + local pid="$1" + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + [[ -n "$(group_members "${pid}")" ]] && return 0 + ps -p "${pid}" >/dev/null 2>&1 +} + +# If the leader has exited, require a surviving group member with our marker. +verify_group_identity() { + local pid="$1" + local expected="$2" + local member + for member in $(group_members "${pid}"); do + if verify_pid_identity "${member}" "${expected}"; then + return 0 + fi + done + verify_pid_identity "${pid}" "${expected}" +} + +signal_process_group() { + local pid="$1" + local signal="$2" + # Signal the group to reach servers launched through shell wrappers. + kill "-${signal}" "-${pid}" 2>/dev/null || kill "-${signal}" "${pid}" 2>/dev/null || true +} + +stop_pid() { + local pid_file="$1" + local expected="$2" + [[ -f "${pid_file}" ]] || return 0 + local pid + pid="$(cat "${pid_file}")" + if process_group_alive "${pid}"; then + if ! verify_group_identity "${pid}" "${expected}"; then + echo "refusing to stop ${pid}: command does not match ${expected}" >&2 + echo "the recorded pid belongs to another process; start on this root again to clear it, or remove ${pid_file}" >&2 + exit 1 + fi + signal_process_group "${pid}" TERM + for _ in {1..20}; do + process_group_alive "${pid}" || break + sleep 1 + done + if process_group_alive "${pid}"; then + signal_process_group "${pid}" KILL + for _ in {1..10}; do + process_group_alive "${pid}" || break + sleep 1 + done + fi + if process_group_alive "${pid}"; then + echo "failed to stop ${pid}; keeping ${pid_file} so it can be retried" >&2 + exit 1 + fi + fi + rm -f "${pid_file}" +} + +start_zeppelin() { + # Environment and JVM properties override the temporary site XML. + # Clear inherited settings that could redirect storage, classpaths or remote connections. + # Keep JAVA_HOME and PATH to select the installed toolchain. + local inherited_name + for inherited_name in "${!ZEPPELIN_@}"; do + unset "${inherited_name}" + done + unset JAVA_OPTS JAVA_TOOL_OPTIONS _JAVA_OPTIONS JDK_JAVA_OPTIONS CLASSPATH + + mkdir -p "${capture_root}/conf" "${capture_root}/notebook" "${capture_root}/index" \ + "${capture_root}/logs" "${capture_root}/run" "${capture_root}/recovery" "${capture_root}/webapps" + cp "${repo_root}/conf/log4j2.properties" "${capture_root}/conf/log4j2.properties" + cp "${repo_root}/conf/zeppelin-site.xml.template" "${capture_root}/conf/zeppelin-site.xml" + if [[ "${capture_mode}" == "auth" ]]; then + cp "${repo_root}/conf/shiro.ini.template" "${capture_root}/conf/shiro.ini" + else + rm -f "${capture_root}/conf/shiro.ini" + fi + + export ZEPPELIN_CONF_DIR="${capture_root}/conf" + export ZEPPELIN_ADDR="127.0.0.1" + export ZEPPELIN_NOTEBOOK_STORAGE="org.apache.zeppelin.notebook.repo.VFSNotebookRepo" + export ZEPPELIN_NOTEBOOK_DIR="${capture_root}/notebook" + export ZEPPELIN_LOG_DIR="${capture_root}/logs" + export ZEPPELIN_PID_DIR="${capture_root}/run" + export ZEPPELIN_WAR_TEMPDIR="${capture_root}/webapps" + # Zeppelin ignores this marker; verify_pid_identity matches it as a whole JVM argument. + export ZEPPELIN_JAVA_OPTS="-Dzeppelin.server.port=${zeppelin_port} -Dzeppelin.notebook.dir=${capture_root}/notebook -Dzeppelin.search.index.path=${capture_root}/index -Dzeppelin.recovery.dir=${capture_root}/recovery ${capture_marker}" + export ZEPPELIN_CAPTURE_ROOT="${capture_root}" + export ZEPPELIN_PORT="${zeppelin_port}" + # Do not inherit Hadoop settings for this fixture server. + export USE_HADOOP=false + + # Give the server its own process group so stop can signal its children too. + set -m + # Tests substitute a stub command and append the same ownership marker as a real server. + # The stub ignores the extra argument. + if [[ -n "${CAPTURE_ZEPPELIN_COMMAND:-}" ]]; then + # Preserve the marker through bash -c parsing, including quotes and dollar signs. + bash -c "${CAPTURE_ZEPPELIN_COMMAND} $(printf '%q' "${capture_marker}")" "${capture_root}/logs/zeppelin-stdout.log" 2>"${capture_root}/logs/zeppelin-stderr.log" & + echo "$!" > "${zeppelin_pid_file}" + else + "${repo_root}/bin/zeppelin.sh" "${capture_root}/logs/zeppelin-stdout.log" 2>"${capture_root}/logs/zeppelin-stderr.log" & + echo "$!" > "${zeppelin_pid_file}" + fi + set +m +} + +wait_for_http() { + local url="$1" + local pid="${2:-}" + for _ in {1..120}; do + # Bound each request so a listener that never responds cannot hang startup. + if curl -fsS --max-time 5 "${url}" >/dev/null 2>&1; then + return 0 + fi + # Report an exited server immediately instead of masking its error with a timeout. + if [[ -n "${pid}" ]] && ! process_group_alive "${pid}"; then + return 2 + fi + sleep 1 + done + return 1 +} + +# Verify listener ownership after binding; another server may take a previously free port. +verify_port_owner() { + local port="$1" + local expected="$2" + local owner + command -v lsof >/dev/null 2>&1 || return 1 + for owner in $(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null); do + if verify_group_identity "${owner}" "${expected}"; then + return 0 + fi + done + return 1 +} + +start_server() { + if ! command -v lsof >/dev/null 2>&1; then + echo "lsof is required to verify capture server listener ownership" >&2 + exit 1 + fi + if [[ "${capture_mode}" != "anonymous" && "${capture_mode}" != "auth" ]]; then + echo "mode must be anonymous or auth" >&2 + exit 2 + fi + if [[ -f "${zeppelin_pid_file}" ]]; then + local recorded_pid + recorded_pid="$(cat "${zeppelin_pid_file}")" + if [[ "${recorded_pid}" == "starting" ]]; then + echo "capture startup is incomplete; inspect ${capture_root} before removing ${zeppelin_pid_file}" >&2 + exit 1 + fi + # Preserve a live capture's PID file so a failed stop can be retried. + # Clear recycled PIDs belonging to unrelated processes so the root remains reusable. + if process_group_alive "${recorded_pid}" && verify_group_identity "${recorded_pid}" "${capture_marker}"; then + echo "capture server ${recorded_pid} is still running for ${capture_root}; stop it first" >&2 + exit 1 + fi + rm -f "${zeppelin_pid_file}" + fi + if port_in_use "${zeppelin_port}"; then + echo "port ${zeppelin_port} is already in use" >&2 + exit 1 + fi + + # Keep a visible claim until start_zeppelin records the process group leader. + if ! (set -o noclobber; echo "starting" > "${zeppelin_pid_file}") 2>/dev/null; then + echo "another capture server start is already in progress for ${capture_root}" >&2 + exit 1 + fi + + write_marker + start_zeppelin + if [[ "${capture_mode}" == "auth" ]]; then + # Direct the login helper to this capture's credentials instead of the repository config. + echo "shiro config: ${capture_root}/conf/shiro.ini" + echo "export ZEPPELIN_E2E_SHIRO_INI=${capture_root}/conf/shiro.ini to log the e2e helper into this server" + fi + local recorded_pid + recorded_pid="$(cat "${zeppelin_pid_file}" 2>/dev/null)" + # Capture failure status without triggering set -e before cleanup. + local ready=0 + wait_for_http "http://127.0.0.1:${zeppelin_port}/api/version" "${recorded_pid}" || ready=$? + if [[ ${ready} -ne 0 ]]; then + # The claim must not outlive a start that never came up. + if [[ ${ready} -eq 2 ]]; then + echo "zeppelin exited before it answered on port ${zeppelin_port}" >&2 + else + echo "zeppelin did not become ready on port ${zeppelin_port}" >&2 + fi + echo "--- ${capture_root}/logs/zeppelin-stderr.log (tail) ---" >&2 + tail -n 20 "${capture_root}/logs/zeppelin-stderr.log" >&2 2>/dev/null || true + stop_server + exit 1 + fi + if ! verify_port_owner "${zeppelin_port}" "${capture_marker}"; then + echo "port ${zeppelin_port} is answered by a server this root did not start" >&2 + stop_server + exit 1 + fi +} + +stop_server() { + if ! verify_root_marker; then + echo "refusing to stop without matching capture root marker: ${marker_file}" >&2 + exit 1 + fi + # Use the recorded port so an unrelated listener on the default port cannot fail stop. + if [[ "${port_given}" == "no" ]]; then + local recorded_port + recorded_port="$(sed -n 's/^port=//p' "${marker_file}")" + if [[ "${recorded_port}" =~ ^[0-9]+$ ]]; then + zeppelin_port="${recorded_port}" + fi + fi + stop_pid "${zeppelin_pid_file}" "${capture_marker}" + # Allow wrapped servers time to release the port after their wrapper exits. + for _ in {1..10}; do + port_in_use "${zeppelin_port}" || return 0 + sleep 1 + done + echo "port ${zeppelin_port} is still in use after stop; another process may hold it" >&2 + exit 1 +} + +case "${command}" in + start) + acquire_operation_lock + start_server + ;; + stop) + acquire_operation_lock + stop_server + ;; + *) + usage + exit 2 + ;; +esac diff --git a/zeppelin-web-angular/e2e/core-contract/capture-server.test.mjs b/zeppelin-web-angular/e2e/core-contract/capture-server.test.mjs new file mode 100644 index 00000000000..6cbce2c31cb --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/capture-server.test.mjs @@ -0,0 +1,527 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync +} from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const script = path.resolve('e2e/core-contract/capture-server.sh'); +const stub = path.resolve('e2e/core-contract/capture-stub-zeppelin.mjs'); + +const temporaryRoots = []; +process.on('exit', () => { + for (const root of temporaryRoots) { + rmSync(root, { force: true, recursive: true }); + } +}); + +for (const [name, whitespace] of [ + ['space', ' '], + ['tab', '\t'], + ['newline', '\n'] +]) { + test(`capture server rejects ${name} in a root before creating files or launching`, () => { + const parent = createRoot(); + const root = path.join(parent.root, `capture${whitespace}root`); + const launched = path.join(parent.root, 'launched'); + const result = run(['start', '--root', root, '--port', String(parent.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `touch '${launched}'` + }); + assert.equal(result.status, 2, result.stderr); + assert.match(result.stderr, /path must not contain whitespace/); + assert.equal(existsSync(root), false); + assert.equal(existsSync(launched), false); + }); +} + +for (const existing of [false, true]) { + test(`capture server rejects a canonical whitespace root (existing=${existing}) before side effects`, () => { + const parent = createRoot(); + const target = path.join(parent.root, 'directory with spaces'); + mkdirSync(target); + const alias = path.join(parent.root, 'alias'); + symlinkSync(target, alias); + const root = existing ? alias : path.join(alias, 'new-root'); + const launched = path.join(parent.root, 'launched'); + const result = run(['start', '--root', root, '--port', String(parent.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `touch '${launched}'` + }); + assert.equal(result.status, 2, result.stderr); + assert.match(result.stderr, /path must not contain whitespace/); + assert.equal(existsSync(path.join(target, 'new-root')), false); + assert.equal(existsSync(path.join(target, '.capture-operation-lock')), false); + assert.equal(existsSync(path.join(target, '.zeppelin-capture-root')), false); + assert.equal(existsSync(launched), false); + }); +} + +test('capture server rejects a repository path with whitespace before creating the root', () => { + const parent = createRoot(); + const directory = path.join(parent.root, 'repo with spaces', 'zeppelin-web-angular', 'e2e', 'core-contract'); + mkdirSync(directory, { recursive: true }); + const copy = path.join(directory, 'capture-server.sh'); + writeFileSync(copy, readFileSync(script)); + const root = path.join(parent.root, 'capture'); + const result = spawnSync('bash', [copy, 'start', '--root', root], { encoding: 'utf8' }); + assert.equal(result.status, 2, result.stderr); + assert.match(result.stderr, /repository path must not contain whitespace/); + assert.equal(existsSync(root), false); +}); + +for (const action of ['start', 'stop']) { + test(`capture server refuses concurrent ${action} while startup owns the root`, async () => { + const root = createRoot(); + const bin = path.join(root.root, 'bin'); + mkdirSync(bin); + const gate = path.join(root.root, 'gate'); + assert.equal(spawnSync('mkfifo', [gate]).status, 0); + writeFileSync( + path.join(bin, 'mkdir'), + `#!/bin/bash +if [[ "$2" == */conf ]]; then + echo CAPTURE_TEST_BARRIER + read -r release < '${gate}' +fi +exec /bin/mkdir "$@" +`, + { mode: 0o755 } + ); + const first = spawn('bash', [script, 'start', '--root', root.root, '--port', String(root.zeppelinPort)], { + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + let errors = ''; + let barrierReached = false; + let startStatus; + let stopped; + first.stderr.on('data', chunk => { + errors += chunk; + }); + const finished = new Promise(resolve => first.on('exit', resolve)); + try { + await new Promise((resolve, reject) => { + let output = ''; + const timeout = setTimeout(() => reject(new Error(`startup barrier timed out: ${errors}`)), 10000); + first.stdout.on('data', chunk => { + output += chunk; + if (output.includes('CAPTURE_TEST_BARRIER')) { + clearTimeout(timeout); + barrierReached = true; + resolve(); + } + }); + }); + const result = run([action, '--root', root.root, '--port', String(freePortSync())], { + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + assert.equal(result.status, 1, `concurrent ${action} succeeded: ${result.stdout}`); + assert.match(result.stderr, /in progress/); + assert.equal(readFileSync(path.join(root.root, 'zeppelin.pid'), 'utf8').trim(), 'starting'); + } finally { + if (barrierReached) writeFileSync(gate, 'release\n'); + else first.kill('SIGKILL'); + startStatus = await finished; + stopped = run(['stop', '--root', root.root]); + // A regressed concurrent start can strand a second server outside the PID file. + const survivors = spawnSync('pgrep', ['-f', marker(root.root)], { encoding: 'utf8' }); + for (const pid of survivors.stdout.trim().split('\n').filter(Boolean)) { + try { + process.kill(Number(pid), 'SIGKILL'); + } catch { + /* already exited */ + } + } + } + assert.equal(startStatus, 0, errors); + assert.equal(stopped.status, 0, stopped.stderr); + }); +} + +test('capture server refuses startup when listener ownership cannot be verified', () => { + const root = createRoot(); + const environment = path.join(root.root, 'bash-env'); + writeFileSync( + environment, + `command() { + if [[ "$1" == -v && "$2" == lsof ]]; then return 1; fi + builtin command "$@" +} +` + ); + try { + const result = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + BASH_ENV: environment, + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + assert.equal(result.status, 1, result.stdout); + assert.match(result.stderr, /lsof.*required/); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), false); + } finally { + if (existsSync(path.join(root.root, 'zeppelin.pid'))) stop(root); + } +}); + +test('capture server replaces inherited storage, binding and JVM configuration', () => { + const root = createRoot(); + const probe = path.join(root.root, 'probe.mjs'); + const observed = path.join(root.root, 'environment.json'); + const inherited = { + ZEPPELIN_NOTEBOOK_STORAGE: 'external.NotebookRepo', + ZEPPELIN_ADDR: '0.0.0.0', + ZEPPELIN_SEARCH_INDEX_PATH: '/outside/index', + ZEPPELIN_RECOVERY_DIR: '/outside/recovery', + ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL: 'https://example.invalid/notebooks.git', + ZEPPELIN_JAVA_OPTS: '-Dzeppelin.notebook.storage=external.NotebookRepo', + JAVA_OPTS: '-Dzeppelin.search.index.path=/outside/index', + JAVA_TOOL_OPTIONS: '-Dzeppelin.recovery.dir=/outside/recovery', + _JAVA_OPTIONS: '-Dzeppelin.server.addr=0.0.0.0', + JDK_JAVA_OPTIONS: '-Dzeppelin.notebook.storage=external.NotebookRepo', + CLASSPATH: '/outside/classes' + }; + writeFileSync( + probe, + `import { writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(observed)}, JSON.stringify(Object.fromEntries( + ${JSON.stringify(Object.keys(inherited))}.map(key => [key, process.env[key] ?? null]) +))); +await import(${JSON.stringify(stub)}); +` + ); + const result = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + ...inherited, + CAPTURE_ZEPPELIN_COMMAND: `node ${probe}` + }); + try { + assert.equal(result.status, 0, result.stderr); + const env = JSON.parse(readFileSync(observed, 'utf8')); + assert.equal(env.ZEPPELIN_NOTEBOOK_STORAGE, 'org.apache.zeppelin.notebook.repo.VFSNotebookRepo'); + assert.equal(env.ZEPPELIN_ADDR, '127.0.0.1'); + for (const key of Object.keys(inherited).filter( + key => !['ZEPPELIN_NOTEBOOK_STORAGE', 'ZEPPELIN_ADDR', 'ZEPPELIN_JAVA_OPTS'].includes(key) + )) { + assert.equal(env[key], null, `${key} escaped environment isolation`); + } + assert.ok(env.ZEPPELIN_JAVA_OPTS.includes(`-Dzeppelin.search.index.path=${root.root}/index`)); + assert.ok(!env.ZEPPELIN_JAVA_OPTS.includes('external.NotebookRepo')); + } finally { + if (existsSync(path.join(root.root, 'zeppelin.pid'))) stop(root); + } +}); + +test('capture-server starts and stops a server in its own root', () => { + const root = createRoot(); + + start(root); + stop(root); + + assert.equal(existsSync(path.join(root.root, '.zeppelin-capture-root')), true); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), false); +}); + +test('capture-server writes anonymous and auth config in an isolated temp root', () => { + const anonymous = createRoot(); + const auth = createRoot(); + + start(anonymous); + stop(anonymous); + start(auth, { mode: 'auth' }); + stop(auth); + + assert.equal(existsSync(path.join(anonymous.root, 'conf/shiro.ini')), false); + assert.equal(existsSync(path.join(auth.root, 'conf/shiro.ini')), true); +}); + +test('capture-server reports explicit port conflicts', async () => { + const server = await listen(); + const root = createRoot(); + + try { + const result = run(['start', '--root', root.root, '--port', String(server.address().port)]); + + assert.equal(result.status, 1); + assert.match(result.stderr, /port .* is already in use/); + } finally { + await close(server); + } +}); + +test('capture server does not inherit a Hadoop setting from the developer shell', () => { + const root = createRoot(); + const result = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `test "$USE_HADOOP" = false && node ${stub}`, + USE_HADOOP: 'true' + }); + assert.equal(result.status, 0, result.stderr); + stop(root); +}); + +test('capture server stops a compound command without orphaning the real server', () => { + const root = createRoot(); + // Killing only the wrapper would leave its server holding the port. + const started = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `test "$USE_HADOOP" = false && node ${stub}` + }); + assert.equal(started.status, 0, started.stderr); + + const stopped = run(['stop', '--root', root.root, '--port', String(root.zeppelinPort)]); + assert.equal(stopped.status, 0, stopped.stderr); + + const survivors = spawnSync('pgrep', ['-f', `-Dzeppelin.capture.root=${root.root}`], { encoding: 'utf8' }); + assert.equal(survivors.error, undefined, 'pgrep has to run for this assertion to mean anything'); + assert.equal((survivors.stdout ?? '').trim(), '', 'the wrapped server must not survive stop'); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), false, 'a clean stop removes the pid file'); +}); + +test('auth mode reports the shiro config the login helper must use', () => { + const root = createRoot(); + const started = run(['start', '--root', root.root, '--mode', 'auth', '--port', String(root.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + assert.equal(started.status, 0, started.stderr); + + // The login helper needs this capture's shiro.ini via ZEPPELIN_E2E_SHIRO_INI. + const expected = path.join(root.root, 'conf', 'shiro.ini'); + assert.ok(existsSync(expected), 'auth mode must install shiro.ini in the capture root'); + assert.match(started.stdout, new RegExp(expected.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); + assert.notEqual(expected, path.resolve('..', 'conf', 'shiro.ini')); + + stop(root); +}); + +test('anonymous mode reports no shiro config and installs none', () => { + const root = createRoot(); + const started = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + assert.equal(started.status, 0, started.stderr); + assert.equal(existsSync(path.join(root.root, 'conf', 'shiro.ini')), false); + assert.doesNotMatch(started.stdout, /shiro config/); + + stop(root); +}); + +test('capture server start clears a stale pid file instead of refusing forever', () => { + const root = createRoot(); + // PID 1 represents a live, unrelated process whose stale record must not block reuse. + writeFileSync(path.join(root.root, 'zeppelin.pid'), '1'); + + const started = run(['start', '--root', root.root, '--port', String(root.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + assert.equal(started.status, 0, started.stderr); + stop(root); +}); + +test('capture server refuses to stop a server whose root only shares a prefix', () => { + const root = createRoot(); + markRoot(root); + // A shared path prefix must not grant ownership of another capture's process. + const decoy = spawnMarked(root, `${root.root}-other`); + try { + writeFileSync(path.join(root.root, 'zeppelin.pid'), String(decoy.pid)); + const result = run(['stop', '--root', root.root, '--port', String(root.zeppelinPort)]); + + assert.equal(result.status, 1, result.stdout); + assert.match(result.stderr, /command does not match/); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), true, 'the pid file has to survive a refusal'); + assert.equal(alive(decoy.pid), true, "the other capture's server was killed"); + } finally { + decoy.kill('SIGKILL'); + } +}); + +test('capture server stops a surviving child after its process leader is gone', async () => { + const root = createRoot(); + markRoot(root); + // The surviving child must be stopped even after its group leader exits. + const leader = spawn( + 'bash', + ['-c', `'${process.execPath}' '${childScript(root)}' '${marker(root.root)}' & echo $! > '${root.root}/child.pid'`], + { + detached: true, + stdio: 'ignore' + } + ); + const leaderPid = leader.pid; + await new Promise(resolve => leader.on('exit', resolve)); + const childPid = Number(readFileSync(path.join(root.root, 'child.pid'), 'utf8').trim()); + assert.equal(alive(childPid), true, 'the child has to outlive its leader for this test to mean anything'); + + try { + writeFileSync(path.join(root.root, 'zeppelin.pid'), String(leaderPid)); + const result = run(['stop', '--root', root.root, '--port', String(root.zeppelinPort)]); + + assert.equal(result.status, 0, result.stderr); + assert.equal(alive(childPid), false, 'stop left the child running'); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), false); + } finally { + try { + process.kill(childPid, 'SIGKILL'); + } catch { + // Already exited. + } + } +}); + +const marker = root => `-Dzeppelin.capture.root=${root}`; + +// Use a script file so Node treats the marker as an argument, not an option to node -e. +function childScript(root) { + const file = path.join(root.root, 'stay-alive.mjs'); + writeFileSync(file, 'setTimeout(() => {}, 60000);\n'); + return file; +} + +function spawnMarked(root, markerRoot) { + return spawn(process.execPath, [childScript(root), marker(markerRoot)], { stdio: 'ignore' }); +} + +function markRoot(root) { + writeFileSync(path.join(root.root, '.zeppelin-capture-root'), `root=${root.root}\n`); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test('capture server refuses a second start while the first is still running', () => { + const root = createRoot(); + const second = { root: root.root, zeppelinPort: freePortSync() }; + start(root); + const recorded = readFileSync(path.join(root.root, 'zeppelin.pid'), 'utf8').trim(); + + try { + // A different port isolates the PID ownership check from port conflict detection. + const result = run(['start', '--root', second.root, '--port', String(second.zeppelinPort)], { + CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` + }); + + assert.equal(result.status, 1, result.stdout); + assert.match(result.stderr, /still running/); + assert.equal( + readFileSync(path.join(root.root, 'zeppelin.pid'), 'utf8').trim(), + recorded, + 'the pid file was overwritten' + ); + } finally { + stop(root); + } +}); + +test('capture server stops a root started on another port without being told the port', () => { + const root = createRoot(); + // Stop must use the recorded port, not the default port an unrelated process may hold. + start(root); + + const result = run(['stop', '--root', root.root]); + + assert.equal(result.status, 0, result.stderr); + assert.equal(existsSync(path.join(root.root, 'zeppelin.pid')), false); +}); + +function start(root, options = {}) { + const result = run( + ['start', '--root', root.root, '--mode', options.mode ?? 'anonymous', '--port', String(root.zeppelinPort)], + { CAPTURE_ZEPPELIN_COMMAND: `node ${stub}` } + ); + assert.equal(result.status, 0, result.stderr); +} + +function stop(root) { + const result = run(['stop', '--root', root.root, '--port', String(root.zeppelinPort)]); + assert.equal(result.status, 0, result.stderr); +} + +function run(args, env = {}) { + return spawnSync('bash', [script, ...args], { + cwd: path.resolve('.'), + encoding: 'utf8', + env: { ...process.env, ...env } + }); +} + +function createRoot() { + const created = mkdtempSync(path.join(os.tmpdir(), 'zeppelin-capture-')); + temporaryRoots.push(created); + // Match pwd -P when writing markers; macOS temporary directories may resolve through symlinks. + return { root: realpathSync(created), zeppelinPort: freePortSync() }; +} + +function freePortSync() { + const result = spawnSync( + process.execPath, + [ + '-e', + "require('net').createServer().listen(0, '127.0.0.1', function () { console.log(this.address().port); this.close(); })" + ], + { + encoding: 'utf8' + } + ); + return Number(result.stdout.trim()); +} + +function listen() { + return new Promise(resolve => { + const server = http.createServer(); + server.listen(0, '127.0.0.1', () => resolve(server)); + }); +} + +function close(server) { + return new Promise(resolve => server.close(resolve)); +} + +test('capture server retains the PID claim when termination cannot stop its process', async () => { + const root = createRoot(); + markRoot(root); + const child = spawnMarked(root, root.root); + const exited = new Promise(resolve => child.once('exit', resolve)); + const pidFile = path.join(root.root, 'zeppelin.pid'); + const shellEnv = path.join(root.root, 'no-signals.sh'); + // Disable signals and polling delays, but use real ps to verify the owned process remains alive. + writeFileSync(shellEnv, 'kill() { return 0; }\nsleep() { :; }\n'); + writeFileSync(pidFile, String(child.pid)); + try { + const result = run(['stop', '--root', root.root], { BASH_ENV: shellEnv }); + assert.notEqual(result.status, 0, 'stop must fail when its process remains alive'); + assert.match(result.stderr, /failed to stop/); + assert.equal(readFileSync(pidFile, 'utf8'), String(child.pid)); + assert.equal(alive(child.pid), true); + } finally { + child.kill('SIGKILL'); + await exited; + } +}); diff --git a/zeppelin-web-angular/e2e/core-contract/capture-stub-zeppelin.mjs b/zeppelin-web-angular/e2e/core-contract/capture-stub-zeppelin.mjs new file mode 100755 index 00000000000..2087e7483f0 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/capture-stub-zeppelin.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import crypto from 'node:crypto'; +import http from 'node:http'; + +const port = Number(process.env.ZEPPELIN_PORT); +const root = process.env.ZEPPELIN_CAPTURE_ROOT; + +if (!port || !root) { + process.stderr.write('ZEPPELIN_PORT and ZEPPELIN_CAPTURE_ROOT are required\n'); + process.exit(2); +} + +const server = http.createServer((request, response) => { + if (request.url === '/api/version') { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end('{"version":"stub"}'); + return; + } + response.writeHead(200, { 'content-type': 'application/json' }); + response.end('{"id":"note-a","paragraphs":[{"id":"paragraph-a"}]}'); +}); + +server.on('upgrade', (request, socket) => { + const key = request.headers['sec-websocket-key']; + const accept = crypto.createHash('sha1').update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest('base64'); + socket.write( + [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Accept: ${accept}`, + '', + '' + ].join('\r\n') + ); + socket.on('data', () => { + const payload = Buffer.from('{"op":"NOTE"}'); + socket.write(Buffer.concat([Buffer.from([0x81, payload.length]), payload])); + }); +}); + +server.listen(port, '127.0.0.1'); +process.on('SIGTERM', () => server.close(() => process.exit(0))); diff --git a/zeppelin-web-angular/e2e/core-contract/fixture-doubles.d.mts b/zeppelin-web-angular/e2e/core-contract/fixture-doubles.d.mts new file mode 100644 index 00000000000..5d13e65be56 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/fixture-doubles.d.mts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { FixtureMetadata, FixtureRecord, FixtureRequestLike, FixtureRest } from './notebook-transport-fixture.mjs'; + +export declare const fixtureMetadata: () => FixtureMetadata; + +export declare const request: ( + method: string, + url: string, + body?: string, + headers?: Record +) => FixtureRequestLike; + +export declare const response: ( + sourceRequest: FixtureRequestLike, + status: number, + body: string | (() => Promise), + headers?: Record +) => { + headers: () => Record; + request: () => FixtureRequestLike; + status: () => number; + text: () => Promise; +}; + +export declare const wsRecord: (sequence: number, direction: string, payloadText: string) => FixtureRecord; diff --git a/zeppelin-web-angular/e2e/core-contract/fixture-doubles.mjs b/zeppelin-web-angular/e2e/core-contract/fixture-doubles.mjs new file mode 100644 index 00000000000..ffd268e0a60 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/fixture-doubles.mjs @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Shared record builders for Node and Playwright tests. + +export const fixtureMetadata = () => ({ + coveredOperations: ['GET_NOTE'], + knownExclusions: [], + owner: 'zeppelin-web-angular', + scenario: 'Notebook transport fixture test' +}); + +export const request = (method, url, body = '', headers = { accept: 'application/json' }) => ({ + headers: () => headers, + method: () => method, + postData: () => body, + url: () => url +}); + +export const response = (sourceRequest, status, body, headers = { 'content-type': 'application/json' }) => ({ + headers: () => headers, + request: () => sourceRequest, + status: () => status, + text: async () => (typeof body === 'function' ? body() : body) +}); + +export const wsRecord = (sequence, direction, payloadText) => ({ + kind: 'websocket', + sequence, + websocket: { direction, payloadText } +}); diff --git a/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.d.mts b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.d.mts new file mode 100644 index 00000000000..3ba3ce406d3 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.d.mts @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// validateFixture checks record shapes at runtime; callers use optional field access. + +export interface FixtureRestRequest { + bodyJson?: unknown; + bodyRaw?: string; + headers: Record; + method: string; + url: string; +} + +export interface FixtureRest { + bodyJson?: unknown; + bodyRaw?: string; + direction: string; + headers?: Record; + request: FixtureRestRequest; + status?: number; +} + +export interface FixtureWebSocket { + direction: string; + payloadBase64?: string; + payloadText?: string; +} + +export interface FixtureRecord { + kind: string; + rest?: FixtureRest; + sequence: number; + websocket?: FixtureWebSocket; +} + +export interface FixtureMetadata { + coveredOperations: string[]; + knownExclusions: string[]; + owner: string; + scenario: string; + // Recorded for provenance; neither is read or validated by this module. + capturedAt?: string; + zeppelinVersion?: string; +} + +export interface TransportFixture { + metadata?: FixtureMetadata; + records: FixtureRecord[]; + version: number; +} + +// Minimal interfaces shared by Playwright objects and test doubles. +// Untyped callbacks allow both implementations. +/* eslint-disable @typescript-eslint/no-explicit-any */ +export interface FixtureRouteLike { + fallback?: (...args: any[]) => any; + fulfill: (...args: any[]) => any; +} + +export interface FixtureRequestLike { + headers: () => Record; + method: () => string; + postData: () => string | null; + url: () => string; +} + +// Require only the page methods each adapter uses. +export interface RecorderPageLike { + on: (...args: any[]) => any; +} + +export interface ReplayPageLike { + on?: (...args: any[]) => any; + route: (...args: any[]) => any; + routeWebSocket: (...args: any[]) => any; +} +/* eslint-enable @typescript-eslint/no-explicit-any */ + +export interface PlaywrightFixtureAdapter { + assertComplete(): void; + install(page: ReplayPageLike): Promise; +} + +export interface NotebookTransportRecorder { + install(page: RecorderPageLike): void; + /** + * Await stop() before asserting or writing a snapshot; response bodies may still be pending. + */ + snapshot(): TransportFixture; + stop(): Promise; + write(fixturePath: string): Promise; +} + +export declare const fixtureVersion: number; + +// Placeholder substitution can change value types, including numbers to strings. +export declare function normalizeFixtureRecord(value: unknown): unknown; +export declare function sanitizeFixture(fixture: TransportFixture): TransportFixture; +export declare function validateFixture(fixture: unknown): string[]; +export declare function validateReplayFixture(fixture: unknown): string[]; + +export declare function createPlaywrightFixtureAdapter(fixture: TransportFixture): PlaywrightFixtureAdapter; +export declare function createNotebookTransportRecorder(metadata: FixtureMetadata): NotebookTransportRecorder; + +export declare function parseRestBody( + body: string, + headers?: Record +): { bodyJson?: unknown; bodyRaw?: string }; +export declare function webSocketPayloadMatches(expectedPayload: unknown, actualMessage: unknown): boolean; +export declare function isNotebookRestUrl(value: string): boolean; diff --git a/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.mjs b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.mjs new file mode 100644 index 00000000000..7967c32ef1b --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.mjs @@ -0,0 +1,1109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +export const fixtureVersion = 1; + +const restDirections = new Set(['request', 'response']); +const websocketDirections = new Set(['send', 'receive']); + +const safeHeaderNames = new Set(['accept', 'content-type']); +const volatileFieldNames = new Set([ + 'dateCreated', + 'dateFinished', + 'dateStarted', + 'dateUpdated', + 'lastUpdated', + 'msgId', + 'time' +]); +// Mask participant identities and permission sets by field name. +// Do not text-scan these names: `user=` can be ordinary notebook content. +const principalFieldNames = new Set(['owners', 'readers', 'roles', 'runners', 'user', 'users', 'writers']); + +// Redact credential containers and identity objects whole. +// Mask permission-array entries individually to preserve their count. +// Recurse into volatile containers: a form named `time` may contain notebook data. +const isContainer = value => value !== null && typeof value === 'object'; + +const maskFieldValue = (key, value) => { + if (!isContainer(value)) { + return `<${key}>`; + } + if (shouldRedactField(key)) { + return `<${key}>`; + } + if (principalFieldNames.has(key)) { + return Array.isArray(value) ? value.map(() => `<${key}>`) : `<${key}>`; + } + return normalizeFixtureRecord(value); +}; + +// Text-scan opaque URLs and raw bodies for credentials. +// sanitizeWebSocket handles payloadText separately to preserve structured notebook content. +const textScannedFieldNames = new Set(['bodyRaw', 'url']); + +export function normalizeFixtureRecord(value) { + if (Array.isArray(value)) { + // Preserve array strings as notebook content; sensitive parent fields are masked before recursion. + return value.map(item => (typeof item === 'string' ? item : normalizeFixtureRecord(item))); + } + // Opaque payloads need text scanning because they have no field names. + if (typeof value === 'string') { + return redactEmbeddedSecrets(value); + } + if (!value || typeof value !== 'object') { + return value; + } + + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => { + if (shouldRedactField(key) || volatileFieldNames.has(key) || principalFieldNames.has(key)) { + return [key, maskFieldValue(key, entry)]; + } + // Preserve keyed notebook text such as `ticketCount = 5`. + // Only the opaque fields above need text scanning in addition to field redaction. + if (typeof entry === 'string') { + return [key, textScannedFieldNames.has(key) ? redactEmbeddedSecrets(entry) : entry]; + } + return [key, normalizeFixtureRecord(entry)]; + }) + ); +} + +function normalizeFixture(fixture) { + return { + ...fixture, + records: fixture.records.map(record => normalizeFixtureRecord(record)) + }; +} + +export function sanitizeFixture(fixture) { + const ids = new Map(); + return normalizeFixture({ + ...fixture, + records: fixture.records.map(record => { + const sanitized = sanitizeRecord(record); + const envelope = parseEnvelope(sanitized.websocket?.payloadText); + if (typeof envelope?.msgId === 'string' && envelope.msgId !== '') { + if (!ids.has(envelope.msgId)) { + ids.set(envelope.msgId, ``); + } + sanitized.websocket.payloadText = JSON.stringify({ ...envelope, msgId: ids.get(envelope.msgId) }); + } + return sanitized; + }) + }); +} + +export function validateFixture(fixture) { + const errors = validateReplayFixture(fixture); + if (!fixture || typeof fixture !== 'object') { + return errors; + } + validateFixtureMetadata(errors, fixture.metadata); + return errors; +} + +export function validateReplayFixture(fixture) { + const errors = []; + if (!fixture || typeof fixture !== 'object') { + return ['fixture must be an object']; + } + if (fixture.version !== fixtureVersion) { + errors.push(`Unsupported fixture version ${fixture.version}`); + } + if (!Array.isArray(fixture.records) || fixture.records.length === 0) { + errors.push('Fixture records must be a non-empty array'); + return errors; + } + + let previousSequence = 0; + const unanswered = []; + for (const [index, record] of fixture.records.entries()) { + const prefix = `records[${index}]`; + if (!record || typeof record !== 'object' || Array.isArray(record)) { + errors.push(`${prefix} must be an object`); + continue; + } + if (!Number.isInteger(record.sequence) || record.sequence <= previousSequence) { + errors.push(`${prefix}.sequence must increase without reordering`); + } + previousSequence = record.sequence; + + if (record.kind === 'rest') { + validateRestRecord(errors, prefix, record); + if (record.rest?.request) { + const key = stableJson(normalizeFixtureRecord(record.rest.request)); + if (record.rest.direction === 'request') { + unanswered.push({ key, prefix }); + } else if (record.rest.direction === 'response') { + const match = unanswered.findIndex(entry => entry.key === key); + if (match !== -1) { + unanswered.splice(match, 1); + } + } + } + } else if (record.kind === 'websocket') { + validateWebSocketRecord(errors, prefix, record); + const envelope = parseEnvelope(record.websocket?.payloadText); + if (envelope && Object.hasOwn(envelope, 'msgId') && envelope.msgId !== null) { + if (envelope.msgId === '') { + errors.push(`${prefix} has ambiguous erased msgId; recapture this fixture`); + } else if (typeof envelope.msgId !== 'string' || !envelope.msgId) { + errors.push(`${prefix} envelope msgId must be a non-empty string or null`); + } + } + } else { + errors.push(`${prefix}.kind must be rest or websocket`); + } + } + for (const entry of unanswered) { + errors.push(`${entry.prefix} request has no response; finish or recapture the operation`); + } + return errors; +} + +export function createPlaywrightFixtureAdapter(fixture) { + const errors = validateReplayFixture(fixture); + if (errors.length > 0) { + throw new Error(errors.join('\n')); + } + + const records = sanitizeFixture(fixture).records; + const pendingRestRequests = []; + const inFlightRestDeliveries = new Set(); + const recordedMessageIds = new Set( + records.map(record => parseEnvelope(record.websocket?.payloadText)?.msgId).filter(id => typeof id === 'string') + ); + const messageIds = new Map(); + const runtimeMessageIds = new Map(); + let cursor = 0; + let webSocket; + let draining = false; + let drainRequested = false; + let fatalError; + let deliveryError; + + const nextRecord = () => records[cursor]; + // Normalize live and recorded URLs identically. + const pendingKey = entry => `${entry.request.method()} ${normalizeFixtureRecord(urlPath(entry.request.url()))}`; + const removePendingRestRequest = entry => { + const index = pendingRestRequests.indexOf(entry); + if (index !== -1) { + pendingRestRequests.splice(index, 1); + } + }; + // Reject requests with no remaining response instead of waiting indefinitely. + const hasRemainingResponseFor = key => + records + .slice(cursor) + .some( + record => + record.kind === 'rest' && + record.rest.direction === 'response' && + `${record.rest.request.method} ${record.rest.request.url}` === key + ); + const failPendingRestRequests = error => { + fatalError ??= error; + for (const entry of pendingRestRequests.splice(0, pendingRestRequests.length)) { + entry.reject(error); + } + }; + const drain = async () => { + if (draining) { + drainRequested = true; + return; + } + + draining = true; + try { + while (true) { + const record = nextRecord(); + if (!record) { + return; + } + if (record.kind === 'websocket') { + if (record.websocket.direction === 'receive' && webSocket) { + const payload = deserializeWebSocketPayload(record.websocket); + const envelope = parseEnvelope(payload); + webSocket.send( + envelope && messageIds.has(envelope.msgId) + ? JSON.stringify({ ...envelope, msgId: messageIds.get(envelope.msgId) }) + : payload + ); + cursor += 1; + continue; + } + return; + } + + if (record.rest.direction === 'request') { + const unmatched = pendingRestRequests.filter(entry => !entry.requestMatched); + const pending = unmatched.find(entry => restRequestMatches(record.rest.request, entry.request)); + if (!pending) { + // Allow arrival-order changes within a consecutive request batch, but not across barriers. + const batch = []; + for (let index = cursor; index < records.length; index += 1) { + const candidate = records[index]; + if (candidate.kind !== 'rest' || candidate.rest.direction !== 'request') { + break; + } + batch.push(candidate.rest.request); + } + const unexpected = unmatched.find( + entry => !batch.some(expected => restRequestMatches(expected, entry.request)) + ); + if (unexpected) { + assertRestRequestMatches( + record.rest.request, + summarizeRequest(unexpected.request), + pendingKey(unexpected) + ); + } + return; + } + const requestKey = pendingKey(pending); + assertRestRequestMatches(record.rest.request, summarizeRequest(pending.request), requestKey); + pending.requestMatched = true; + cursor += 1; + continue; + } + + const responseKey = `${record.rest.request.method} ${record.rest.request.url}`; + const keyed = pendingRestRequests.filter(entry => pendingKey(entry) === responseKey); + // Match concurrent requests by shape, not arrival order. + const shaped = keyed.filter(entry => restRequestMatches(record.rest.request, entry.request)); + const pending = shaped.find(entry => entry.requestMatched) ?? shaped[0]; + if (!pending) { + const drifted = keyed.find(entry => entry.requestMatched) ?? keyed[0]; + if (drifted) { + // Matching method and URL with a different request shape is fixture drift. + assertRestRequestMatches(record.rest.request, summarizeRequest(drifted.request), responseKey); + } + const waiting = pendingRestRequests.find(entry => !entry.requestMatched); + if (waiting) { + throw new Error(`REST fixture request out of order: expected ${responseKey}, got ${pendingKey(waiting)}`); + } + return; + } + pending.requestMatched = true; + removePendingRestRequest(pending); + inFlightRestDeliveries.add(pending); + cursor += 1; + try { + await pending.route.fulfill({ + body: serializeRestBody(record.rest), + contentType: record.rest.headers['content-type'] ?? 'application/json', + headers: record.rest.headers, + status: record.rest.status + }); + pending.resolve(); + } catch (error) { + // Continue independent routes, but retain delivery failures for assertComplete(). + deliveryError ??= error; + pending.reject(error); + } finally { + inFlightRestDeliveries.delete(pending); + } + } + } catch (error) { + failPendingRestRequests(error); + throw error; + } finally { + draining = false; + if (drainRequested) { + drainRequested = false; + void drain().catch(() => undefined); + } + } + }; + + return { + install: async page => { + await page.route('**/api/**', async (route, request) => { + if (!isNotebookRestUrl(request.url())) { + // fallback() preserves earlier route handlers; continue() bypasses them. + await route.fallback?.(); + return; + } + let resolve; + let reject; + const completed = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + completed.catch(() => undefined); + if (fatalError) { + throw fatalError; + } + const pending = { reject, request, requestMatched: false, resolve, route }; + pendingRestRequests.push(pending); + try { + await drain(); + if (fatalError && pendingRestRequests.includes(pending)) { + throw fatalError; + } + const next = nextRecord(); + if ( + pendingRestRequests.includes(pending) && + !pending.requestMatched && + next?.kind === 'websocket' && + next.websocket.direction === 'send' && + // Allow an early request if a later record answers it. + !hasRemainingResponseFor(pendingKey(pending)) + ) { + removePendingRestRequest(pending); + throw new Error(`Transport fixture out of order: expected WebSocket send, got REST ${pendingKey(pending)}`); + } + if (pendingRestRequests.includes(pending) && !hasRemainingResponseFor(pendingKey(pending))) { + removePendingRestRequest(pending); + throw new Error(`REST fixture has no remaining response for ${pendingKey(pending)}`); + } + } catch (error) { + // Fixture errors invalidate every waiting route. + removePendingRestRequest(pending); + failPendingRestRequests(error); + throw error; + } + + try { + await completed; + } catch (error) { + // A route delivery failure leaves independent routes available. + removePendingRestRequest(pending); + throw error; + } + }); + const failFixture = error => { + failPendingRestRequests(error); + throw error; + }; + await page.routeWebSocket( + url => isNotebookWebSocketUrl(url), + ws => { + if (webSocket) { + throw new Error('Transport fixture supports one WebSocket connection per fixture'); + } + webSocket = ws; + void drain().catch(() => undefined); + ws.onMessage(message => { + const record = nextRecord(); + if (!record) { + failFixture( + new Error( + `WebSocket fixture messages exhausted before client send: ${stringifyWebSocketMessage(message)}` + ) + ); + } + if (record.kind !== 'websocket' || record.websocket.direction !== 'send') { + failFixture( + new Error( + `Transport fixture out of order: expected ${describeReplayRecord(record)}, got WebSocket send` + ) + ); + } + const expectedPayload = deserializeWebSocketPayload(record.websocket); + const expected = parseEnvelope(expectedPayload); + const actual = parseEnvelope(stringifyWebSocketMessage(message)); + let comparisonPayload = expectedPayload; + if (expected && typeof expected.msgId === 'string') { + const id = expected.msgId; + const liveId = actual?.msgId; + if ( + typeof liveId !== 'string' || + !liveId || + (recordedMessageIds.has(liveId) && liveId !== id) || + (messageIds.has(id) && messageIds.get(id) !== liveId) || + (runtimeMessageIds.has(liveId) && runtimeMessageIds.get(liveId) !== id) + ) { + failFixture(new Error('WebSocket fixture msgId correlation mismatch')); + } + comparisonPayload = JSON.stringify({ ...expected, msgId: liveId }); + } + if (!webSocketPayloadMatches(comparisonPayload, message)) { + failFixture( + new Error( + `WebSocket fixture send mismatch: expected ${expectedPayload}, got ${stringifyWebSocketMessage(message)}` + ) + ); + } + if (expected && typeof expected.msgId === 'string') { + messageIds.set(expected.msgId, actual.msgId); + runtimeMessageIds.set(actual.msgId, expected.msgId); + } + cursor += 1; + void drain().catch(() => undefined); + }); + } + ); + }, + assertComplete: () => { + if (fatalError) { + throw fatalError; + } + if (deliveryError) { + throw deliveryError; + } + if (records.some(record => record.kind === 'websocket') && !webSocket) { + throw new Error('WebSocket fixture was never connected'); + } + const waiting = pendingRestRequests.length + inFlightRestDeliveries.size; + if (waiting > 0 || cursor !== records.length) { + const unconsumed = records.length - cursor; + throw new Error( + `Transport fixture has ${unconsumed} unconsumed record(s) and ${waiting} unfulfilled REST route(s)` + ); + } + } + }; +} + +export function createNotebookTransportRecorder(metadata) { + const metadataErrors = []; + validateFixtureMetadata(metadataErrors, metadata); + if (metadataErrors.length > 0) { + throw new Error(metadataErrors.join('\n')); + } + + const records = []; + const pending = new Set(); + const outstandingRequests = new Set(); + const responseRecords = new Map(); + let sequence = 0; + let captureFailure; + let webSocketSeen = false; + + const settleCaptures = async () => { + while (pending.size > 0) { + await Promise.all([...pending]); + } + if (outstandingRequests.size > 0) { + captureFailure ??= new Error(`${outstandingRequests.size} outstanding notebook request(s) without a response`); + } + if (captureFailure) { + throw new Error(`Notebook transport capture failed: ${captureFailure?.message ?? captureFailure}`, { + cause: captureFailure + }); + } + }; + + const record = value => { + const entry = { + ...value, + sequence: ++sequence + }; + records.push(entry); + return entry; + }; + + // Track subscriptions so stop() excludes traffic from later navigation or teardown. + let installedPage; + const subscriptions = []; + const subscribe = (target, event, handler) => { + target.on(event, handler); + subscriptions.push([target, event, handler]); + }; + + return { + install: page => { + // Duplicate listeners produce a doubled capture that still passes structural validation. + if (installedPage) { + throw new Error('Transport recorder is already installed on a page'); + } + installedPage = page; + subscribe(page, 'request', request => { + if (!isNotebookRestUrl(request.url())) { + return; + } + outstandingRequests.add(request); + record({ + kind: 'rest', + rest: { + direction: 'request', + request: summarizeRequest(request) + } + }); + }); + subscribe(page, 'response', async response => { + const request = response.request(); + if (!isNotebookRestUrl(request.url())) { + return; + } + outstandingRequests.add(request); + const rest = { + direction: 'response', + headers: filterHeaders(response.headers()), + request: summarizeRequest(request), + status: response.status(), + bodyRaw: '' + }; + responseRecords.set(request, rest); + const bodyRead = response + .text() + .then(body => { + const parsed = parseRestBody(body, response.headers()); + if ('bodyJson' in parsed) delete rest.bodyRaw; + Object.assign(rest, parsed); + }) + .catch(error => { + captureFailure ??= error; + }) + .finally(() => pending.delete(bodyRead)); + pending.add(bodyRead); + }); + subscribe(page, 'requestfinished', request => { + const rest = responseRecords.get(request); + if (!rest) return; + // response fires at headers; text() returns asynchronously. + // Reserve ordering at download completion, before later WebSocket frames. + record({ kind: 'rest', rest }); + responseRecords.delete(request); + outstandingRequests.delete(request); + }); + // Retain the transport error for stop() and write(). + subscribe(page, 'requestfailed', request => { + if (!isNotebookRestUrl(request.url())) { + return; + } + outstandingRequests.delete(request); + responseRecords.delete(request); + const reason = request.failure?.()?.errorText ?? 'unknown error'; + captureFailure ??= new Error( + `Notebook request failed during capture: ${request.method()} ${request.url()} (${reason})` + ); + }); + subscribe(page, 'websocket', socket => { + if (!isNotebookWebSocketUrl(socket.url())) { + return; + } + if (webSocketSeen) { + captureFailure ??= new Error('Transport capture supports one WebSocket connection per fixture'); + return; + } + webSocketSeen = true; + const captureFrame = direction => frame => { + try { + recordCapturedWebSocketFrame(record, direction, framePayload(frame)); + } catch (error) { + // Retain listener errors so stop() and write() reject incomplete captures. + captureFailure ??= error; + throw error; + } + }; + subscribe(socket, 'framesent', captureFrame('send')); + subscribe(socket, 'framereceived', captureFrame('receive')); + }); + }, + stop: async () => { + const detach = ([target, event, handler]) => (target.off ?? target.removeListener)?.call(target, event, handler); + const finishing = []; + for (const subscription of subscriptions.splice(0)) { + // Finish responses already being read while refusing new capture traffic. + if (subscription[1] === 'requestfinished') finishing.push(subscription); + else detach(subscription); + } + try { + await settleCaptures(); + } finally { + finishing.forEach(detach); + } + }, + snapshot: () => sanitizeFixture({ metadata, records: [...records], version: fixtureVersion }), + write: async fixturePath => { + await settleCaptures(); + const sanitized = sanitizeFixture({ metadata, records: [...records], version: fixtureVersion }); + mkdirSync(path.dirname(fixturePath), { recursive: true }); + writeFileSync(fixturePath, `${JSON.stringify(sanitized, null, 2)}\n`); + return sanitized; + } + }; +} + +function serializeRestBody(rest) { + if ('bodyJson' in rest) { + return JSON.stringify(rest.bodyJson); + } + return rest.bodyRaw ?? ''; +} + +export function parseRestBody(body, headers = {}) { + const contentType = headers['content-type'] ?? headers['Content-Type'] ?? ''; + const trimmed = body.trim(); + if (!trimmed) { + return { bodyRaw: '' }; + } + if (contentType.includes('application/json') || /^[{[]/.test(trimmed)) { + try { + return { bodyJson: JSON.parse(body) }; + } catch { + return { bodyRaw: redactRawSensitiveValues(body) }; + } + } + return { bodyRaw: redactRawSensitiveValues(body) }; +} + +function stringifyWebSocketMessage(message) { + return Buffer.isBuffer(message) ? message.toString('utf8') : String(message); +} + +export function webSocketPayloadMatches(expectedPayload, actualMessage) { + const expectedBinary = toBinaryBuffer(expectedPayload); + const actualBinary = toBinaryBuffer(actualMessage); + if (expectedBinary || actualBinary) { + return Boolean(expectedBinary && actualBinary && expectedBinary.equals(actualBinary)); + } + + const actualPayload = stringifyWebSocketMessage(actualMessage); + if (looksLikeJson(expectedPayload) && looksLikeJson(actualPayload)) { + try { + return ( + stableJson(normalizeEnvelope(JSON.parse(actualPayload))) === + stableJson(normalizeEnvelope(JSON.parse(expectedPayload))) + ); + } catch { + // Invalid JSON was captured as raw text and must use the same comparison rules. + } + } + // Redact both sides identically; existing placeholders remain unchanged. + return redactRawSensitiveValues(expectedPayload) === redactRawSensitiveValues(actualPayload); +} + +function summarizeRequest(request) { + const body = request.postData() ?? ''; + return { + headers: filterHeaders(request.headers()), + method: request.method(), + url: urlPath(request.url()), + ...parseRestBody(body, request.headers()) + }; +} + +function sanitizeRecord(record) { + if (record.kind === 'websocket') { + return { + ...record, + websocket: sanitizeWebSocket(record.websocket) + }; + } + if (record.kind !== 'rest') { + return record; + } + return { + ...record, + rest: { + ...record.rest, + ...(record.rest.headers ? { headers: filterHeaders(record.rest.headers) } : {}), + ...(record.rest.request + ? { + request: { + ...record.rest.request, + headers: filterHeaders(record.rest.request.headers) + } + } + : {}) + } + }; +} + +function filterHeaders(headers = {}) { + return Object.fromEntries( + Object.entries(headers) + .map(([key, value]) => [key.toLowerCase(), Array.isArray(value) ? value.join(', ') : String(value ?? '')]) + .filter(([key, value]) => safeHeaderNames.has(key) && !isDefaultAccept(key, value)) + ); +} + +// Playwright may omit Accept during capture but expose its default */* during routing. +// Treat both forms as the same request. +function isDefaultAccept(key, value) { + return key === 'accept' && value.trim() === '*/*'; +} + +function webSocketRecord(direction, payload) { + return { + kind: 'websocket', + websocket: { + direction, + ...(Buffer.isBuffer(payload) ? { payloadBase64: payload.toString('base64') } : { payloadText: String(payload) }) + } + }; +} + +function recordCapturedWebSocketFrame(record, direction, payload) { + if (toBinaryBuffer(payload)) { + throw new Error('Binary WebSocket frames cannot be captured until a binary redaction policy is defined'); + } + record(webSocketRecord(direction, payload)); +} + +export function isNotebookRestUrl(value) { + const url = new URL(value); + return url.pathname === '/api/notebook' || url.pathname.startsWith('/api/notebook/'); +} + +function isNotebookWebSocketUrl(value) { + const url = new URL(value); + return url.pathname === '/ws'; +} + +function urlPath(value) { + const url = new URL(value); + for (const [key] of url.searchParams) { + if (shouldRedactField(key)) { + url.searchParams.set(key, `<${key}>`); + } else if (volatileFieldNames.has(key)) { + url.searchParams.set(key, `<${key}>`); + } + } + return `${url.pathname}${url.search}`; +} + +function stableJson(value) { + if (Array.isArray(value)) { + return `[${value.map(entry => stableJson(entry)).join(',')}]`; + } + if (value && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${stableJson(value[key])}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function parseEnvelope(payload) { + if (typeof payload !== 'string') { + return undefined; + } + try { + const value = JSON.parse(payload); + return value && typeof value === 'object' && !Array.isArray(value) ? value : undefined; + } catch { + return undefined; + } +} + +// Preserve top-level msgId for request/reply correlation. +// Nested IDs keep their normal redaction rules. +function normalizeEnvelope(value) { + const normalized = normalizeFixtureRecord(value); + if (value && !Array.isArray(value) && typeof value === 'object' && Object.hasOwn(value, 'msgId')) { + normalized.msgId = value.msgId; + } + return normalized; +} + +function sanitizeWebSocket(websocket) { + if (!websocket?.payloadText) { + return websocket; + } + if (!looksLikeJson(websocket.payloadText)) { + return { ...websocket, payloadText: redactRawSensitiveValues(websocket.payloadText) }; + } + try { + return { + ...websocket, + payloadText: JSON.stringify(normalizeEnvelope(JSON.parse(websocket.payloadText))) + }; + } catch { + return { ...websocket, payloadText: redactRawSensitiveValues(websocket.payloadText) }; + } +} + +function deserializeWebSocketPayload(websocket) { + if ('payloadBase64' in websocket) { + return Buffer.from(websocket.payloadBase64, 'base64'); + } + return websocket.payloadText ?? ''; +} + +function framePayload(frame) { + if (frame && typeof frame === 'object' && 'payload' in frame) { + return frame.payload; + } + return frame; +} + +function restRequestMatches(expectedRequest, actualRequest) { + const expected = normalizeFixtureRecord(sanitizeRestRequest(expectedRequest)); + const actual = normalizeFixtureRecord(sanitizeRestRequest(summarizeRequest(actualRequest))); + return stableJson(expected) === stableJson(actual); +} + +function assertRestRequestMatches(expectedRequest, actualRequest, requestKey) { + const expected = normalizeFixtureRecord(sanitizeRestRequest(expectedRequest)); + const actual = normalizeFixtureRecord(sanitizeRestRequest(actualRequest)); + if (stableJson(expected) !== stableJson(actual)) { + throw new Error( + `REST fixture request mismatch for ${requestKey}: expected ${stableJson(expected)}, got ${stableJson(actual)}` + ); + } +} + +function sanitizeRestRequest(request) { + return { + ...request, + headers: filterHeaders(request.headers) + }; +} + +// Match sensitive suffixes such as accessToken and PGPASSWORD, but not tokenizer. +// Avoid a prefix quantifier to keep matching linear. +const sensitiveWordPattern = + '(?:api[-_]?key|authorization|client[-_]?secret|cookie|credential(?:s)?|jsessionid|passphrase|passwd|password|principal|private[-_]?key|secret|ticket|token)'; +const sensitiveNamePattern = `${sensitiveWordPattern}(?![A-Za-z0-9_])`; +const sensitiveWholeWordPattern = new RegExp(`^${sensitiveWordPattern}$`, 'i'); +const sensitiveSuffixPattern = new RegExp(`${sensitiveWordPattern}$`, 'i'); +// Bound the candidate-name scan on both sides to keep it linear. +const nameScanPattern = `[A-Za-z0-9_.-]{0,40}${sensitiveWordPattern}[A-Za-z0-9_.-]{0,40}`; + +// Split separators and camel case to find credential names without matching ordinary words. +// Join adjacent parts for names such as apiKey and client_secret. +const nameWords = name => + String(name) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Za-z])([0-9])/g, '$1 $2') + .split(/[^A-Za-z0-9]+/) + .filter(Boolean); + +const sensitiveAnywherePattern = new RegExp(sensitiveWordPattern, 'i'); + +function shouldRedactField(key) { + // Uppercase names such as AWSSECRETKEY have no camel-case boundaries. + if (/^[A-Z0-9_]+$/.test(key) && sensitiveAnywherePattern.test(key)) { + return true; + } + // Also recognize unsplit suffixes such as PGPASSWORD. + if (sensitiveSuffixPattern.test(key)) { + return true; + } + const words = nameWords(key); + return words.some( + (word, index) => + sensitiveWholeWordPattern.test(word) || + (index + 1 < words.length && sensitiveWholeWordPattern.test(word + words[index + 1])) + ); +} +// Group header alternatives so the separator applies to every name. +const sensitiveHeaderNamePattern = '(?:(?:proxy-)?authorization|(?:set-)?cookie)(?![A-Za-z0-9_])'; +// Preserve exact placeholders, including URL-encoded forms, to keep redaction idempotent. +// Named placeholders must match the field; prefixes and mismatched names are not safe. +// For example, password=REAL_SECRET and password= must be masked. +const placeholderValuePattern = /^(?:<([A-Za-z0-9_.-]*)>|%3[Cc]([A-Za-z0-9_.-]*)%3[Ee])$/; +const isPlaceholderValue = (name, body) => { + const match = placeholderValuePattern.exec(body); + if (!match) { + return false; + } + const inner = match[1] ?? match[2]; + return inner === 'redacted' || inner.toLowerCase() === String(name).toLowerCase(); +}; +// Unquoted values end at whitespace or structural delimiters. +// Credentials containing spaces must be quoted to be masked whole. +const unquotedValuePattern = '(?:[{\\[][^\\r\\n]*|[^\\s,;&#}\\]"\'\\r\\n]+)'; + +// Preserve numeric `principal = 1000` in raw text for accounting content. +// Structured principal fields and all other credential names are always masked. +const numericValueNames = /principal$/i; +const keepsNumericValue = (name, body) => numericValueNames.test(name) && /^\d+(?:\.\d+)?$/.test(body); + +function redactEmbeddedSecrets(value) { + const text = String(value); + // Skip repeated name scans when no assignment delimiter exists. + if (!text.includes(':') && !text.includes('=')) { + return text; + } + return ( + text + // URL credentials are identified by position, without a field name. + .replace(/(:\/\/[^\s/:@]+:)(?!@)([^\s@/]+)(?=@)/g, (_match, prefix) => `${prefix}`) + // ?access_token=abc&view=stable + .replace(new RegExp(`([?&](${sensitiveNamePattern})=)([^&#\\s]+)`, 'gi'), (match, prefix, name, body) => + isPlaceholderValue(name, body) ? match : `${prefix}` + ) + // Preserve quotes for valid syntax and repeatable redaction. + .replace( + new RegExp(`(["']?(${nameScanPattern})["']?\\s*(?::|={1,3})\\s*)(["'])((?:\\\\.|(?!\\3)[^\\\\])*)\\3`, 'gi'), + (match, prefix, name, quote, body) => + shouldRedactField(name) && !isPlaceholderValue(name, body) ? `${prefix}${quote}${quote}` : match + ) + // Header credentials can contain spaces and extend to the end of the line. + // Reject leading whitespace so a second pass leaves placeholders intact. + .replace( + new RegExp( + `(${sensitiveHeaderNamePattern}\\s*[:=](?![=])\\s*)(?!)([^\\s,;}\\]"']["']?[^,;}\\]"'\\r\\n]*|[^\\s,;}\\]"'\\r\\n])`, + 'gi' + ), + (_match, prefix) => `${prefix}` + ) + // Match keys at unindented line, string, object or array boundaries. + // Leave prose such as `buy a ticket: today`, indented YAML and list items outside this rule. + .replace( + new RegExp( + `(^|[{,"']\\s*)((["']?)(${nameScanPattern})\\3\\s*:(?![=])\\s*["']?)(${unquotedValuePattern})`, + 'gim' + ), + (match, lead, prefix, _quote, name, body) => + shouldRedactField(name) && !keepsNumericValue(name, body) && !isPlaceholderValue(name, body) + ? `${lead}${prefix}` + : match + ) + // export PGPASSWORD=x, ;password=x, token=x&user=bob, accessToken = "x" + .replace( + new RegExp(`((${nameScanPattern})\\s*=(?![=])\\s*["']?)(${unquotedValuePattern})`, 'gi'), + (match, prefix, name, body) => + shouldRedactField(name) && !keepsNumericValue(name, body) && !isPlaceholderValue(name, body) + ? `${prefix}` + : match + ) + ); +} + +// Use identical redaction for captured raw payloads and live frames. +const redactRawSensitiveValues = redactEmbeddedSecrets; + +function describeReplayRecord(record) { + if (!record) { + return 'end of fixture'; + } + if (record.kind === 'rest') { + return `REST ${record.rest.request.method} ${record.rest.request.url}`; + } + return `WebSocket ${record.websocket.direction}`; +} + +function toBinaryBuffer(value) { + if (Buffer.isBuffer(value)) { + return value; + } + if (value instanceof ArrayBuffer) { + return Buffer.from(value); + } + if (ArrayBuffer.isView(value)) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength); + } + return null; +} + +const validateRestRecord = (errors, prefix, record) => { + if (!record.rest || typeof record.rest !== 'object') { + errors.push(`${prefix}.rest is required`); + return; + } + if (!restDirections.has(record.rest.direction)) { + errors.push(`${prefix}.rest.direction must be request or response`); + } + if (!isHttpMethod(record.rest.request?.method)) { + errors.push(`${prefix}.rest.request.method is required`); + } + if (typeof record.rest.request?.url !== 'string') { + errors.push(`${prefix}.rest.request.url is required`); + } + if (!isHeaderRecord(record.rest.request?.headers)) { + errors.push(`${prefix}.rest.request.headers must be an object`); + } + if (record.rest.direction === 'response') { + if (!Number.isInteger(record.rest.status)) { + errors.push(`${prefix}.rest.status is required for responses`); + } + if (!isHeaderRecord(record.rest.headers)) { + errors.push(`${prefix}.rest.headers must be an object`); + } + if (!hasRestBody(record.rest)) { + errors.push(`${prefix}.rest.bodyJson or bodyRaw is required to preserve response shape`); + } + } else if (!hasRestBody(record.rest.request)) { + errors.push(`${prefix}.rest.request.bodyJson or bodyRaw is required to preserve request shape`); + } +}; + +const validateFixtureMetadata = (errors, metadata) => { + if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) { + errors.push('metadata must be an object'); + return; + } + if (typeof metadata.scenario !== 'string' || !metadata.scenario.trim()) { + errors.push('metadata.scenario must be a non-empty string'); + } + if (typeof metadata.owner !== 'string' || !metadata.owner.trim()) { + errors.push('metadata.owner must be a non-empty string'); + } + if ( + !Array.isArray(metadata.coveredOperations) || + metadata.coveredOperations.length === 0 || + metadata.coveredOperations.some(operation => typeof operation !== 'string' || !operation.trim()) + ) { + errors.push('metadata.coveredOperations must be a non-empty string array'); + } + if ( + !Array.isArray(metadata.knownExclusions) || + metadata.knownExclusions.some(exclusion => typeof exclusion !== 'string' || !exclusion.trim()) + ) { + errors.push('metadata.knownExclusions must be a string array'); + } +}; + +const validateWebSocketRecord = (errors, prefix, record) => { + if (!record.websocket || typeof record.websocket !== 'object') { + errors.push(`${prefix}.websocket is required`); + return; + } + if (!websocketDirections.has(record.websocket.direction)) { + errors.push(`${prefix}.websocket.direction must be send or receive`); + } + const hasText = 'payloadText' in record.websocket; + const hasBase64 = 'payloadBase64' in record.websocket; + if (!hasText && !hasBase64) { + errors.push(`${prefix}.websocket payloadText or payloadBase64 is required to preserve message shape`); + } else if (hasText && hasBase64) { + errors.push(`${prefix}.websocket must contain exactly one of payloadText or payloadBase64`); + } + // Reject malformed payloads before string coercion or permissive Base64 decoding loses data. + if (hasText && typeof record.websocket.payloadText !== 'string') { + errors.push(`${prefix}.websocket.payloadText must be a string`); + } + if (hasBase64 && !isBase64(record.websocket.payloadBase64)) { + errors.push(`${prefix}.websocket.payloadBase64 must be base64`); + } +}; + +// Buffer.from silently drops invalid Base64 characters. +// Require a round trip, allowing equivalent padding. +const isBase64 = value => { + if (typeof value !== 'string' || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) { + return false; + } + const unpadded = text => text.replace(/=+$/, ''); + return unpadded(Buffer.from(value, 'base64').toString('base64')) === unpadded(value); +}; + +const isHttpMethod = value => typeof value === 'string' && /^[A-Z]+$/.test(value); + +const isHeaderRecord = value => + Boolean(value) && + typeof value === 'object' && + !Array.isArray(value) && + Object.values(value).every(entry => typeof entry === 'string'); + +// bodyRaw must be a string; bodyJson accepts any JSON value, including null. +const hasRestBody = value => + Boolean(value) && ('bodyJson' in value || ('bodyRaw' in value && typeof value.bodyRaw === 'string')); + +const looksLikeJson = value => /^[{[]/.test(String(value).trim()); diff --git a/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.test.mjs b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.test.mjs new file mode 100644 index 00000000000..42590e529c0 --- /dev/null +++ b/zeppelin-web-angular/e2e/core-contract/notebook-transport-fixture.test.mjs @@ -0,0 +1,2629 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +import { fixtureMetadata, request, response, wsRecord } from './fixture-doubles.mjs'; +import * as fixtureModule from './notebook-transport-fixture.mjs'; +import { + createNotebookTransportRecorder, + normalizeFixtureRecord, + createPlaywrightFixtureAdapter, + fixtureVersion, + isNotebookRestUrl, + parseRestBody, + validateFixture, + webSocketPayloadMatches +} from './notebook-transport-fixture.mjs'; + +// Temporary roots accumulate across repeated suite runs. +const temporaryRoots = []; +process.on('exit', () => { + for (const root of temporaryRoots) { + rmSync(root, { force: true, recursive: true }); + } +}); + +function emitCompletedResponse(page, capturedResponse) { + page.emit('response', capturedResponse); + page.emit('requestfinished', capturedResponse.request()); +} + +test('replay rejects a request whose safe headers drifted from the record', () => { + // Only accept and content-type survive header filtering; drift in either must fail replay. + const record = { + bodyRaw: '', + headers: { accept: 'application/json' }, + method: 'GET', + url: '/api/notebook/note-a' + }; + const adapter = createPlaywrightFixtureAdapter({ + metadata: fixtureMetadata(), + records: [ + { kind: 'rest', sequence: 1, rest: { direction: 'request', request: record } }, + { + kind: 'rest', + sequence: 2, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: record, + status: 200 + } + } + ], + version: fixtureVersion + }); + + let routeHandler; + adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + return assert.rejects( + () => + routeHandler( + { fulfill: async () => undefined }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a', '', { accept: 'text/plain' }) + ), + /REST fixture request mismatch/ + ); +}); + +test('a structured value is masked whole, not just its opening brace', () => { + // The old value pattern stopped at the first closing brace, leaking the credential and corrupting JSON. + const cases = [ + ['password={"a":1,"b":"hunter2"}', 'password='], + ['token: ["hunter2"]', 'token: '], + ['{"datasource":{"password":{"value":"hunter2"}}}', '{"datasource":{"password":'] + ]; + for (const [input, expected] of cases) { + const masked = normalizeFixtureRecord(input); + assert.doesNotMatch(masked, /hunter2/, `structured value leaked: ${masked}`); + assert.equal(masked, expected); + } +}); + +test('an uppercase name with nothing to split on is still a name', () => { + // SECRETKEY is one uppercase run; without the suffix rule it was missed. + for (const name of ['SECRETKEY', 'AWSSECRETKEY', 'PASSWORDFILE', 'TOKENVALUE', 'APIKEYID', 'PGPASSWORD']) { + assert.equal(normalizeFixtureRecord(`${name}=abc`), `${name}=`, `${name} leaked as text`); + assert.equal(normalizeFixtureRecord({ [name]: 'abc' })[name], `<${name}>`, `${name} leaked as a key`); + } + assert.equal(normalizeFixtureRecord('secretary=alice'), 'secretary=alice'); + assert.equal(normalizeFixtureRecord('tokenizer=bpe'), 'tokenizer=bpe'); +}); + +test('a binary WebSocket frame fails the capture rather than dropping the frame', async () => { + const page = new EventEmitter(); + const socket = new EventEmitter(); + socket.url = () => 'http://127.0.0.1:8080/ws'; + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + recorder.install(page); + page.emit('websocket', socket); + + // Playwright does not turn listener throws into capture failures, so the recorder must remember them. + assert.throws( + () => socket.emit('framesent', { payload: Buffer.from([1, 2, 3]) }), + /Binary WebSocket frames cannot be captured/ + ); + await assert.rejects(() => recorder.stop(), /Binary WebSocket frames cannot be captured/); + await assert.rejects( + () => recorder.write(path.join(createRoot().root, 'f.json')), + /Binary WebSocket frames cannot be captured/ + ); +}); + +test('replay normalizes a live url the same way the record was normalized', async () => { + const adapter = createPlaywrightFixtureAdapter({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { + bodyRaw: '', + headers: { accept: 'application/json' }, + method: 'GET', + url: '/api/notebook/note-a?ticket=%3Cticket%3E&msgId=%3CmsgId%3E' + }, + status: 200 + } + } + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const fulfilled = []; + await routeHandler( + { fulfill: async value => fulfilled.push(value.body) }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a?ticket=runtime-secret&msgId=runtime-id') + ); + + // The record has placeholders and the live URL has real values; both normalize before comparison. + assert.deepEqual(fulfilled, ['{"id":"note-a"}']); + adapter.assertComplete(); + + assert.equal( + webSocketPayloadMatches('{"op":"GET_NOTE","msgId":"runtime"}', '{"msgId":"runtime","op":"GET_NOTE"}'), + true + ); +}); + +test('a name is judged the same way as an object key and as text', () => { + // Key and text redaction used to disagree, corrupting captured interpreter settings. + const masked = [ + 'access_token', + 'accessToken', + 'x-api-key', + 'password', + 'PGPASSWORD', + 'ticket', + // Sensitive word embedded in a larger field name. + 'secretKey', + 'passwordHash', + 'password2', + 'credentialsFile', + 'tokenData', + 'aws_secret_access_key', + 'spark.hadoop.fs.s3a.secret.key', + 'ticketId', + 'private_key', + 'privateKey', + 'PRIVATE_KEY', + 'passphrase' + ]; + const kept = [ + 'max_tokens', + 'tokenizer', + 'tokens', + 'secretary', + 'maxTokens', + 'privately', + 'keyboard', + 'passphraseless' + ]; + + for (const name of masked) { + assert.equal( + normalizeFixtureRecord({ [name]: 'abc' })[name], + `<${name}>`, + `${name} should be masked as an object key` + ); + assert.equal(normalizeFixtureRecord(`${name}=abc`), `${name}=`, `${name} should be masked as text`); + } + + for (const name of kept) { + assert.equal(normalizeFixtureRecord({ [name]: 'abc' })[name], 'abc', `${name} should survive as an object key`); + assert.equal(normalizeFixtureRecord(`${name}=abc`), `${name}=abc`, `${name} should survive as text`); + } + + // The numeric exception is for free text; an actual principal field is still identity data. + assert.equal(normalizeFixtureRecord('principal=1000'), 'principal=1000'); + assert.equal(normalizeFixtureRecord({ principal: 1000 }).principal, ''); + assert.equal(normalizeFixtureRecord({ principal: '10234' }).principal, ''); + assert.equal(normalizeFixtureRecord({ principal: 's3cr3t' }).principal, ''); +}); + +test('a value that opens with an angle bracket is still a value', () => { + // Values beginning with '<' still need redaction unless they are known placeholders. + assert.equal(normalizeFixtureRecord('password='), 'password='); + // Known placeholders stay idempotent. + assert.equal(normalizeFixtureRecord('?ticket=%3Cticket%3E'), '?ticket=%3Cticket%3E'); + assert.equal(normalizeFixtureRecord('password='), 'password='); +}); + +test('recorder captures notebook REST and WebSocket browser events only', async () => { + const page = new EventEmitter(); + const socket = new EventEmitter(); + socket.url = () => 'http://127.0.0.1:8080/ws'; + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + const capturedRequest = request('POST', 'http://127.0.0.1:8080/api/notebook/note-a', '{"msgId":"runtime"}'); + page.emit('request', capturedRequest); + emitCompletedResponse(page, response(capturedRequest, 200, '{"id":"note-a"}')); + page.emit('request', request('GET', 'http://127.0.0.1:8080/assets/app.js')); + page.emit('websocket', socket); + socket.emit('framesent', { payload: '{"op":"GET_NOTE","msgId":"runtime"}' }); + socket.emit('framereceived', { payload: '{"op":"NOTE","noteId":"note-a"}' }); + + await recorder.stop(); + const fixture = recorder.snapshot(); + + assert.deepEqual(validateFixture(fixture), []); + assert.deepEqual( + fixture.records.map(record => [ + record.sequence, + record.kind, + record.rest?.direction ?? record.websocket?.direction + ]), + [ + [1, 'rest', 'request'], + [2, 'rest', 'response'], + [3, 'websocket', 'send'], + [4, 'websocket', 'receive'] + ] + ); + assert.equal(fixture.records[0].rest.request.bodyJson.msgId, ''); + assert.equal(fixture.records[1].rest.bodyJson.id, 'note-a'); + assert.deepEqual(fixture.metadata, fixtureMetadata()); +}); + +test('recorder requires scenario ownership and coverage metadata before capture', () => { + assert.throws(() => createNotebookTransportRecorder(), /metadata must be an object/); + assert.throws( + () => createNotebookTransportRecorder({ ...fixtureMetadata(), coveredOperations: [] }), + /metadata.coveredOperations must be a non-empty string array/ + ); + assert.deepEqual( + validateFixture({ + metadata: { ...fixtureMetadata(), scenario: '' }, + records: [wsRecord(1, 'send', '{}')], + version: fixtureVersion + }), + ['metadata.scenario must be a non-empty string'] + ); + assert.deepEqual(validateFixture({ records: [wsRecord(1, 'send', '{}')], version: fixtureVersion }), [ + 'metadata must be an object' + ]); +}); + +test('recorder redacts sensitive headers and fields before writing fixture files', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.on('request', capturedRequest => emitCompletedResponse(page, response(capturedRequest, 200, '{}'))); + page.emit( + 'request', + request('POST', 'http://127.0.0.1:8080/api/notebook', '{"ticket":"secret","id":"stable-id"}', { + accept: 'application/json', + authorization: 'Bearer secret', + cookie: 'ticket=secret', + 'content-type': 'application/json' + }) + ); + const written = await recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + const text = readFileSync(path.join(root.root, 'fixtures/notebook-transport.json'), 'utf8'); + + assert.equal(text.includes('Bearer secret'), false); + assert.equal(text.includes('ticket=secret'), false); + assert.deepEqual(written.records[0].rest.request.headers, { + accept: 'application/json', + 'content-type': 'application/json' + }); + assert.deepEqual(written.records[0].rest.request.bodyJson, { id: 'stable-id', ticket: '' }); +}); + +test('recorder redacts WebSocket JSON payload secrets before writing fixture files', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const socket = new EventEmitter(); + socket.url = () => 'http://127.0.0.1:8080/ws'; + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.emit('websocket', socket); + socket.emit('framesent', { + payload: + '{"op":"GET_NOTE","id":"stable-id","noteId":"note-a","ticket":"secret-ticket","principal":"alice","msgId":"runtime"}' + }); + + await recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + const text = readFileSync(path.join(root.root, 'fixtures/notebook-transport.json'), 'utf8'); + const fixture = JSON.parse(text); + + assert.equal(text.includes('secret-ticket'), false); + assert.equal(text.includes('alice'), false); + assert.equal(text.includes('runtime'), false); + assert.equal(fixture.records[0].websocket.payloadText.includes('"op":"GET_NOTE"'), true); + assert.equal(fixture.records[0].websocket.payloadText.includes('"id":"stable-id"'), true); + assert.equal(fixture.records[0].websocket.payloadText.includes('"noteId":"note-a"'), true); +}); + +test('a recorder refuses a second install and stops listening when it stops', async () => { + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + recorder.install(page); + + // Double installation records duplicate frames that validate but cannot replay cleanly. + assert.throws(() => recorder.install(page), /already installed/); + + const first = request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'); + page.emit('request', first); + emitCompletedResponse(page, response(first, 200, '{"id":"note-a"}')); + await recorder.stop(); + const captured = recorder.snapshot().records.length; + + // Traffic after stop() must not extend the fixture's claimed endpoint. + const late = request('DELETE', 'http://127.0.0.1:8080/api/notebook/note-a'); + page.emit('request', late); + emitCompletedResponse(page, response(late, 200, '{}')); + + assert.equal(recorder.snapshot().records.length, captured, 'traffic after stop was recorded'); +}); + +test('a json response record carries no leftover empty bodyRaw', async () => { + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + recorder.install(page); + + const live = request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'); + page.emit('request', live); + emitCompletedResponse(page, response(live, 200, '{"id":"note-a"}')); + await recorder.stop(); + + const record = recorder.snapshot().records.find(entry => entry.rest?.direction === 'response'); + assert.deepEqual(record.rest.bodyJson, { id: 'note-a' }); + assert.equal('bodyRaw' in record.rest, false, 'an empty bodyRaw was left beside bodyJson'); +}); + +test('recorder fails the capture when a notebook request never got a response', async () => { + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + const failed = request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'); + failed.failure = () => ({ errorText: 'net::ERR_CONNECTION_RESET' }); + + recorder.install(page); + page.emit('request', failed); + page.emit('requestfailed', failed); + + await assert.rejects(() => recorder.stop(), /never got a response|failed during capture/); +}); + +test('recorder rejects binary WebSocket frames until binary redaction is defined', () => { + const page = new EventEmitter(); + const socket = new EventEmitter(); + socket.url = () => 'http://127.0.0.1:8080/ws'; + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.emit('websocket', socket); + + assert.throws( + () => socket.emit('framesent', { payload: Buffer.from([0, 255, 1]) }), + /Binary WebSocket frames cannot be captured/ + ); +}); + +test('recorder normalizes sensitive and volatile REST URL query values before writing', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.on('request', capturedRequest => emitCompletedResponse(page, response(capturedRequest, 200, '{}'))); + page.emit( + 'request', + request( + 'GET', + 'http://127.0.0.1:8080/api/notebook/note-a?ticket=secret-ticket&token=secret-token&msgId=runtime&view=stable' + ) + ); + + await recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + const text = readFileSync(path.join(root.root, 'fixtures/notebook-transport.json'), 'utf8'); + const fixture = JSON.parse(text); + + assert.equal(text.includes('secret-ticket'), false); + assert.equal(text.includes('secret-token'), false); + assert.equal( + fixture.records[0].rest.request.url, + '/api/notebook/note-a?ticket=%3Cticket%3E&token=%3Ctoken%3E&msgId=%3CmsgId%3E&view=stable' + ); +}); + +test('recorder redacts credential-shaped body and query fields before writing', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.on('request', capturedRequest => emitCompletedResponse(page, response(capturedRequest, 200, '{}'))); + page.emit( + 'request', + request( + 'POST', + 'http://127.0.0.1:8080/api/notebook/note-a?apiKey=query-key&clientSecret=query-secret&view=stable', + '{"apiKey":"body-key","credential":"body-credential","secret":"body-secret","id":"stable-id"}', + { 'content-type': 'application/json' } + ) + ); + + await recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + const text = readFileSync(path.join(root.root, 'fixtures/notebook-transport.json'), 'utf8'); + const fixture = JSON.parse(text); + + for (const value of ['query-key', 'query-secret', 'body-key', 'body-credential', 'body-secret']) { + assert.equal(text.includes(value), false); + } + assert.deepEqual(fixture.records[0].rest.request.bodyJson, { + apiKey: '', + credential: '', + id: 'stable-id', + secret: '' + }); + assert.equal( + fixture.records[0].rest.request.url, + '/api/notebook/note-a?apiKey=%3CapiKey%3E&clientSecret=%3CclientSecret%3E&view=stable' + ); +}); + +test('recorder fails closed when response body capture fails', async () => { + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.emit( + 'response', + response(request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'), 200, async () => { + throw new Error('body unavailable'); + }) + ); + + await assert.rejects(() => recorder.stop(), /body unavailable/); +}); + +test('Playwright adapter replays WebSocket payloadBase64 as binary data', async () => { + const calls = []; + const page = { + route: async () => undefined, + routeWebSocket: async (_pattern, handler) => calls.push(handler) + }; + await createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'websocket', + sequence: 1, + websocket: { direction: 'send', payloadText: 'client-ready' } + }, + { + kind: 'websocket', + sequence: 2, + websocket: { direction: 'receive', payloadBase64: Buffer.from([0, 255, 1, 2]).toString('base64') } + } + ], + version: fixtureVersion + }).install(page); + + const replies = []; + const handlers = []; + calls[0]({ + onMessage: handler => handlers.push(handler), + send: message => replies.push(message) + }); + handlers[0]('client-ready'); + + assert.equal(Buffer.isBuffer(replies[0]), true); + assert.deepEqual([...replies[0]], [0, 255, 1, 2]); +}); + +test('WebSocket binary payload matching compares bytes instead of UTF-8 replacement text', () => { + assert.equal(webSocketPayloadMatches(Buffer.from([0xff]), Buffer.from([0xff])), true); + assert.equal(webSocketPayloadMatches(Buffer.from([0xff]), Buffer.from([0xfe])), false); + assert.equal(webSocketPayloadMatches(Buffer.from([0xff]), '\ufffd'), false); +}); + +test('recorder write waits for pending response body capture', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + let resolveBody; + const live = request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'); + + recorder.install(page); + page.emit( + 'response', + response(live, 200, () => { + return new Promise(resolve => { + resolveBody = resolve; + }); + }) + ); + const writePromise = recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + resolveBody('{"id":"note-a"}'); + page.emit('requestfinished', live); + const written = await writePromise; + + assert.deepEqual(written.records[0].rest.bodyJson, { id: 'note-a' }); +}); + +test('notebook REST predicate excludes unrelated API traffic', async () => { + assert.equal(isNotebookRestUrl('http://127.0.0.1:8080/api/notebook'), true); + assert.equal(isNotebookRestUrl('http://127.0.0.1:8080/api/notebook/note-a'), true); + assert.equal(isNotebookRestUrl('http://127.0.0.1:8080/api/security/ticket'), false); + assert.equal(isNotebookRestUrl('http://127.0.0.1:8080/api/configurations/all'), false); +}); + +test('REST body parsing preserves raw non-JSON and parses JSON-looking bodies for normalization', () => { + assert.deepEqual(parseRestBody('plain text', { 'content-type': 'text/plain' }), { bodyRaw: 'plain text' }); + assert.deepEqual(parseRestBody('{"noteId":"note-a","stable":true}', { 'content-type': 'application/json' }), { + bodyJson: { noteId: 'note-a', stable: true } + }); +}); + +test('recorder redacts sensitive values from malformed JSON REST and WebSocket payloads', async () => { + const root = createRoot(); + const page = new EventEmitter(); + const socket = new EventEmitter(); + socket.url = () => 'http://127.0.0.1:8080/ws'; + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + + recorder.install(page); + page.on('request', capturedRequest => emitCompletedResponse(page, response(capturedRequest, 200, '{}'))); + page.emit('request', request('POST', 'http://127.0.0.1:8080/api/notebook', '{"token":"rest-secret')); + page.emit('websocket', socket); + socket.emit('framesent', { payload: '{"credential":"socket-secret' }); + + const fixture = await recorder.write(path.join(root.root, 'fixtures/notebook-transport.json')); + const serialized = JSON.stringify(fixture); + assert.equal(serialized.includes('rest-secret'), false); + assert.equal(serialized.includes('socket-secret'), false); +}); + +test('Playwright adapter replays WebSocket fixtures with cursors and no server forwarding by default', async () => { + const calls = []; + const page = { + route: async () => undefined, + routeWebSocket: async (_pattern, handler) => calls.push(handler) + }; + await createPlaywrightFixtureAdapter({ + records: [ + wsRecord(1, 'send', '{"op":"GET_NOTE","msgId":""}'), + wsRecord(2, 'receive', '{"op":"NOTE","noteId":"note-a"}'), + wsRecord(3, 'send', '{"op":"RUN_PARAGRAPH","paragraphId":"paragraph-a"}'), + wsRecord(4, 'receive', '{"op":"PARAGRAPH","paragraphId":"paragraph-a"}') + ], + version: fixtureVersion + }).install(page); + + const replies = []; + const forwarded = []; + const handlers = []; + calls[0]({ + connectToServer: () => ({ send: message => forwarded.push(message) }), + onMessage: handler => handlers.push(handler), + send: message => replies.push(message) + }); + + handlers[0]('{"op":"GET_NOTE","msgId":"runtime"}'); + handlers[0]('{"op":"RUN_PARAGRAPH","paragraphId":"paragraph-a"}'); + + assert.deepEqual(forwarded, []); + assert.deepEqual(replies, ['{"op":"NOTE","noteId":"note-a"}', '{"op":"PARAGRAPH","paragraphId":"paragraph-a"}']); + assert.throws(() => handlers[0]('{"op":"EXTRA"}'), /messages exhausted/); +}); + +test('Playwright adapter emits server-first messages and reports unconsumed messages', async () => { + const calls = []; + const page = { + route: async () => undefined, + routeWebSocket: async (_pattern, handler) => calls.push(handler) + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [wsRecord(1, 'receive', '{"op":"CONNECTED"}'), wsRecord(2, 'send', '{"op":"GET_NOTE"}')], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + const replies = []; + const handlers = []; + calls[0]({ + onMessage: handler => handlers.push(handler), + send: message => replies.push(message) + }); + + assert.deepEqual(replies, ['{"op":"CONNECTED"}']); + assert.throws(() => fixtureAdapter.assertComplete(), /1 unconsumed record\(s\) and 0 unfulfilled/); + handlers[0]('{"op":"GET_NOTE"}'); + assert.doesNotThrow(() => fixtureAdapter.assertComplete()); +}); + +test('Playwright adapter requires every REST response and WebSocket connection to be consumed', async () => { + const restAdapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + await restAdapter.install({ route: async () => undefined, routeWebSocket: async () => undefined }); + assert.throws(() => restAdapter.assertComplete(), /1 unconsumed record\(s\) and 0 unfulfilled/); + + const webSocketAdapter = createPlaywrightFixtureAdapter({ + records: [wsRecord(1, 'send', '{"op":"GET_NOTE"}')], + version: fixtureVersion + }); + await webSocketAdapter.install({ route: async () => undefined, routeWebSocket: async () => undefined }); + assert.throws(() => webSocketAdapter.assertComplete(), /was never connected/); +}); + +test('Playwright adapter preserves a REST request, server WebSocket frame, and REST response ordering', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async (_pattern, handler) => calls.push({ handler, kind: 'websocket' }) + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + direction: 'request', + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' } + } + }, + wsRecord(2, 'receive', '{"op":"NOTE","noteId":"note-a"}'), + { + kind: 'rest', + sequence: 3, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + const events = []; + calls + .find(call => call.kind === 'websocket') + .handler({ + onMessage: () => undefined, + send: message => events.push(`websocket:${message}`) + }); + await calls + .find(call => call.kind === 'route') + .handler( + { fulfill: async value => events.push(`rest:${value.body}`) }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a') + ); + + assert.deepEqual(events, ['websocket:{"op":"NOTE","noteId":"note-a"}', 'rest:{"id":"note-a"}']); + assert.doesNotThrow(() => fixtureAdapter.assertComplete()); +}); + +test('Playwright adapter waits for a REST response before sending a following WebSocket frame', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async (_pattern, handler) => calls.push({ handler, kind: 'websocket' }) + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + direction: 'request', + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' } + } + }, + { + kind: 'rest', + sequence: 2, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + }, + wsRecord(3, 'receive', '{"op":"AFTER_REST"}') + ], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + const events = []; + calls + .find(call => call.kind === 'websocket') + .handler({ + onMessage: () => undefined, + send: message => events.push(`websocket:${message}`) + }); + let releaseFulfill; + const routePromise = calls + .find(call => call.kind === 'route') + .handler( + { + fulfill: async value => { + events.push(`fulfill:${value.body}`); + await new Promise(resolve => { + releaseFulfill = resolve; + }); + events.push('fulfilled'); + } + }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a') + ); + + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(events, ['fulfill:{"id":"note-a"}']); + releaseFulfill(); + await routePromise; + + assert.deepEqual(events, ['fulfill:{"id":"note-a"}', 'fulfilled', 'websocket:{"op":"AFTER_REST"}']); + assert.doesNotThrow(() => fixtureAdapter.assertComplete()); +}); + +test('Playwright adapter rejects REST requests whose body does not match the captured request record', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async () => undefined + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + direction: 'request', + request: { + bodyJson: { paragraphId: 'paragraph-a' }, + headers: { accept: 'application/json', 'content-type': 'application/json' }, + method: 'POST', + url: '/api/notebook/note-a/paragraph' + } + } + }, + { + kind: 'rest', + sequence: 2, + rest: { + bodyJson: { status: 'ok' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { + bodyJson: { paragraphId: 'paragraph-a' }, + headers: { accept: 'application/json', 'content-type': 'application/json' }, + method: 'POST', + url: '/api/notebook/note-a/paragraph' + }, + status: 200 + } + } + ], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + await assert.rejects( + () => + calls + .find(call => call.kind === 'route') + .handler( + { fulfill: async () => undefined }, + request('POST', 'http://127.0.0.1:8080/api/notebook/note-a/paragraph', '{"paragraphId":"paragraph-b"}', { + accept: 'application/json', + 'content-type': 'application/json' + }) + ), + /REST fixture request mismatch/ + ); +}); + +test('Playwright adapter waits for an interleaved client WebSocket frame before fulfilling REST', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async (_pattern, handler) => calls.push({ handler, kind: 'websocket' }) + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + direction: 'request', + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' } + } + }, + wsRecord(2, 'send', '{"op":"GET_NOTE"}'), + { + kind: 'rest', + sequence: 3, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + const handlers = []; + calls + .find(call => call.kind === 'websocket') + .handler({ + onMessage: handler => handlers.push(handler), + send: () => undefined + }); + const routePromise = calls + .find(call => call.kind === 'route') + .handler({ fulfill: async () => undefined }, request('GET', 'http://127.0.0.1:8080/api/notebook/note-a')); + handlers[0]('{"op":"GET_NOTE"}'); + await routePromise; + + assert.doesNotThrow(() => fixtureAdapter.assertComplete()); +}); + +test('Playwright adapter rejects a REST request that arrives before an expected client WebSocket frame', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async (_pattern, handler) => calls.push({ handler, kind: 'websocket' }) + }; + const fixtureAdapter = createPlaywrightFixtureAdapter({ + records: [wsRecord(1, 'send', '{"op":"GET_NOTE"}')], + version: fixtureVersion + }); + await fixtureAdapter.install(page); + + await assert.rejects( + calls + .find(call => call.kind === 'route') + .handler({ fulfill: async () => undefined }, request('GET', 'http://127.0.0.1:8080/api/notebook/note-a')), + /expected WebSocket send, got REST GET \/api\/notebook\/note-a/ + ); +}); + +test('fixture validation reports non-object records instead of throwing', () => { + assert.deepEqual(validateFixture({ metadata: fixtureMetadata(), records: [null], version: fixtureVersion }), [ + 'records[0] must be an object' + ]); +}); + +function createRoot() { + const root = mkdtempSync(path.join(os.tmpdir(), 'zeppelin-capture-')); + temporaryRoots.push(root); + return { root }; +} + +test('Playwright adapter fulfills interleaved REST responses without hanging a route', async () => { + const restRecord = (sequence, direction, url, extra = {}) => ({ + kind: 'rest', + sequence, + rest: { + direction, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url }, + ...extra + } + }); + const responseExtra = url => ({ + bodyJson: { url }, + headers: { 'content-type': 'application/json' }, + status: 200 + }); + + // Concurrent notebook reads record responses in completion order. + const adapter = createPlaywrightFixtureAdapter({ + records: [ + restRecord(1, 'request', '/api/notebook/note-a'), + restRecord(2, 'request', '/api/notebook/note-b'), + restRecord(3, 'response', '/api/notebook/note-b', responseExtra('/api/notebook/note-b')), + restRecord(4, 'response', '/api/notebook/note-a', responseExtra('/api/notebook/note-a')) + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const fulfilled = []; + const call = url => + routeHandler( + { + fulfill: async ({ body }) => { + fulfilled.push(`${url} ${body}`); + } + }, + request('GET', `http://127.0.0.1:8080${url}`) + ); + + const settled = await Promise.race([ + Promise.all([call('/api/notebook/note-a'), call('/api/notebook/note-b')]).then(() => 'settled'), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + + assert.equal(settled, 'settled'); + assert.deepEqual(fulfilled.sort(), [ + '/api/notebook/note-a {"url":"/api/notebook/note-a"}', + '/api/notebook/note-b {"url":"/api/notebook/note-b"}' + ]); + adapter.assertComplete(); +}); + +test('Playwright adapter rejects a REST route that no remaining response record can match', async () => { + const adapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const call = routeHandler( + { fulfill: async () => undefined }, + request('GET', 'http://127.0.0.1:8080/api/notebook/other') + ); + const outcome = await Promise.race([ + call.then( + () => 'resolved', + error => error + ), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + assert.notEqual(outcome, 'hung'); + assert.match(String(outcome), /out of order/); +}); + +test('capture redacts sensitive values in non-JSON WebSocket frames', () => { + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + let socketHandler; + recorder.install({ + on: (event, handler) => { + if (event === 'websocket') { + socketHandler = handler; + } + } + }); + + socketHandler({ + on: (event, handler) => { + if (event === 'framesent') { + handler({ payload: 'password=hunter2 Authorization: Bearer abc ticket=t1' }); + } + }, + url: () => 'ws://127.0.0.1:8080/ws' + }); + + const captured = JSON.stringify(recorder.snapshot()); + for (const secret of ['hunter2', 'Bearer abc', 't1']) { + assert.ok(!captured.includes(secret), `non-JSON frame leaked ${secret}: ${captured}`); + } +}); + +test('capture redacts sensitive keys in bodies that are not valid JSON', () => { + const truncated = parseRestBody('{"Authorization":"Bearer abc", "cookie":"JSESSIONID=xyz", "principal":"admin"', {}); + const serialized = JSON.stringify(truncated); + for (const secret of ['Bearer abc', 'JSESSIONID=xyz', 'admin']) { + assert.ok(!serialized.includes(secret), `malformed JSON leaked ${secret}: ${serialized}`); + } + + const spaced = parseRestBody('{"password":"hunter two"', {}); + assert.ok(!JSON.stringify(spaced).includes('hunter two'), `quoted value leaked: ${spaced.bodyRaw}`); +}); + +test('recorder fails closed when a response body read rejects before the fixture is written', async () => { + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + let responseHandler; + recorder.install({ + on: (event, handler) => { + if (event === 'response') { + responseHandler = handler; + } + } + }); + + const failing = response(request('GET', 'http://127.0.0.1:8080/api/notebook/note-a'), 200, () => + Promise.reject(new Error('body unavailable')) + ); + await responseHandler(failing); + // A pre-stop body read failure must reject the capture, not become an unhandled rejection. + await new Promise(resolve => setTimeout(resolve, 10)); + await assert.rejects(() => recorder.stop(), /body unavailable/); + await assert.rejects(() => recorder.write(path.join(createRoot().root, 'f.json')), /body unavailable/); +}); + +test('Playwright adapter fulfills a response-only record followed by a client WebSocket frame', async () => { + const calls = []; + const page = { + route: async (_pattern, handler) => calls.push({ handler, kind: 'route' }), + routeWebSocket: async (_pattern, handler) => calls.push({ handler, kind: 'websocket' }) + }; + // Shorthand: response-only record, then the client frame. + const adapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + }, + wsRecord(2, 'send', '{"op":"GET_NOTE"}') + ], + version: fixtureVersion + }); + await adapter.install(page); + + const handlers = []; + calls + .find(call => call.kind === 'websocket') + .handler({ + onMessage: handler => handlers.push(handler), + send: () => undefined + }); + + const fulfilled = []; + const routePromise = calls + .find(call => call.kind === 'route') + .handler( + { + fulfill: async ({ body }) => { + fulfilled.push(body); + } + }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a') + ); + + const settled = await Promise.race([ + routePromise.then( + () => 'settled', + error => error + ), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + assert.equal(settled, 'settled', `response-only record must resolve its route, got ${settled}`); + assert.deepEqual(fulfilled, ['{"id":"note-a"}']); + + handlers[0]('{"op":"GET_NOTE"}'); + assert.doesNotThrow(() => adapter.assertComplete()); +}); + +test('Playwright adapter rejects a response-only record whose route body does not match', async () => { + const adapter = createPlaywrightFixtureAdapter({ + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { + bodyRaw: '{"name":"expected"}', + headers: { accept: 'application/json' }, + method: 'POST', + url: '/api/notebook/note-a' + }, + status: 200 + } + } + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + await assert.rejects( + routeHandler( + { fulfill: async () => undefined }, + request('POST', 'http://127.0.0.1:8080/api/notebook/note-a', '{"name":"other"}') + ), + /REST fixture request mismatch/ + ); +}); + +test('Playwright adapter keeps replaying when one route cannot be fulfilled', async () => { + const restRecord = (sequence, direction, url, extra = {}) => ({ + kind: 'rest', + sequence, + rest: { + direction, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url }, + ...extra + } + }); + const adapter = createPlaywrightFixtureAdapter({ + records: [ + restRecord(1, 'request', '/api/notebook/note-a'), + restRecord(2, 'request', '/api/notebook/note-b'), + restRecord(3, 'response', '/api/notebook/note-a', { + bodyJson: { id: 'note-a' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }), + restRecord(4, 'response', '/api/notebook/note-b', { + bodyJson: { id: 'note-b' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }) + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + // A failed fulfill belongs to that route; later independent records must still replay. + const first = routeHandler( + { + fulfill: async () => { + throw new Error('route was already handled'); + } + }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-a') + ); + const second = routeHandler( + { fulfill: async () => undefined }, + request('GET', 'http://127.0.0.1:8080/api/notebook/note-b') + ); + + const outcome = await Promise.race([ + Promise.allSettled([first, second]), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + assert.notEqual(outcome, 'hung', 'a failed fulfill must not leave a route waiting'); + assert.deepEqual( + outcome.map(result => result.status), + ['rejected', 'fulfilled'] + ); + assert.match(String(outcome[0].reason), /route was already handled/); + assert.throws(() => adapter.assertComplete(), /route was already handled/); +}); + +test('capture redacts a credential carried inside a JSON string value', () => { + const sanitized = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { note: { url: '/api/notebook/note-a?ticket=abc123&view=stable' } }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + + const url = sanitized.records[0].rest.bodyJson.note.url; + assert.ok(!url.includes('abc123'), `embedded query secret leaked: ${url}`); + // Only the credential is masked; the rest of the value is preserved. + assert.ok(url.includes('view=stable'), `redaction destroyed the surrounding value: ${url}`); +}); + +test('raw redaction masks the credential without swallowing the rest of the line', () => { + const parsed = parseRestBody('token=abc user=bob action=run', { 'content-type': 'text/plain' }); + assert.ok(!parsed.bodyRaw.includes('abc'), `token leaked: ${parsed.bodyRaw}`); + assert.match(parsed.bodyRaw, /user=bob action=run/); +}); + +test('Playwright adapter tells concurrent same-url requests apart by request body', async () => { + const postRecord = (sequence, direction, paragraph, extra = {}) => ({ + kind: 'rest', + sequence, + rest: { + direction, + request: { + bodyJson: { paragraph }, + headers: { accept: 'application/json' }, + method: 'POST', + url: '/api/notebook/run' + }, + ...extra + } + }); + const adapter = createPlaywrightFixtureAdapter({ + records: [ + postRecord(1, 'request', 'one'), + postRecord(2, 'request', 'two'), + postRecord(3, 'response', 'two', { + bodyJson: { ran: 'two' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }), + postRecord(4, 'response', 'one', { + bodyJson: { ran: 'one' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }) + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const fulfilled = []; + const call = body => + routeHandler( + { + fulfill: async ({ body: responseBody }) => { + fulfilled.push(`${body} -> ${responseBody}`); + } + }, + request('POST', 'http://127.0.0.1:8080/api/notebook/run', body) + ); + + const settled = await Promise.race([ + Promise.all([call('{"paragraph":"two"}'), call('{"paragraph":"one"}')]).then(() => 'settled'), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + assert.equal(settled, 'settled'); + // Match same-URL routes by body, not arrival order. + assert.deepEqual(fulfilled.sort(), ['{"paragraph":"one"} -> {"ran":"one"}', '{"paragraph":"two"} -> {"ran":"two"}']); + adapter.assertComplete(); +}); + +test('capture redacts a credential in a payload that has no field name to key on', () => { + // Raw payloads and URL queries have no field key, so text redaction is the only layer. + const leaks = [ + '%sh export PASSWORD=LEAK5', + 'token=LEAK3&user=bob', + 'curl -H "Authorization: Bearer LEAK4"', + 'Authorization=Bearer LEAK7', + 'jdbc:hive2://host:1/db;password=LEAK6', + '/api/notebook/x?access_token=LEAK1', + '/x?x-api-key=LEAK2', + 'token="LEAK8 LEAK9"', + 'password="LEAK10 with spaces"', + 'ticket=LEAK11, note=x' + ]; + + for (const leak of leaks) { + const sanitized = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyRaw: leak, + direction: 'response', + headers: { 'content-type': 'text/plain' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/a' }, + status: 200 + } + }, + { kind: 'websocket', sequence: 2, websocket: { direction: 'send', payloadText: leak } } + ], + version: fixtureVersion + }); + + assert.doesNotMatch(sanitized.records[0].rest.bodyRaw, /LEAK\d/, `body leaked: ${leak}`); + assert.doesNotMatch(sanitized.records[1].websocket.payloadText, /LEAK\d/, `frame leaked: ${leak}`); + } + + // URL query credentials are redacted wherever the URL appears. + const withUrl = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { + bodyRaw: '', + headers: { accept: 'application/json' }, + method: 'GET', + url: '/api/notebook/note-a?ticket=LEAK12' + }, + status: 200 + } + } + ], + version: fixtureVersion + }); + assert.doesNotMatch(withUrl.records[0].rest.request.url, /LEAK\d/); + + const preserved = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyRaw: 'please buy a ticket: today and user=bob action=run', + direction: 'response', + headers: { 'content-type': 'text/plain' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + assert.equal(preserved.records[0].rest.bodyRaw, 'please buy a ticket: today and user=bob action=run'); +}); + +test('a json field keeps its text, because the field name is what redaction acts on', () => { + // JSON string values can be note text, so value-wide text scanning would corrupt paragraphs. + // The helper captures from an isolated empty server so the note text is test-owned. + const sanitized = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { + kind: 'rest', + sequence: 1, + rest: { + bodyJson: { + password: 'masked-by-name', + text: 'ticketCount = df.count()\nconst cookieBanner = document.getElementById("x")' + }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/a' }, + status: 200 + } + } + ], + version: fixtureVersion + }); + + const body = sanitized.records[0].rest.bodyJson; + assert.equal(body.password, ''); + assert.equal(body.text, 'ticketCount = df.count()\nconst cookieBanner = document.getElementById("x")'); +}); + +test('Playwright adapter fails a route that arrives after the fixture has already failed', async () => { + const restRecord = (sequence, direction, paragraph, extra = {}) => ({ + kind: 'rest', + sequence, + rest: { + direction, + request: { + bodyJson: { paragraph }, + headers: { accept: 'application/json' }, + method: 'POST', + url: '/api/notebook/run' + }, + ...extra + } + }); + const adapter = createPlaywrightFixtureAdapter({ + records: [ + restRecord(1, 'request', 'one'), + restRecord(2, 'response', 'one', { + bodyJson: { ran: 'one' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }) + ], + version: fixtureVersion + }); + + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const call = paragraph => + routeHandler( + { fulfill: async () => undefined }, + request('POST', 'http://127.0.0.1:8080/api/notebook/run', JSON.stringify({ paragraph })) + ); + + // Once the first route proves fixture drift, later routes must fail fast. + await assert.rejects(() => call('unexpected'), /REST fixture request mismatch/); + const outcome = await Promise.race([ + call('one').then( + () => 'resolved', + error => String(error) + ), + new Promise(resolve => setTimeout(() => resolve('hung'), 2000)) + ]); + assert.notEqual(outcome, 'hung', 'a route arriving after a fixture error must not hang'); + assert.match(outcome, /REST fixture request mismatch/); + assert.throws(() => adapter.assertComplete(), /REST fixture request mismatch/); +}); + +test('Playwright adapter rejects a response record whose request shape drifted before installation', () => { + const restRecord = (sequence, direction, paragraph, extra = {}) => ({ + kind: 'rest', + sequence, + rest: { + direction, + request: { + bodyJson: { paragraph }, + headers: { accept: 'application/json' }, + method: 'POST', + url: '/api/notebook/run' + }, + ...extra + } + }); + // A response whose request shape disagrees with its request record is stale fixture drift. + assert.throws( + () => + createPlaywrightFixtureAdapter({ + records: [ + restRecord(1, 'request', 'one'), + restRecord(2, 'response', 'one-edited', { + bodyJson: { ran: 'drifted' }, + headers: { 'content-type': 'application/json' }, + status: 200 + }) + ], + version: fixtureVersion + }), + /request has no response/ + ); +}); + +test('a captured fixture replays the request it was captured from', async () => { + // Normalization runs on both stored fixtures and live requests; it must be idempotent. + const noteTexts = [ + 'docker run -e PASSWORD=a -e TOKEN=b img', + 'tokens = text.split()', + 'tokenizer = AutoTokenizer.from_pretrained("bert")', + 'export TOKEN=abc', + 'principal = 1000' + ]; + + for (const text of noteTexts) { + const page = new EventEmitter(); + const recorder = createNotebookTransportRecorder(fixtureMetadata()); + recorder.install(page); + + const body = JSON.stringify({ paragraphs: [{ text }] }); + const live = request('PUT', 'http://127.0.0.1:8080/api/notebook/note-a', body); + page.emit('request', live); + emitCompletedResponse(page, response(live, 200, '{"status":"OK"}')); + await recorder.stop(); + + const adapter = createPlaywrightFixtureAdapter(recorder.snapshot()); + let routeHandler; + await adapter.install({ + route: (_pattern, handler) => { + routeHandler = handler; + }, + routeWebSocket: () => undefined + }); + + const fulfilled = []; + await routeHandler( + { + fulfill: async value => fulfilled.push(value.body) + }, + request('PUT', 'http://127.0.0.1:8080/api/notebook/note-a', body) + ); + assert.deepEqual(fulfilled, ['{"status":"OK"}'], `replay failed for note text: ${text}`); + adapter.assertComplete(); + } +}); + +test('normalization is idempotent so a fixture and a live request agree', () => { + const samples = [ + 'docker run -e PASSWORD=a -e TOKEN=b img', + 'tokens = text.split()', + 'tokens = ', + 'password=a token=b', + '/api/notebook/x?access_token=abc&view=stable', + '/api/notebook/x?ticket=%3Cticket%3E', + 'curl -H "Authorization: Bearer abc"', + '{"access_token":"abc","expires":3600}', + 'password: secret', + 'please buy a ticket: today', + 'principal = 1000' + ]; + + for (const sample of samples) { + const once = normalizeFixtureRecord(sample); + const twice = normalizeFixtureRecord(once); + assert.equal(twice, once, `normalization is not idempotent for ${JSON.stringify(sample)}`); + } +}); + +test('normalization stays linear on long dotted identifiers', () => { + // Guard against regex backtracking on minified input. + // Include : and = so samples exercise the name scan instead of its early return. + const samples = [ + 'org.apache.shiro.authc.credential.PasswordMatcher.doCredentialsMatch '.repeat(600), + 'a.token.'.repeat(5000), + 'org.apache.shiro.authc.credential.PasswordMatcher.doCredentialsMatch: x '.repeat(600), + 'a.token.: '.repeat(4000), + '{"spark.hadoop.fs.s3a.access.name":"value"},'.repeat(900), + 'accessToken = "abc" and '.repeat(2000) + ]; + + for (const sample of samples) { + const started = process.hrtime.bigint(); + normalizeFixtureRecord(sample); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + assert.ok(elapsedMs < 1000, `normalization took ${elapsedMs.toFixed(0)}ms for ${sample.length} chars`); + } +}); + +test('redaction masks every credential on a line and leaves the rest readable', () => { + const line = normalizeFixtureRecord('docker run -e PASSWORD=first -e TOKEN=second img'); + assert.doesNotMatch(line, /first|second/); + assert.equal(line, 'docker run -e PASSWORD= -e TOKEN= img'); +}); + +test('redaction rewrites only the credential and keeps the text around it', () => { + // Assert the exact output so redaction cannot delete surrounding text. + const cases = [ + ['Authorization: Bearer abc', 'Authorization: '], + ['authorization=Bearer x', 'authorization='], + ['Cookie: theme=dark', 'Cookie: '], + ['ticket=abc, note=x', 'ticket=, note=x'], + ['token=first second', 'token= second'], + ['PGPASSWORD=hunter2 psql -h db', 'PGPASSWORD= psql -h db'], + ['curl -H "X-Api-Key: abc"', 'curl -H "X-Api-Key: "'], + ['val accessToken = "eyJhbGciOi"', 'val accessToken = ""'], + ['postgres://user:pass@host/db', 'postgres://user:@host/db'], + ["if password == 'hunter2':", "if password == '':"], + ['token === "abc"', 'token === ""'] + ]; + for (const [input, expected] of cases) { + assert.equal(normalizeFixtureRecord(input), expected, `redaction changed more than the value of ${input}`); + } + + const untouched = [ + 'see org.apache.shiro.authz.Authorization.check(user) for details', + 'Cookie.parse(header)', + 'tokenizer = AutoTokenizer.from_pretrained("bert")', + 'tokens = text.split()', + 'max_tokens=100, temperature=0.2', + 'secretary=alice', + 'please buy a ticket: today', + // A comparison against a variable is code, not a credential. + 'if token == expected: pass', + 'https://host/path' + ]; + for (const input of untouched) { + assert.equal(normalizeFixtureRecord(input), input, `redaction damaged note text: ${input}`); + } +}); + +test('a numeric value is kept only where the name is also an everyday word', () => { + // `principal` is an authentication field and an accounting term; the rest are not. + assert.equal(normalizeFixtureRecord('principal = 1000'), 'principal = 1000'); + assert.equal(normalizeFixtureRecord('principal = s3cr3t'), 'principal = '); + for (const input of ['password=123456', 'token=482913', 'ticket=1234']) { + assert.doesNotMatch(normalizeFixtureRecord(input), /\d/, `${input} kept a numeric credential`); + } +}); + +test('validateFixture rejects an unsupported version, lost ordering and a missing payload', () => { + const restRecord = (sequence = 1) => ({ + kind: 'rest', + sequence, + rest: { + bodyJson: { id: 'note-a' }, + direction: 'response', + headers: { 'content-type': 'application/json' }, + request: { bodyRaw: '', headers: { accept: 'application/json' }, method: 'GET', url: '/api/notebook/note-a' }, + status: 200 + } + }); + + assert.deepEqual(validateFixture({ metadata: fixtureMetadata(), records: [restRecord()], version: 999 }), [ + 'Unsupported fixture version 999' + ]); + + assert.deepEqual( + validateFixture({ + metadata: fixtureMetadata(), + records: [restRecord(2), restRecord(1)], + version: fixtureVersion + }), + ['records[1].sequence must increase without reordering'] + ); + + assert.deepEqual( + validateFixture({ + metadata: fixtureMetadata(), + records: [{ kind: 'websocket', sequence: 1, websocket: { direction: 'receive' } }], + version: fixtureVersion + }), + ['records[0].websocket payloadText or payloadBase64 is required to preserve message shape'] + ); +}); + +test("a value is only left alone when the whole value is that field's placeholder", () => { + // Prefix-style placeholder matching could hide a real credential. + assert.equal(normalizeFixtureRecord('password=REAL_SECRET'), 'password='); + assert.equal(normalizeFixtureRecord('password=%3Cprivate%3E'), 'password='); + assert.equal(normalizeFixtureRecord('https://u:evil@h/db'), 'https://u:@h/db'); + assert.equal(normalizeFixtureRecord('?apiKey=%3CapiKey%3EMORE&view=stable'), '?apiKey=&view=stable'); + // URL placeholders may be percent-encoded; both forms must stay idempotent. + assert.equal(normalizeFixtureRecord('?apiKey=%3CapiKey%3E&view=stable'), '?apiKey=%3CapiKey%3E&view=stable'); + assert.equal(normalizeFixtureRecord('password='), 'password='); +}); + +test('a quoted credential is masked whole whatever the name around the sensitive word', () => { + // Quoted values must use the same credential-name rules as object fields. + // Names such as passwordHash and aws_secret_access_key must still mask the entire quoted value. + assert.equal(normalizeFixtureRecord('secretKey="first second"'), 'secretKey=""'); + assert.equal(normalizeFixtureRecord('aws_secret_access_key="first second"'), 'aws_secret_access_key=""'); + assert.equal(normalizeFixtureRecord("passwordHash='a b c'"), "passwordHash=''"); + assert.equal(normalizeFixtureRecord('tokenizer="first second"'), 'tokenizer="first second"'); +}); + +test('a json websocket frame keeps its note text and replays its own capture', () => { + // JSON WebSocket frames are redacted by key before stringification; + // scanning the result as text corrupts note content and can break replay. + const frames = [ + '{"op":"RUN_PARAGRAPH","data":{"paragraph":"ticketCount = df.count()"}}', + '{"op":"RUN_PARAGRAPH","data":{"paragraph":"SELECT * FROM t WHERE ticket_id = 42"}}', + '{"op":"RUN_PARAGRAPH","data":{"paragraph":"const cookieBanner = document.getElementById(\\"x\\")"}}' + ]; + + for (const frame of frames) { + const sanitized = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [{ kind: 'websocket', sequence: 1, websocket: { direction: 'send', payloadText: frame } }], + version: fixtureVersion + }); + const stored = sanitized.records[0].websocket.payloadText; + + assert.doesNotThrow(() => JSON.parse(stored), `fixture holds invalid json for ${frame}`); + assert.equal(JSON.parse(stored).data.paragraph, JSON.parse(frame).data.paragraph, 'note text was rewritten'); + assert.equal(webSocketPayloadMatches(stored, frame), true, 'the frame no longer replays its own capture'); + } + + // Keyed JSON and raw frames exercise different redaction paths. + const masked = fixtureModule.sanitizeFixture({ + metadata: fixtureMetadata(), + records: [ + { kind: 'websocket', sequence: 1, websocket: { direction: 'send', payloadText: '{"ticket":"abc"}' } }, + { kind: 'websocket', sequence: 2, websocket: { direction: 'send', payloadText: 'token=abc' } } + ], + version: fixtureVersion + }); + assert.equal(masked.records[0].websocket.payloadText, '{"ticket":""}'); + assert.equal(masked.records[1].websocket.payloadText, 'token='); +}); + +test('a string inside an array keeps its text, like a string under a key', () => { + assert.deepEqual(normalizeFixtureRecord({ tags: ['ticket = 1', 'password = 2'] }), { + tags: ['ticket = 1', 'password = 2'] + }); + assert.deepEqual(normalizeFixtureRecord({ ticket: ['a', 'b'] }), { ticket: '' }); +}); + +test('a masked name holding a container is treated by why the name is masked', () => { + // A note-authored field named time is content; masking it as volatile metadata would change fixture shape. + assert.deepEqual(normalizeFixtureRecord({ settings: { forms: { time: { name: 'time', defaultValue: '9' } } } }), { + settings: { forms: { time: { name: 'time', defaultValue: '9' } } } + }); + assert.deepEqual(normalizeFixtureRecord({ credentials: { apiKey: 'k', note: 'keep' } }), { + credentials: '' + }); + assert.deepEqual(normalizeFixtureRecord({ authenticationInfo: { user: { name: 'alice' } } }), { + authenticationInfo: { user: '' } + }); + assert.deepEqual(normalizeFixtureRecord({ time: '2026-09-08' }), { time: '