diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index aadec63cf..9376cf876 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -318,6 +318,9 @@ jobs: if-no-files-found: warn mysql-execution: + # Named explicitly so the added native-tests-b combinations read like the + # others instead of listing every matrix value. + name: mysql-execution (${{ matrix.label }}, ${{ matrix.part }}) needs: published-sdk-adapters runs-on: ubuntu-latest # Each part takes about 20 to 35 minutes; the fixture step runs every @@ -328,8 +331,10 @@ jobs: # Each database runs in six parts on separate runners: the native # database suites and TLS checks, the shared corpus, its profiles and # restart cases, the published-SDK direction profiles, the corrected - # marker cohort, and the other cohorts. Every part bootstraps the same - # database and native build; no part depends on another. + # marker cohort, and the other cohorts. The two MySQL 8 labels add a + # seventh, native-tests-b, for half their execution tests. Every part + # bootstraps the same database and native build; no part depends on + # another. matrix: label: [mysql-8.0, existing-php-mysql-matrix, mariadb-10.11] part: [native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers] @@ -338,16 +343,35 @@ jobs: image: mirror.gcr.io/library/mysql@sha256:62fb722c78b24245ddff1796a0fcee4a49cc5b87e0aaaf20c92d1da9e0a2497b # The expanded original was OOM-killed at 1 GiB during native pressure. memory: 2g + # The two MySQL 8 labels' native suites were the slowest parts + # (about 21 min each, most of it the execution tests), so their + # execution tests run in two halves: native-tests runs the first + # and native-tests-b the second. The bytewise-sorted test list is + # dealt round-robin, so each listed test runs in exactly one half + # and a new test joins one. + execution_shards: 2 + - label: mysql-8.0 + part: native-tests-b + image: mirror.gcr.io/library/mysql@sha256:62fb722c78b24245ddff1796a0fcee4a49cc5b87e0aaaf20c92d1da9e0a2497b + memory: 2g + execution_shards: 2 - label: existing-php-mysql-matrix image: mirror.gcr.io/library/mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 # The expanded shared database was OOM-killed at 1 GiB before TLS checks. memory: 2g + execution_shards: 2 + - label: existing-php-mysql-matrix + part: native-tests-b + image: mirror.gcr.io/library/mysql@sha256:679e7e924f38a3cbb62a3d7df32924b83f7321a602d3f9f967c01b3df18495d6 + memory: 2g + execution_shards: 2 - label: mariadb-10.11 image: mirror.gcr.io/library/mariadb@sha256:7db29378d4fdab73f8123bbc2b48905c90d1a4b00cf848b028f1e81e623257f2 # Both expanded originals exhausted the 512 MiB DB fixture ceiling. memory: 1g env: DW_PARITY_PART: ${{ matrix.part }} + DW_EXECUTION_SHARDS: ${{ matrix.execution_shards || 1 }} RESOURCE_SCOPE: dw-mysql-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-${{ matrix.label }} MYSQL_IMAGE: ${{ matrix.image }} DB_MEMORY: ${{ matrix.memory }} @@ -461,13 +485,13 @@ jobs: "$layout" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt # Compiling the library tests with two jobs peaks near 4.2 GB; a 4 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=6g --memory-swap=6g \ - -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 -e DW_PARITY_PART \ + -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 -e DW_PARITY_PART -e DW_EXECUTION_SHARDS \ -e DW_TEST_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \ -e DW_EXECUTION_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \ -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \ -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \ mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 -c \ - 'if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt && cargo test --locked --offline --test execution -- --nocapture --test-threads=2 > /evidence/mysql-execution-tests.txt; fi && cargo build --locked --offline' + 'case "$DW_PARITY_PART" in native-tests) shard=1 suffix= ;; native-tests-b) shard=2 suffix=-b ;; *) shard= ;; esac; if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt || exit 1; fi; if [ -n "$shard" ]; then cargo test --locked --offline --test execution -- --list --format terse | sed -n "s/: test\$//p" | LC_ALL=C sort | awk -v count="$DW_EXECUTION_SHARDS" -v shard="$shard" "(NR - 1) % count == shard - 1" > /evidence/execution-shard$suffix.txt && test -s /evidence/execution-shard$suffix.txt && cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 $(cat /evidence/execution-shard$suffix.txt) > /evidence/mysql-execution-tests$suffix.txt || exit 1; fi; cargo build --locked --offline' docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=native_ref \ -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ @@ -727,25 +751,45 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress # Each profile brings its own published PHP and embedded databases and - # shares this part's native node, as the profiles shared the corpus's. - docker run -d --name "$RESOURCE_SCOPE-native-a" --network "$RESOURCE_SCOPE" --network-alias=parity-native-a \ - --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ - -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ - -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=native_ref \ + # shares its lane's native node, as the profiles shared the corpus's. + # Node b has its own database, so the namespace-signal profile can run + # beside the other two signal profiles, whose task queue it shares. + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --entrypoint mysql \ + "$MYSQL_IMAGE" -h parity-mysql -uroot -pparity-disposable-password \ + -e 'CREATE DATABASE native_ref_b CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ + -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=native_ref_b \ -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ - -v "$PWD/rust-target:/target:ro" --tmpfs /state --entrypoint /target/debug/durable-workflow-server \ - mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 - ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then - ready=1 - break - fi - sleep 1 + -v "$PWD/rust-target:/target:ro" --entrypoint /target/debug/durable-workflow-server \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ + schema-bootstrap > parity-evidence/sdk-native-b-bootstrap.json + for node in a b; do + database=native_ref + [ "$node" = b ] && database=native_ref_b + docker run -d --name "$RESOURCE_SCOPE-native-$node" --network "$RESOURCE_SCOPE" --network-alias="parity-native-$node" \ + --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ + -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ + -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE="$database" \ + -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ + -v "$PWD/rust-target:/target:ro" --tmpfs /state --entrypoint /target/debug/durable-workflow-server \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 done - test "$ready" = 1 - bash scripts/conformance/server-parity/sdk-direction-lanes.sh mysql http://parity-native-a:8080 + # Literal node URLs keep each probe's target a fixed container name. + for url in http://parity-native-a:8080 http://parity-native-b:8080; do + name=${url#*//parity-} + name=${name%:*} + ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ + -fsS --connect-timeout 2 --max-time 5 "$url/api/ready" > "parity-evidence/sdk-$name-readiness.json"; then + ready=1 + break + fi + sleep 1 + done + test "$ready" = 1 + done + bash scripts/conformance/server-parity/sdk-direction-lanes.sh mysql http://parity-native-a:8080 http://parity-native-b:8080 - name: Start the shared native node the cohorts use id: prepare_cohorts if: ${{ matrix.part == 'cohorts' }} @@ -1329,25 +1373,44 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress # Each profile brings its own published PHP and embedded databases and - # shares this part's native node, as the profiles shared the corpus's. - docker run -d --name "$RESOURCE_SCOPE-native-a" --network "$RESOURCE_SCOPE" --network-alias=parity-native-a \ - --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ - -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ - -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=native_ref \ + # shares its lane's native node, as the profiles shared the corpus's. + # Node b has its own database, so the namespace-signal profile can run + # beside the other two signal profiles, whose task queue it shares. + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ + -e PGPASSWORD=parity-disposable-password --entrypoint createdb "$POSTGRES_IMAGE" -h parity-pg -U parity native_ref_b + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ + -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=native_ref_b \ -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ - -v "$PWD/rust-target:/target:ro" --tmpfs /state --entrypoint /target/debug/durable-workflow-server \ - mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 - ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS --connect-timeout 2 --max-time 5 http://parity-native-a:8080/api/ready > parity-evidence/sdk-native-readiness.json; then - ready=1 - break - fi - sleep 1 + -v "$PWD/rust-target:/target:ro" --entrypoint /target/debug/durable-workflow-server \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ + schema-bootstrap > parity-evidence/sdk-native-b-bootstrap.json + for node in a b; do + database=native_ref + [ "$node" = b ] && database=native_ref_b + docker run -d --name "$RESOURCE_SCOPE-native-$node" --network "$RESOURCE_SCOPE" --network-alias="parity-native-$node" \ + --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ + -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token -e DW_BIND_ADDRESS=0.0.0.0:8080 \ + -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE="$database" \ + -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ + -v "$PWD/rust-target:/target:ro" --tmpfs /state --entrypoint /target/debug/durable-workflow-server \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 done - test "$ready" = 1 - bash scripts/conformance/server-parity/sdk-direction-lanes.sh pgsql http://parity-native-a:8080 + # Literal node URLs keep each probe's target a fixed container name. + for url in http://parity-native-a:8080 http://parity-native-b:8080; do + name=${url#*//parity-} + name=${name%:*} + ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ + -fsS --connect-timeout 2 --max-time 5 "$url/api/ready" > "parity-evidence/sdk-$name-readiness.json"; then + ready=1 + break + fi + sleep 1 + done + test "$ready" = 1 + done + bash scripts/conformance/server-parity/sdk-direction-lanes.sh pgsql http://parity-native-a:8080 http://parity-native-b:8080 - name: Start the shared native node the cohorts use id: prepare_cohorts if: ${{ matrix.part == 'cohorts' }} @@ -1944,25 +2007,34 @@ jobs: setfacl -m u:1000:rwx,d:u:1000:rwx parity-state parity-evidence docker network create "$RESOURCE_SCOPE" # Each profile brings its own published PHP and embedded databases and - # shares this part's native node, as the profiles shared the corpus's. - docker run -d --name "$RESOURCE_SCOPE-rust" --network "$RESOURCE_SCOPE" --network-alias=parity-rust \ - --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ - -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token \ - -e DB_DATABASE=/state/rust.sqlite -e DW_BIND_ADDRESS=0.0.0.0:8080 \ - -v "$PWD/rust-target:/target:ro" -v "$PWD/parity-state:/state" \ - --entrypoint /target/debug/durable-workflow-server \ - mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 - ready=0 - for attempt in $(seq 1 30); do - if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ - -fsS --connect-timeout 2 --max-time 5 http://parity-rust:8080/api/ready > parity-evidence/sdk-native-readiness.json; then - ready=1 - break - fi - sleep 1 + # shares its lane's native node, as the profiles shared the corpus's. + # Node rust-b has its own database file, so the namespace-signal profile + # can run beside the other two signal profiles, whose task queue it shares. + for node in rust rust-b; do + docker run -d --name "$RESOURCE_SCOPE-$node" --network "$RESOURCE_SCOPE" --network-alias="parity-$node" \ + --user="$(id -u):$(id -g)" --cpus=1 --memory=256m --memory-swap=256m \ + -e DW_RUST_EXPERIMENTAL=1 -e DW_AUTH_TOKEN=parity-test-token \ + -e DB_DATABASE="/state/$node.sqlite" -e DW_BIND_ADDRESS=0.0.0.0:8080 \ + -v "$PWD/rust-target:/target:ro" -v "$PWD/parity-state:/state" \ + --entrypoint /target/debug/durable-workflow-server \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 done - test "$ready" = 1 - bash scripts/conformance/server-parity/sdk-direction-lanes.sh sqlite http://parity-rust:8080 + # Literal node URLs keep each probe's target a fixed container name. + for url in http://parity-rust:8080 http://parity-rust-b:8080; do + name=${url#*//parity-} + name=${name%:*} + ready=0 + for attempt in $(seq 1 30); do + if docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint curl "$image" \ + -fsS --connect-timeout 2 --max-time 5 "$url/api/ready" > "parity-evidence/sdk-$name-readiness.json"; then + ready=1 + break + fi + sleep 1 + done + test "$ready" = 1 + done + bash scripts/conformance/server-parity/sdk-direction-lanes.sh sqlite http://parity-rust:8080 http://parity-rust-b:8080 - name: Start the shared native node the cohorts use id: prepare_cohorts if: ${{ matrix.part == 'cohorts' }} @@ -2116,6 +2188,7 @@ jobs: docker logs "$RESOURCE_SCOPE-server" > parity-evidence/server.log 2>&1 || true docker logs "$RESOURCE_SCOPE-php-maintenance" > parity-evidence/php-maintenance.log 2>&1 || true docker logs "$RESOURCE_SCOPE-rust" > parity-evidence/rust.log 2>&1 || true + docker logs "$RESOURCE_SCOPE-rust-b" > parity-evidence/rust-b.log 2>&1 || true docker logs "$RESOURCE_SCOPE-recovered" > parity-evidence/rust-recovered.log 2>&1 || true docker inspect --format '{{json .State}}' "$RESOURCE_SCOPE-rust" > parity-evidence/rust-state.json 2>&1 || true docker inspect --format '{{json .State}}' "$RESOURCE_SCOPE-recovered" > parity-evidence/rust-recovered-state.json 2>&1 || true @@ -2146,6 +2219,7 @@ jobs: docker rm -f "$RESOURCE_SCOPE-server" || true docker rm -f "$RESOURCE_SCOPE-php-maintenance" || true docker rm -f "$RESOURCE_SCOPE-rust" || true + docker rm -f "$RESOURCE_SCOPE-rust-b" || true docker rm -f "$RESOURCE_SCOPE-recovered" || true docker network rm "$RESOURCE_SCOPE" || true - name: Summarize fixture evidence diff --git a/scripts/conformance/server-parity/failure-excerpt.mjs b/scripts/conformance/server-parity/failure-excerpt.mjs index 61a143c83..be561541b 100644 --- a/scripts/conformance/server-parity/failure-excerpt.mjs +++ b/scripts/conformance/server-parity/failure-excerpt.mjs @@ -2,7 +2,9 @@ // in observation order, so a flake can be diagnosed from the retained artifact // without keeping the record. It names each request's method, path, status, // transport flags, worker and poll identity. Request and response bodies, -// payloads and credentials are never read into it. +// payloads and credentials are never read into it. Stored workflow task rows +// are listed by their counters and clock fields too, so a reference repair +// pass that touched a task during a phase shows which task and when. const SKIP = new Set(['request', 'response', 'request_body', 'response_body', 'response_bytes_base64', 'transport_jsonl', 'stdout', 'stderr', 'fixture']); @@ -20,6 +22,21 @@ function line(path, item) { return fields.join(' '); } +// The task fields a stored-state line names: identities, enums, counters and +// clocks. Payloads, errors and leases' owners are left out. +const TASK_FIELDS = ['workflow_run_id', 'task_type', 'status', 'attempt_count', 'repair_count', 'available_at', + 'leased_at', 'lease_expires_at', 'last_dispatched_at', 'last_claim_failed_at', 'repair_available_at', + 'created_at', 'updated_at']; + +function taskLine(path, row) { + const fields = [`${path} task ${typeof row.id === 'string' || typeof row.id === 'number' ? row.id : '?'}`]; + for (const key of TASK_FIELDS) { + const value = row[key]; + if (typeof value === 'string' || typeof value === 'number') fields.push(`${key}=${String(value).slice(0, 64)}`); + } + return fields.join(' '); +} + function walk(value, path, out, limit) { if (out.length >= limit || value === null || typeof value !== 'object') return; if (Array.isArray(value)) { @@ -29,8 +46,15 @@ function walk(value, path, out, limit) { if (typeof value.method === 'string' && typeof value.path === 'string' && Object.hasOwn(value, 'status')) { out.push(line(path, value)); } + const tasks = value.tables?.workflow_tasks?.rows; + if (Array.isArray(tasks)) { + for (const [index, row] of tasks.entries()) { + if (out.length >= limit) return; + if (row && typeof row === 'object') out.push(taskLine(`${path}.tables.workflow_tasks[${index}]`, row)); + } + } for (const [key, child] of Object.entries(value)) { - if (!SKIP.has(key)) walk(child, `${path}.${key}`, out, limit); + if (!SKIP.has(key) && key !== 'tables') walk(child, `${path}.${key}`, out, limit); } } diff --git a/scripts/conformance/server-parity/sdk-direction-lanes.sh b/scripts/conformance/server-parity/sdk-direction-lanes.sh index d083f81a5..668d5ac27 100755 --- a/scripts/conformance/server-parity/sdk-direction-lanes.sh +++ b/scripts/conformance/server-parity/sdk-direction-lanes.sh @@ -1,36 +1,43 @@ #!/usr/bin/env bash -# Run every published-SDK direction profile against one native server, in -# three concurrent lanes. Each profile already has its own containers, PHP and -# embedded databases, evidence and state directory, and its own workflow-ID -# prefix on the shared native server, so profiles in different lanes cannot -# see each other unless they share a task queue. Within a lane they run in order, and each profile's PHP -# server is stopped as soon as its attempt ends, so at most three run at once. +# Run every published-SDK direction profile in three concurrent lanes, against +# two independent native servers, each with its own database. Each profile +# already has its own containers, PHP and embedded databases, evidence and +# state directory, and its own workflow-ID prefix, so profiles in different +# lanes cannot see each other unless they share a task queue on one server. +# Within a lane they run in order, and each profile's PHP server is stopped as +# soon as its attempt ends, so at most three run at once. # Every profile runs even after another fails; the script fails if any did, # after printing each log. The job's sdk-direction-cleanup steps remove every # profile's containers afterwards, as before. set -euo pipefail backend="${1:?Choose mysql, pgsql or sqlite}" native_url="${2:?Set the independently isolated native URL}" +second_native_url="${3:?Set the second native URL, a server with its own database}" if [[ -z "${RESOURCE_SCOPE:-}" ]]; then echo 'Set the owning isolated network/resource scope' >&2 exit 2 fi evidence=parity-evidence/sdk-direction-lanes mkdir -p "$evidence" -# Profile variant and its tag (evidence, log and container names). Profiles -# that share a task queue on the native server must share a lane: the three -# signal profiles all use server-parity-sdk-signals, and one's workers would -# show in another's routing diagnostics. Every other profile has its own queues. +# Each lane names its server ("a" is the first URL, "b" the second), then each +# profile variant and its tag (evidence, log and container names). Profiles +# that share a task queue on one server must share a lane: the three signal +# profiles all use server-parity-sdk-signals, and one's workers would show in +# another's routing diagnostics. The namespace-signal profile therefore runs on +# server b, so the signal profiles no longer form one long lane. Every other +# profile has its own queues. The lanes are balanced by measured durations. lanes=( - "update:sdk-updates update-failure:sdk-update-failures child:sdk-child-directions" - "signal:sdk-signals namespace-signal:sdk-namespace-signals selected-run-signal:sdk-selected-run-signals" - "timer:sdk-timers activity:sdk-directions query:sdk-queries" + "a signal:sdk-signals selected-run-signal:sdk-selected-run-signals query:sdk-queries" + "b namespace-signal:sdk-namespace-signals timer:sdk-timers activity:sdk-directions" + "a update:sdk-updates update-failure:sdk-update-failures child:sdk-child-directions" ) run_lane() { - local lane=$1 entry variant tag status started + local server=${1%% *} lane=${1#* } entry variant tag status started url + url=$native_url + [[ $server == b ]] && url=$second_native_url for entry in $lane; do variant=${entry%%:*} tag=${entry#*:} status=0 started=$SECONDS - bash scripts/conformance/server-parity/sdk-direction-profile.sh "$backend" "$native_url" "$variant" \ + bash scripts/conformance/server-parity/sdk-direction-profile.sh "$backend" "$url" "$variant" \ > "$evidence/$tag.log" 2>&1 || status=$? printf '%s\n' "$status" > "$evidence/$tag.status" # Each profile's duration, so lane balance can be measured from the log. @@ -47,7 +54,7 @@ done wait failed=0 for lane in "${lanes[@]}"; do - for entry in $lane; do + for entry in ${lane#* }; do tag=${entry#*:} status=$(cat "$evidence/$tag.status" 2>/dev/null || echo missing) seconds=$(cat "$evidence/$tag.seconds" 2>/dev/null || echo unknown) diff --git a/tests/Unit/ServerParityFailureExcerptTest.mjs b/tests/Unit/ServerParityFailureExcerptTest.mjs index 38417c57a..5fbe8cce1 100644 --- a/tests/Unit/ServerParityFailureExcerptTest.mjs +++ b/tests/Unit/ServerParityFailureExcerptTest.mjs @@ -37,3 +37,17 @@ test('the outline is bounded per case and in total', () => { test('a case without an observation still names its error', () => { assert.equal(failureExcerpt([{fixture_id: 'x', outcome: 'runner-blocked', error: 'boom'}]), '== x (runner-blocked): boom\n'); }); + +test('stored workflow tasks are listed by counters and clocks, never payloads or errors', () => { + const text = failureExcerpt([{fixture_id: 'signal-selected-run-sdk-rust-php', outcome: 'product-fail', error: '1 !== 0', + observation: {mode: 'embedded', pairs: [{final_stored: {tables: { + workflow_tasks: {rows: [{id: '01TASK', workflow_run_id: '01RUN', task_type: 'workflow', status: 'completed', + attempt_count: 1, repair_count: 1, available_at: '2026-10-25 06:40:01', last_dispatched_at: '2026-10-25 06:40:07', + payload: {secret: 'payload-bytes'}, last_error: 'error-text', lease_owner: 'owner-name'}]}, + workflow_commands: {rows: [{method: 'POST', path: '/not-a-request', status: 200}]}}}}]}}]); + assert.equal(text, [ + '== signal-selected-run-sdk-rust-php (product-fail): 1 !== 0', + 'observation.pairs[0].final_stored.tables.workflow_tasks[0] task 01TASK workflow_run_id=01RUN task_type=workflow status=completed attempt_count=1 repair_count=1 available_at=2026-10-25 06:40:01 last_dispatched_at=2026-10-25 06:40:07', + ''].join('\n')); + assert(!text.includes('payload-bytes') && !text.includes('error-text') && !text.includes('owner-name') && !text.includes('/not-a-request')); +}); diff --git a/tests/Unit/ServerParityPartsTest.mjs b/tests/Unit/ServerParityPartsTest.mjs index 1cf74c2da..bc707fddf 100644 --- a/tests/Unit/ServerParityPartsTest.mjs +++ b/tests/Unit/ServerParityPartsTest.mjs @@ -6,6 +6,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; +import {spawnSync} from 'node:child_process'; const workflow = readFileSync('.github/workflows/server-parity.yml', 'utf8'); const PARTS = ['native-tests', 'corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers']; @@ -116,16 +117,55 @@ for (const [name, key] of [['mysql-execution', 'Compare three runtimes'], ['post }); } -test('the database bootstraps run the native suites only in the native-tests part', () => { - for (const name of ['mysql-execution', 'postgres-storage']) { - const bootstrap = steps(job(name)).find(step => step.name.startsWith('Bootstrap independent ')); - assert.match(bootstrap.text, /-e DW_PARITY_PART \\\n/, name); - assert.match(bootstrap.text, /'if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']* --test execution [^']*; fi && cargo build --locked --offline'/, name); +test('the database bootstraps run the native suites only in the native-tests parts', () => { + const postgres = steps(job('postgres-storage')).find(step => step.name.startsWith('Bootstrap independent ')); + assert.match(postgres.text, /-e DW_PARITY_PART \\\n/); + assert.match(postgres.text, /'if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']* --test execution [^']*; fi && cargo build --locked --offline'/); + const mysql = steps(job('mysql-execution')).find(step => step.name.startsWith('Bootstrap independent ')); + assert.match(mysql.text, /-e DW_PARITY_PART -e DW_EXECUTION_SHARDS \\\n/); + // native-tests runs the database suite and the first half of the execution + // tests; native-tests-b, where a label has it, runs the second half. + assert.match(mysql.text, /'case "\$DW_PARITY_PART" in native-tests\) shard=1 suffix= ;; native-tests-b\) shard=2 suffix=-b ;; \*\) shard= ;; esac; if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']*runtime::mysql::[^']* > \/evidence\/mysql-tests\.txt \|\| exit 1; fi; if \[ -n "\$shard" \]; then [^']*cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 \$\(cat \/evidence\/execution-shard\$suffix\.txt\) > \/evidence\/mysql-execution-tests\$suffix\.txt \|\| exit 1; fi; cargo build --locked --offline'/); +}); + +test('the MySQL 8 labels split their execution tests into two parts, every test in exactly one', () => { + const block = job('mysql-execution'); + assert.match(block, /^ {6}DW_EXECUTION_SHARDS: \$\{\{ matrix\.execution_shards \|\| 1 \}\}$/m); + // Matrix values come from the label's include entry; the added + // native-tests-b combination names its own, the same ones. + const include = block.slice(block.indexOf('\n include:\n'), block.indexOf('\n env:\n')); + const entries = include.split(/\n {10}- /).slice(1).map(text => Object.fromEntries( + [...('label: ' + text.replace(/^label: /, '')).matchAll(/^\s*([a-z_]+): (\S+)$/gm)].map(match => match.slice(1)))); + const sharded = ['mysql-8.0', 'existing-php-mysql-matrix']; + for (const name of sharded) { + const pair = entries.filter(entry => entry.label === name); + assert.equal(pair.length, 2, JSON.stringify(entries)); + const [label, half] = pair; + assert.equal(label.part, undefined); + assert.equal(label.execution_shards, '2'); + assert.deepEqual(half, {...label, part: 'native-tests-b'}); } + assert.ok(entries.filter(entry => !sharded.includes(entry.label)).every(entry => !entry.execution_shards && !entry.part)); + // The selection the bootstrap runs: list, sort bytewise, deal round-robin. + const bootstrap = steps(block).find(step => step.name.startsWith('Bootstrap independent ')).text; + const selection = /cargo test --locked --offline --test execution -- --list --format terse \| sed -n "s\/: test\\\$\/\/p" \| LC_ALL=C sort \| awk -v count="\$DW_EXECUTION_SHARDS" -v shard="\$shard" "([^"]+)" > \/evidence\/execution-shard\$suffix\.txt && test -s \/evidence\/execution-shard\$suffix\.txt && /.exec(bootstrap); + assert.ok(selection, 'the execution tests are selected by the listed round-robin'); + const names = Array.from({length: 489}, (_, index) => `suite::test_${String(index).padStart(3, '0')}`); + const deal = (shard, count) => { + const run = spawnSync('awk', ['-v', `count=${count}`, '-v', `shard=${shard}`, selection[1]], {input: names.join('\n') + '\n', encoding: 'utf8'}); + assert.equal(run.status, 0, run.stderr); + return run.stdout.trim().split('\n'); + }; + const halves = [deal(1, 2), deal(2, 2)]; + assert.deepEqual([...halves[0], ...halves[1]].sort(), names); + assert.equal(new Set([...halves[0], ...halves[1]]).size, names.length); + assert.ok(Math.abs(halves[0].length - halves[1].length) <= 1); + // A label with one shard runs the whole suite in native-tests. + assert.deepEqual(deal(1, 1), names); }); test('every readiness probe bounds each request, so one silent connection cannot stall a part', () => { const probes = workflow.match(/curl[^\n]*\n?[^\n]*\/api\/ready/g) ?? []; assert.ok(probes.length >= 20, `${probes.length} readiness probes`); - for (const probe of probes) assert.match(probe, /--connect-timeout \d+ --max-time \d+ "?http:/, probe); + for (const probe of probes) assert.match(probe, /--connect-timeout \d+ --max-time \d+ "?(http:|\$url\/)/, probe); }); diff --git a/tests/Unit/ServerParitySdkDirectionLanesTest.mjs b/tests/Unit/ServerParitySdkDirectionLanesTest.mjs index 4970acc80..ce45deba3 100644 --- a/tests/Unit/ServerParitySdkDirectionLanesTest.mjs +++ b/tests/Unit/ServerParitySdkDirectionLanesTest.mjs @@ -11,6 +11,9 @@ const PROFILES = { 'namespace-signal': 'sdk-namespace-signals', timer: 'sdk-timers', activity: 'sdk-directions', 'selected-run-signal': 'sdk-selected-run-signals', signal: 'sdk-signals', query: 'sdk-queries', }; +// The profiles that run against the second native server. +const SECOND_SERVER = new Set(['namespace-signal', 'timer', 'activity']); +const URLS = {a: 'http://native-a:8080', b: 'http://native-b:8080'}; // Runs the wrapper in a scratch tree whose profile script and `docker` are // stubs that append to one event log: a profile starts its PHP server and @@ -43,7 +46,7 @@ fail="STOP_FAIL_\${tag//-/_}" exit "\${!fail:-0}" `); chmodSync(join(bin, 'docker'), 0o755); - const child = spawnSync('bash', ['scripts/conformance/server-parity/sdk-direction-lanes.sh', 'sqlite', 'http://native:8080'], { + const child = spawnSync('bash', ['scripts/conformance/server-parity/sdk-direction-lanes.sh', 'sqlite', URLS.a, URLS.b], { cwd: root, encoding: 'utf8', timeout: 30000, env: {...process.env, ...environment, PATH: `${bin}:${process.env.PATH}`, EVENTS: events, RESOURCE_SCOPE: 'dw-scope-1'}, }); @@ -57,15 +60,16 @@ exit "\${!fail:-0}" const started = (events, variant) => events.findIndex(event => event.startsWith(`start ${variant} `)); -// Each profile ran once against the given backend and URL, and its PHP server -// was stopped after the profile ended and before the lane's next profile. +// Each profile ran once against the given backend and its lane's server, and +// its PHP server was stopped after the profile ended and before the lane's +// next profile. function assertEveryServerRetired(events) { const running = new Set(); let peak = 0; for (const event of events) { const [kind, ...rest] = event.split(' '); if (kind === 'start') { - assert.deepEqual(rest.slice(1), ['sqlite', 'http://native:8080'], event); + assert.deepEqual(rest.slice(1), ['sqlite', SECOND_SERVER.has(rest[0]) ? URLS.b : URLS.a], event); running.add(PROFILES[rest[0]]); peak = Math.max(peak, running.size); } else if (kind === 'docker') { @@ -124,21 +128,38 @@ test('a failed profile and a failed stop of its server keep the profile status', assert.match(child.stderr, /^::error::SDK direction profile sdk-queries exited 3$/m); }); +test('the wrapper refuses to run without a second native server', () => { + const child = spawnSync('bash', [LANES, 'sqlite', URLS.a], { + encoding: 'utf8', env: {...process.env, RESOURCE_SCOPE: 'dw-scope-1'}, cwd: tmpdir(), + }); + assert.notEqual(child.status, 0); + assert.match(child.stderr, /Set the second native URL, a server with its own database/); +}); + test('the wrapper refuses to run without an owning resource scope', () => { - const child = spawnSync('bash', [LANES, 'sqlite', 'http://native:8080'], { + const child = spawnSync('bash', [LANES, 'sqlite', URLS.a, URLS.b], { encoding: 'utf8', env: {...process.env, RESOURCE_SCOPE: ''}, cwd: tmpdir(), }); assert.equal(child.status, 2); assert.match(child.stderr, /Set the owning isolated network\/resource scope/); }); -// The native server is shared by every lane. Two profiles whose fixtures name -// the same task queue would see each other's workers, so they share a lane. -test('profiles that share a task queue run in the same lane', () => { +// A native server is shared by the lanes that name it. Two profiles whose +// fixtures name the same task queue would see each other's workers on one +// server, so they share a lane or run on different servers. +test('profiles that share a task queue on one server run in the same lane', () => { const script = readFileSync(LANES, 'utf8'); - const lanes = [...script.slice(script.indexOf('lanes=(')).split(')')[0].matchAll(/^ {2}"([^"]+)"$/gm)] - .map(match => match[1].split(' ').map(entry => entry.split(':')[1])); + const parsed = [...script.slice(script.indexOf('lanes=(')).split(')')[0].matchAll(/^ {2}"([^"]+)"$/gm)] + .map(match => match[1].split(' ')); + const servers = parsed.map(([server]) => server); + assert.deepEqual([...new Set(servers)].sort(), ['a', 'b'], 'each lane names server a or b'); + const lanes = parsed.map(([, ...entries]) => entries.map(entry => entry.split(':')[1])); assert.deepEqual(lanes.flat().sort(), Object.values(PROFILES).sort(), 'every profile in exactly one lane'); + for (const [index, lane] of parsed.entries()) { + for (const entry of lane.slice(1)) { + assert.equal(SECOND_SERVER.has(entry.split(':')[0]), servers[index] === 'b', `${entry} runs on its declared server`); + } + } const queues = tag => { const directory = new URL(`../Fixtures/ServerParityProfiles/published-${tag}/`, import.meta.url); const text = readdirSync(directory).filter(name => name.endsWith('.json')) @@ -146,7 +167,8 @@ test('profiles that share a task queue run in the same lane', () => { return new Set(text.match(/server-parity-sdk-[a-z-]+/g) ?? []); }; for (const [index, lane] of lanes.entries()) { - for (const other of lanes.slice(index + 1)) { + for (const [offset, other] of lanes.slice(index + 1).entries()) { + if (servers[index] !== servers[index + 1 + offset]) continue; for (const tag of lane) { for (const peer of other) { const shared = [...queues(tag)].filter(queue => queues(peer).has(queue));