diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 9cecaf1bf..d78521f2e 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -71,6 +71,9 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore PHP's SQLite maintenance loop on every path + run: node --test tests/Unit/PhpMaintenanceWindowTest.mjs + - name: Validate embedded Workflow package authority run: php scripts/ci/test-workflow-package-authority.php diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 81878ea9f..1eddf1d17 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -43,6 +43,8 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityUpdateValidationContractTest.mjs + - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -99,6 +101,8 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityUpdateValidationContractTest.mjs + - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -114,6 +118,13 @@ on: - rust/** workflow_dispatch: +# A newer push to the same pull request supersedes the run in progress, so a +# replaced head does not keep holding the organization's shared runners. +# Manual dispatches always finish. +concurrency: + group: shared-server-fixtures-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -1522,7 +1533,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt @@ -1581,6 +1592,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-server:8080 --target php --output /evidence/php.json \ + --exclude-fixture tests/Fixtures/ServerParity/update-validation.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -1593,6 +1605,7 @@ jobs: -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/embedded.sqlite -e DW_MODE=embedded \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \ --mode embedded --application-root /app --target embedded --output /evidence/embedded.json \ + --exclude-fixture tests/Fixtures/ServerParity/update-validation.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -1605,6 +1618,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-rust:8080 --target rust --output /evidence/rust.json \ + --exclude-fixture tests/Fixtures/ServerParity/update-validation.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -1616,6 +1630,7 @@ jobs: docker run --rm --user=1000:1000 --entrypoint node \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \ + --exclude-fixture tests/Fixtures/ServerParity/update-validation.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -1623,6 +1638,10 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? + # Server #723: update-validation records apart from the main corpus, and + # its published PHP record runs with PHP's maintenance loop stopped. + update_validation_status=0 + bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 || update_validation_status=$? bash scripts/conformance/server-parity/published-sdk-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/repeated-patch-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 bash scripts/conformance/server-parity/legacy-marker-profile.sh sqlite http://parity-server:8080 http://parity-rust:8080 @@ -1737,6 +1756,7 @@ jobs: # check has run, so one difference cannot hide the rest. test "$native_record_status" -eq 0 test "$compare_status" -eq 0 + test "$update_validation_status" -eq 0 - name: Compare separate configured role-token cohort env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} diff --git a/docs/rust-server-port-checkpoints/2026-10-23-mysql-statement-close.md b/docs/rust-server-port-checkpoints/2026-10-23-mysql-statement-close.md new file mode 100644 index 000000000..7d4c22a89 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-mysql-statement-close.md @@ -0,0 +1,71 @@ +# October 23, MySQL statement closes and late timers + +- **Found by.** Server #714. Twice on October 23, the hosted MariaDB 10.11 + `codec-deadline-health` cell recorded a native timer more than the 1 s + maximum late. In one, `TimerFired` came 1.098 s after `fire_at`. MySQL 8.0 + and 8.4 cells fail the same way, less often. +- **Reproduction.** A disposable 4-vCPU VM ran the unchanged cell on main + `f67f2816e`, with temporary timing in a local-only build: + - MariaDB failed 2 of the first 8 cells, with timers 1.228 s and 1.213 s + late; + - PostgreSQL 17 and SQLite passed all 8 cells, at most 171 ms late. +- **Attribution.** Every late timer waited behind the global transition lock: + - In the worst MariaDB cell, the tick's activity-deadline sweep waited + 543 ms behind a workflow completion that held the lock 690 ms. The sweep + then held it 407 ms itself, and the timer fired 1.228 s late. + - Ordinary transitions held the lock 300 ms to 1 s. With every statement + logged by MariaDB, those transactions spent a median of 5 ms in the + database (p90 9 ms) across their 30 to 48 statements. + - Native was not CPU-starved. Wake-up probes on an async task and an OS + thread rarely saw 50 ms delays, and giving native 2 CPUs left the holds + unchanged. + - Timestamps on each statement inside the transactions showed gaps of + 40 to 100 ms spread across ordinary statements, many near 58 ms. That is + Nagle's algorithm waiting for a delayed acknowledgement. +- **Cause.** `COM_STMT_CLOSE` has no reply. The published `sqlx-mysql` 0.9.0 + flushes it in its own write: when its 100-entry statement cache evicts a + statement, and after executing a statement it does not cache. Before the + next command it also flushes anything still buffered. The next command is + then a second small write while the close is still unacknowledged, so the + kernel holds it about 40 ms. SQLx sets no `TCP_NODELAY`; PHP's mysqlnd does. + PostgreSQL's driver sends its closes together with the next request. +- **Change.** In the vendored driver, `MySqlStream::queue_packet` buffers each + statement close with sequence number 0. On the path before the next command, + `wait_until_ready` no longer flushes a buffer that holds only replyless + closes, so they leave in the same write as that command. `Connection::flush` + and `clear_cached_statements` still send queued closes before returning; + Security's review of the first revision found that they did not. Packets, + order and server behavior are unchanged. The patch record is + `rust/vendor/sqlx-mysql/DURABLE-PATCH.md`. +- **Tests.** A proxy records the TCP read chunks it receives from the client. + TCP is a byte stream, so a chunk boundary shows bytes that arrived + separately. + - `qualification_statement_close_leaves_with_the_next_command` uses a + one-entry statement cache and one uncached statement, so both close paths + are exercised. With the fix it passes on MySQL 8.0.46 and MariaDB 10.11 in + 0.01 s. With the published driver it fails on both: every close arrives + as a chunk of its own, and the same 14 statements take 0.6 s. + - `qualification_explicit_flush_sends_queued_statement_closes`: a queued + close stays buffered until `flush`, which sends it with no later command + and leaves `should_flush` false. + - `qualification_clearing_the_statement_cache_closes_them_now`: + `clear_cached_statements` sends all three closes before it returns. On + MySQL 8.0 with the performance schema on, the server's statements for the + connection drop from three to none. + - Both lifecycle tests fail with the first revision (`f31c4c361`) on MySQL + 8.0 and MariaDB, and pass with this one. The existing cancelled-boundary + qualification still passes. +- **Hosted-cell A/B on the VM.** Interleaved cells with the spec and limits + unchanged. Both builds were instrumented: one without the driver change, one + with its first revision. That revision already buffered the closes, but + `wait_until_ready` still flushed them before draining. + + | | Cells passed | Timer lateness | Holds of 300 ms or more | + | --- | --- | --- | --- | + | MariaDB without the fix | 3 of 6 | 190 to 1045 ms | 54 | + | MariaDB with the fix | 6 of 6 | 49 to 289 ms | 0 | + | MySQL 8.0 and 8.4 without the fix | 5 of 8 | 138 to 1099 ms | 77 | + | MySQL 8.0 and 8.4 with the fix | 8 of 8 | 116 to 299 ms | 3 | + + Clean release builds of main and of this branch are compared across all five + families in the PR's evidence comment. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-policy-map-json.md b/docs/rust-server-port-checkpoints/2026-10-23-policy-map-json.md new file mode 100644 index 000000000..80db765d9 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-policy-map-json.md @@ -0,0 +1,33 @@ +# October 23, policy maps must be JSON objects + +- **Found by.** PHP Server #705 (merged to main at `f67f2816e`, not yet in a + published image). Published PHP turned malformed admission-policy JSON + into an empty policy, so a configured cap could disappear without notice. + Native had the same defect. + - `DW_TASK_QUEUE_ADMISSION_OVERRIDES` and + `DW_NAMESPACE_ADMISSION_OVERRIDES` accepted malformed JSON. + - All five maps native refuses accepted `null`, `[]`, booleans and + numbers. + - The external-payload namespace overrides, which native honors, became + an empty policy when malformed. +- **Change.** The six maps PHP now reads with `EnvAuditor::jsonObject` are + read by the same rule in `config_audit::json_object`: + - the value is either unset or raw text that decodes to a JSON object; + - Laravel's `env()` conversions do not apply, so `null`, `(null)`, + `empty`, quoted text and an empty value are refused; + - a set `DW_*` name wins over its legacy name even when the `DW_*` value + is invalid; + - the startup refusal names the variable it read, never the value. + + Native still refuses a non-empty object in the five maps it has no policy + for. It still honors `DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES`, and the + external payload quota now reads that value with the same decoder. +- **Reference.** This follows unreleased PHP source, not the published + 2.11.1 image. For any valid configuration the two behave the same. The + only values refused are ones published PHP silently discarded and the + next PHP image refuses at boot. +- **Test.** `policy_maps_must_be_json_objects_like_php_705` covers all six + maps under both names, 17 invalid values, padded whitespace around `{}`, + DW-over-legacy precedence in both directions, and that the refusal never + repeats the value. The shipped Compose and Helm values (`{}` for the + durable maps) still start. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-update-validation.md b/docs/rust-server-port-checkpoints/2026-10-23-update-validation.md new file mode 100644 index 000000000..340999874 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-update-validation.md @@ -0,0 +1,85 @@ +# October 23, synchronous update validation + +- **The gap.** The Python SDK lets a workflow declare update validators. A + worker with validators registers only when the server advertises + `update_validation_tasks` and its `synchronous_update_validation` contract. + Native advertised neither, and refused a validated update with 422 + `update_validation_not_implemented`. So a Python worker with validators + could not run against native. +- **How PHP 2.11.1 does it.** Learned from `ValidatedExternalWorkflowUpdateAdmission` + and `WorkflowUpdateValidationTaskBroker`, then observed live against the + published image. + - A worker declares `update_validators` per workflow type and advertises + the `update_validation_tasks` capability. The run's `WorkflowStarted` + records `declared_update_validators` when the worker's contract has that + field. + - For a live run with a complete durable contract and a declared update, + admission checks, in order: + 1. a recorded receipt for the `request_id`; + 2. whether the contract states validators at all, else 409 + `update_validator_contract_missing`, for any remote update; + 3. the update's arguments; + 4. a `request_id`, which a validated update requires. + + None of these refusals is recorded. + - The validation task is keyed by run, update name and `request_id`, and + holds a hash of the arguments. Different arguments under the same key + conflict with 409. A task that has finished replays its result. The + queue's fresh workers must include one that is capable and matches the + run, else 409 with a routing reason. + - The caller waits up to `DW_UPDATE_VALIDATION_TIMEOUT` seconds (10). A + worker claims the task through `POST /api/worker/update-validation-tasks/poll` + or the workflow poll with `task_kinds`. The claim lasts + `DW_UPDATE_VALIDATION_LEASE_TIMEOUT` seconds (5). The task carries the + run's replay history and history export. + - Approval admits the update with its id equal to the task id. A validator + rejection records a rejected command and update under the task id, with + `UpdateRejected`, and answers 422. Any other worker failure answers 409 + and records nothing. Without an answer, the task times out with 504 and + a reason: not claimed, worker lost, or execution timeout. + - Approve and reject are fenced, in order: task found, not finished, + leased, owner, attempt, lease unexpired, worker fresh. An expired lease + can be claimed again with the next attempt. + - The multiplexed poll keeps a cursor per queue. It tries the kind the + cursor names first, and turns the cursor to the other kind after each + lease. Every workflow task names `task_kind: "workflow"`. +- **What native does now.** + - Admission follows the same order, with the same bodies, and waits + without holding a transaction. Approval re-enters admission with the + task id, after checking that the run is still current and the approval + still stands. + - The validation task poll, approve and reject routes, with Laravel's + request rules. + - The workflow poll validates `task_kinds`. With validation tasks + requested, it refuses a worker without the capability, answers + `no_workflow_capability` for a worker without workflow types, and + alternates kinds by the `workflow_task_poll_cursors` row. + - The capability, its contract and both timeouts (`DW_UPDATE_VALIDATION_*`, + now honored settings) are advertised. + - A validated task's payload carries the unsigned history export, as + `HistoryExport::forRun` builds it. +- **Native changes found on the way.** + - An accepted update's receipt no longer carries `accepted: true`. PHP's + result mapper sends `accepted` only when it is false. + - A worker contract without `update_validators` now makes a remote update + refuse with 409, as on PHP. Current PHP, Python and Rust SDK workers all + send the field. +- **How it was checked.** + - The `update-validation` fixture records 49 steps: refusals before + validation, a rejection through the multiplexed poll, an approval through + the dedicated poll with its fences, retries, three timeouts run side by + side, lease expiry and re-claim, a worker failure, multiplexed ordering + and every routing refusal. + - Two published PHP runs normalize identically. Native matches all 49 + steps on SQLite. The contract test refuses 19 changed observations. + - Native integration tests cover the same paths on SQLite, PostgreSQL and + MySQL. +- **Not done yet** (Server #325 follow-ups): + - **Poll request ids.** PHP replays a leased validation task for a retried + `poll_request_id` and binds the request id to its task kinds. Native + does neither, so a retried multiplexed poll waits for the lease to + expire before the task is offered again. + - **Cluster info.** PHP's `workflow_update_runtime_contract` manifest and + its `service_mode_update_validation` block are not published yet; that + belongs to the cluster info work. + - **Takeover.** PHP data with validation tasks is still refused. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-worker-history-presentation.md b/docs/rust-server-port-checkpoints/2026-10-23-worker-history-presentation.md new file mode 100644 index 000000000..63f110abb --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-worker-history-presentation.md @@ -0,0 +1,48 @@ +# October 23, what a worker reads in history + +- **Found by.** Comparing the message-stream fixture's worker polls against + published Server 2.11.1. +- **The gaps.** + - PHP's poll and history-page responses show a consumed signal's + `workflow_sequence` on its `SignalReceived` event, read from the signal's + record (`WorkflowTaskPoller::historyEventsWithSignalArgumentEnvelopes`). + Native showed the event as recorded. + - PHP's task snapshots in history keep every field that is set + (`WorkflowHistoryEvent::taskSnapshot`). Native's left out + `compatibility`, `last_dispatch_attempt_at`, `last_dispatched_at`, + `last_dispatch_error`, `last_claim_failed_at` and `last_claim_error`. +- **What native does now.** + - Worker polls and history pages add `workflow_sequence` to a consumed + signal's `SignalReceived`, when the payload does not have one. The + run's own history route still shows the events as recorded, as PHP's + does. + - Task snapshots keep those fields when they are set. +- **Which signals carry a sequence.** PHP's `applySignal` takes a sequence + only from the signal's own wait. A signal the run's contract declares is + applied without one even while a condition wait is open, and + `markConditionWaitSignalConsumed` then finds it applied, so its record keeps + no sequence. Only an undeclared signal takes the condition wait's sequence. + Native had recorded the condition wait's sequence for declared signals too. + That difference was invisible until worker polls showed the record, and the + published-SDK signal and query cohorts then caught it: their worker history + must equal the run's history prefix. Native now records a sequence only from + the signal's own wait, or from a condition wait for an undeclared signal. +- **How it was checked.** A test runs on SQLite, PostgreSQL, MySQL and + MariaDB: + - a condition wait consumes an undeclared signal; + - the next poll and history page show that wait's sequence, and the + run's history does not; + - the completing task's snapshot has its dispatch stamps. + + A second test applies a declared signal under a condition wait twice. The + worker's history shows no sequence on either `SignalReceived`, and it fails + without the record change. +- **Not changed yet.** Some events PHP records with a task snapshot have + none in native, such as `ConditionWaitOpened` and `SignalWaitOpened`. + That audit is a #325 follow-up. +- **Known difference now visible in snapshots.** PHP stamps a dispatch only + on tasks a control-plane call dispatches, such as a signal's task. Tasks a + workflow task completion creates, such as the next message task, carry + none. Native stamps every new task, so those snapshots show stamps where + PHP's do not. This is the cause of the known `tasks_dispatched_last_minute` + difference too; fixing it is a #325 follow-up. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 773b6b4e4..0bc460521 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -138,13 +138,18 @@ any other unknown path or method answers PHP's own 404 or 405 reported as its candidate's failure, with 207. - **Worker protocol features.** - Worker sessions. - - Update validation tasks. - Local activities (prepare, checkpoint, recover, heartbeat, control, outcome and cancellation acknowledgement). - Cancellation scopes. - Activity cancellation acknowledgement. (Lease renewal, from Server 2.11, is served: `POST /api/worker/activity-tasks/{task}/renew-lease`, with the `activity_lease_renewal` capability and the `activity-lease-renewal` fixture.) + - (Synchronous update validation is served: the validation task poll, + approve and reject routes, the multiplexed workflow poll with + `task_kinds`, and validator-gated update admission, with the + `update-validation` fixture. Native does not replay a validation task by + `poll_request_id` or bind a request id to its task kinds; see + `docs/rust-server-port-checkpoints/2026-10-23-update-validation.md`.) - **Run operations.** - Imports from embedded v2 and Waterline v1. - **Other families.** diff --git a/rust/src/config_audit.rs b/rust/src/config_audit.rs index fcbed5eb5..c8f35a5ea 100644 --- a/rust/src/config_audit.rs +++ b/rust/src/config_audit.rs @@ -224,6 +224,21 @@ where .map_or(10, |value| value.int().max(1)) } +/// `server.update_validation`: how long an update request waits for its +/// validator (`timeout`, at least 0, 10 when unset) and how long a worker's +/// claim on a validation task lasts (`lease_timeout`, at least 1, 5 when +/// unset). The broker casts each with `(int)`. +pub fn update_validation_timeouts(lookup: F) -> (i64, i64) +where + F: Fn(&str) -> Option, +{ + let read = |name, legacy, default: i64| effective(&lookup, name, Some(legacy)).map_or(default, |value: PhpValue| value.int()); + ( + read("DW_UPDATE_VALIDATION_TIMEOUT", "WORKFLOW_SERVER_UPDATE_VALIDATION_TIMEOUT", 10).max(0), + read("DW_UPDATE_VALIDATION_LEASE_TIMEOUT", "WORKFLOW_SERVER_UPDATE_VALIDATION_LEASE_TIMEOUT", 5).max(1), + ) +} + /// `server.worker_protocol.history_page_size_default` and `_max`: `(int)` /// casts of the settings, 500 and 1000 when unset. The Workflow package's /// bridge then keeps every page between 1 and 1000 events. @@ -238,6 +253,44 @@ where ) } +/// The raw value PHP's `EnvAuditor::jsonObject` reads: the `DW_*` name +/// whenever it is set, even to an invalid value, else the legacy name. +/// Laravel's `env()` conversions do not apply. +fn selected<'a, F>(lookup: &F, name: &'a str, legacy: Option<&'a str>) -> Option<(String, &'a str)> +where + F: Fn(&str) -> Option, +{ + lookup(name) + .map(|raw| (raw, name)) + .or_else(|| legacy.and_then(|legacy| lookup(legacy).map(|raw| (raw, legacy)))) +} + +/// How a refusal names the variable it read. +fn read_from(name: &str, source: &str) -> String { + if source == name { name.to_owned() } else { format!("{name} (read from {source})") } +} + +/// `EnvAuditor::jsonObject` (Server #705): a policy map is unset, or raw text +/// that decodes to a JSON object. Malformed JSON, `null`, booleans, numbers, +/// strings and lists are refused instead of becoming an empty policy, so a +/// mistyped cap cannot silently disappear. The refusal never repeats the value. +pub fn json_object(lookup: F, name: &str, legacy: Option<&str>) -> Result, String> +where + F: Fn(&str) -> Option, +{ + let Some((raw, source)) = selected(&lookup, name, legacy) else { + return Ok(serde_json::Map::new()); + }; + match serde_json::from_str(&raw) { + Ok(serde_json::Value::Object(policy)) => Ok(policy), + _ => Err(format!("{} must contain a valid JSON object", read_from(name, source))), + } +} + +/// Policy maps native honors. Each must still be unset or a JSON object. +const HONORED_JSON_OBJECTS: &[(&str, &str)] = + &[("DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES", "WORKFLOW_SERVER_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES")]; + /// What a refused setting must be for native to start. #[derive(Clone, Copy, Debug)] pub enum Rule { @@ -256,6 +309,9 @@ pub enum Rule { /// `json_decode` gives nothing. Strict settings also refuse invalid JSON, /// which PHP either rejects or cannot have meant. EmptyJson { strict: bool }, + /// A policy map read by `EnvAuditor::jsonObject`: unset or a JSON object + /// (see [`json_object`]), and empty, since native has no such policy. + EmptyObject, /// The `(int)` cast equals PHP's default. Int(i64), /// `filter_var(..., FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE) ?? default` @@ -287,6 +343,9 @@ impl Rule { Ok(decoded) => !php_truthy(&decoded), Err(_) => !strict, }, + Self::EmptyObject => { + matches!(serde_json::from_str(text), Ok(serde_json::Value::Object(policy)) if policy.is_empty()) + } Self::Int(expected) => value.int() == expected, Self::FilterBool(default) => value.filter_bool().unwrap_or(default) == default, Self::CastBool(default) => value.cast_bool() == default, @@ -308,7 +367,7 @@ impl Rule { Self::AnyCase(value) => format!("unset or `{value}`"), Self::Empty => "unset or empty".into(), Self::NoPositiveInt => "unset, empty or not a positive number".into(), - Self::EmptyJson { .. } => "unset or `{}`".into(), + Self::EmptyJson { .. } | Self::EmptyObject => "unset or `{}`".into(), Self::Int(value) => format!("unset or `{value}`"), Self::FilterBool(value) | Self::CastBool(value) => format!("unset or {value}"), Self::DatabaseConnection => "unset, empty or the DB_CONNECTION name".into(), @@ -380,6 +439,17 @@ pub fn audit(environment: &BTreeMap) -> Report { let Disposition::Refused(rule, reason) = setting.disposition else { continue; }; + if let Rule::EmptyObject = rule { + match json_object(lookup, setting.name, setting.legacy) { + Err(message) => report.refusals.push(message), + Ok(policy) if !policy.is_empty() => { + let source = selected(&lookup, setting.name, setting.legacy).map_or(setting.name, |(_, source)| source); + report.refusals.push(format!("{} must be {}: {reason}", read_from(setting.name, source), rule.expectation())); + } + Ok(_) => {} + } + continue; + } let read = |name: &'static str| lookup(name).map(|raw| (PhpValue::from_env(&raw), name)); let primary = read(setting.name).filter(|(value, _)| *value != PhpValue::Null); let Some((value, source)) = primary.or_else(|| setting.legacy.and_then(read)) else { @@ -395,6 +465,11 @@ pub fn audit(environment: &BTreeMap) -> Report { if let Err(message) = activity_task_lease_seconds(lookup) { report.refusals.push(message.into()); } + for (name, legacy) in HONORED_JSON_OBJECTS { + if let Err(message) = json_object(lookup, name, Some(legacy)) { + report.refusals.push(message); + } + } let unknown: Vec<&str> = environment .keys() .map(String::as_str) @@ -458,7 +533,7 @@ const STORAGE_CONNECTION: &str = "native keeps workflow tables on its one databa const LEGACY_SCHEDULE_TIMEZONE: &str = "native does not convert pre-UTC schedule timestamps"; use Disposition::{Honored, Ignored, Refused}; -use Rule::{AnyCase, CastBool, DatabaseConnection, Empty, EmptyJson, Exact, FilterBool, Int, NoPositiveInt, NotAnyCase, NotIn}; +use Rule::{AnyCase, CastBool, DatabaseConnection, Empty, EmptyJson, EmptyObject, Exact, FilterBool, Int, NoPositiveInt, NotAnyCase, NotIn}; /// Settings PHP reads that its contract does not list. PHP's audit warns about /// them as unknown names; native also refuses values it cannot honor. @@ -530,10 +605,10 @@ pub const SETTINGS: &[Setting] = &[ Setting { name: "DW_METRICS_PROMETHEUS_WORKFLOW_SERIES_LIMIT", legacy: Some("WORKFLOW_SERVER_METRICS_PROMETHEUS_WORKFLOW_SERIES_LIMIT"), disposition: Refused(Int(100), METRICS) }, Setting { name: "DW_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT", legacy: Some("WORKFLOW_SERVER_METRICS_WORKFLOW_TASK_FAILURE_TYPE_LIMIT"), disposition: Refused(Int(20), METRICS) }, Setting { name: "DW_MODE", legacy: Some("WORKFLOW_SERVER_MODE"), disposition: Refused(Exact(&["service"]), TOPOLOGY) }, - Setting { name: "DW_NAMESPACE_ADMISSION_OVERRIDES", legacy: Some("WORKFLOW_SERVER_NAMESPACE_ADMISSION_OVERRIDES"), disposition: Refused(EmptyJson { strict: false }, NAMESPACE_ADMISSION) }, - Setting { name: "DW_NAMESPACE_DURABLE_HARD_LIMITS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_HARD_LIMITS"), disposition: Refused(EmptyJson { strict: true }, DURABLE_LIMITS) }, - Setting { name: "DW_NAMESPACE_DURABLE_LIMITS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_LIMITS"), disposition: Refused(EmptyJson { strict: true }, DURABLE_LIMITS) }, - Setting { name: "DW_NAMESPACE_DURABLE_OVERRIDES", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_OVERRIDES"), disposition: Refused(EmptyJson { strict: true }, DURABLE_LIMITS) }, + Setting { name: "DW_NAMESPACE_ADMISSION_OVERRIDES", legacy: Some("WORKFLOW_SERVER_NAMESPACE_ADMISSION_OVERRIDES"), disposition: Refused(EmptyObject, NAMESPACE_ADMISSION) }, + Setting { name: "DW_NAMESPACE_DURABLE_HARD_LIMITS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_HARD_LIMITS"), disposition: Refused(EmptyObject, DURABLE_LIMITS) }, + Setting { name: "DW_NAMESPACE_DURABLE_LIMITS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_LIMITS"), disposition: Refused(EmptyObject, DURABLE_LIMITS) }, + Setting { name: "DW_NAMESPACE_DURABLE_OVERRIDES", legacy: Some("WORKFLOW_SERVER_NAMESPACE_DURABLE_OVERRIDES"), disposition: Refused(EmptyObject, DURABLE_LIMITS) }, Setting { name: "DW_NAMESPACE_HARD_MAX_CONCURRENT_REQUESTS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_HARD_MAX_CONCURRENT_REQUESTS"), disposition: Refused(Empty, NAMESPACE_ADMISSION) }, Setting { name: "DW_NAMESPACE_HARD_MAX_REQUESTS_PER_MINUTE", legacy: Some("WORKFLOW_SERVER_NAMESPACE_HARD_MAX_REQUESTS_PER_MINUTE"), disposition: Refused(Empty, NAMESPACE_ADMISSION) }, Setting { name: "DW_NAMESPACE_MAX_CONCURRENT_REQUESTS", legacy: Some("WORKFLOW_SERVER_NAMESPACE_MAX_CONCURRENT_REQUESTS"), disposition: Refused(Empty, NAMESPACE_ADMISSION) }, @@ -573,10 +648,10 @@ pub const SETTINGS: &[Setting] = &[ Setting { name: "DW_STORAGE_ADMISSION_MAX_AGE_SECONDS", legacy: None, disposition: Ignored }, Setting { name: "DW_STORAGE_ADMISSION_SOURCE", legacy: None, disposition: Refused(Empty, STORAGE_ADMISSION) }, Setting { name: "DW_TASK_DISPATCH_MODE", legacy: Some("WORKFLOW_V2_TASK_DISPATCH_MODE"), disposition: Refused(Exact(&["poll"]), POLL_DISPATCH) }, - Setting { name: "DW_TASK_QUEUE_ADMISSION_OVERRIDES", legacy: Some("WORKFLOW_SERVER_TASK_QUEUE_ADMISSION_OVERRIDES"), disposition: Refused(EmptyJson { strict: false }, TASK_ADMISSION) }, + Setting { name: "DW_TASK_QUEUE_ADMISSION_OVERRIDES", legacy: Some("WORKFLOW_SERVER_TASK_QUEUE_ADMISSION_OVERRIDES"), disposition: Refused(EmptyObject, TASK_ADMISSION) }, Setting { name: "DW_TRUST_FORWARDED_ATTRIBUTION_HEADERS", legacy: Some("WORKFLOW_SERVER_TRUST_FORWARDED_ATTRIBUTION_HEADERS"), disposition: Refused(FilterBool(false), FORWARDED) }, - Setting { name: "DW_UPDATE_VALIDATION_LEASE_TIMEOUT", legacy: Some("WORKFLOW_SERVER_UPDATE_VALIDATION_LEASE_TIMEOUT"), disposition: Ignored }, - Setting { name: "DW_UPDATE_VALIDATION_TIMEOUT", legacy: Some("WORKFLOW_SERVER_UPDATE_VALIDATION_TIMEOUT"), disposition: Ignored }, + Setting { name: "DW_UPDATE_VALIDATION_LEASE_TIMEOUT", legacy: Some("WORKFLOW_SERVER_UPDATE_VALIDATION_LEASE_TIMEOUT"), disposition: Honored }, + Setting { name: "DW_UPDATE_VALIDATION_TIMEOUT", legacy: Some("WORKFLOW_SERVER_UPDATE_VALIDATION_TIMEOUT"), disposition: Honored }, Setting { name: "DW_V1_ENABLED", legacy: Some("WORKFLOW_V1_ENABLED"), disposition: Refused(CastBool(false), V1) }, Setting { name: "DW_V2_CACHE_VALIDATION_MODE", legacy: Some("WORKFLOW_V2_CACHE_VALIDATION_MODE"), disposition: Ignored }, Setting { name: "DW_V2_COMPATIBILITY_HEARTBEAT_TTL", legacy: Some("WORKFLOW_V2_COMPATIBILITY_HEARTBEAT_TTL"), disposition: Refused(Int(30), COMPATIBILITY) }, @@ -808,6 +883,18 @@ mod tests { assert!(!cast("") && !cast("0")); } + #[test] + fn update_validation_timeouts_are_read_like_the_broker() { + let read = |values: &[(&str, &str)]| { + let environment = environment(values); + update_validation_timeouts(|key| environment.get(key).cloned()) + }; + assert_eq!(read(&[]), (10, 5)); + assert_eq!(read(&[("DW_UPDATE_VALIDATION_TIMEOUT", "3"), ("DW_UPDATE_VALIDATION_LEASE_TIMEOUT", "2")]), (3, 2)); + assert_eq!(read(&[("WORKFLOW_SERVER_UPDATE_VALIDATION_TIMEOUT", "4")]), (4, 5)); + assert_eq!(read(&[("DW_UPDATE_VALIDATION_TIMEOUT", "-1"), ("DW_UPDATE_VALIDATION_LEASE_TIMEOUT", "0")]), (0, 1)); + } + #[test] fn update_wait_is_read_like_the_package() { let read = |values: &[(&str, &str)]| { @@ -875,8 +962,10 @@ mod tests { &[("DW_WORKFLOW_TASK_MAX_ACTIVE_LEASES_PER_QUEUE", "0")], &[("DW_WORKFLOW_TASK_MAX_ACTIVE_LEASES_PER_QUEUE", "true")], &[("DW_WORKFLOW_TASK_MAX_ACTIVE_LEASES_PER_QUEUE", "none")], - &[("DW_TASK_QUEUE_ADMISSION_OVERRIDES", "not json")], - &[("DW_TASK_QUEUE_ADMISSION_OVERRIDES", "[]")], + &[("DW_TASK_QUEUE_ADMISSION_OVERRIDES", "{}")], + &[("DW_TASK_QUEUE_ADMISSION_OVERRIDES", " \n{ } ")], + &[("WORKFLOW_SERVER_NAMESPACE_ADMISSION_OVERRIDES", "{}")], + &[("DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES", "{\"billing\":{\"max_bytes\":5}}")], &[("DW_V2_FLEET_VALIDATION_MODE", "warn")], &[("DW_V2_PIN_TO_RECORDED_FINGERPRINT", "no")], &[("DW_SERIALIZER", "")], @@ -926,6 +1015,57 @@ mod tests { } } + #[test] + fn policy_maps_must_be_json_objects_like_php_705() { + // The six maps PHP reads with `EnvAuditor::jsonObject`. + let maps = [ + ("DW_TASK_QUEUE_ADMISSION_OVERRIDES", "WORKFLOW_SERVER_TASK_QUEUE_ADMISSION_OVERRIDES"), + ("DW_NAMESPACE_ADMISSION_OVERRIDES", "WORKFLOW_SERVER_NAMESPACE_ADMISSION_OVERRIDES"), + ("DW_NAMESPACE_DURABLE_LIMITS", "WORKFLOW_SERVER_NAMESPACE_DURABLE_LIMITS"), + ("DW_NAMESPACE_DURABLE_HARD_LIMITS", "WORKFLOW_SERVER_NAMESPACE_DURABLE_HARD_LIMITS"), + ("DW_NAMESPACE_DURABLE_OVERRIDES", "WORKFLOW_SERVER_NAMESPACE_DURABLE_OVERRIDES"), + ("DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES", "WORKFLOW_SERVER_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES"), + ]; + let invalid = [ + "", " ", "not json", "{", "{\"cap\":1", "null", "(null)", "empty", "true", "false", "0", "1", "[]", + "[{\"cap\":1}]", "\"{}\"", "'{}'", "\"text\"", + ]; + for (name, legacy) in maps { + for value in ["{}", " \n\t{}\r\n"] { + assert_eq!(refusals(&[(name, value)]), Vec::::new(), "{name}={value:?}"); + assert_eq!(refusals(&[(legacy, value)]), Vec::::new(), "{legacy}={value:?}"); + } + for value in invalid { + assert_eq!(refusals(&[(name, value)]), [format!("{name} must contain a valid JSON object")], "{name}={value:?}"); + assert_eq!( + refusals(&[(legacy, value)]), + [format!("{name} (read from {legacy}) must contain a valid JSON object")], + "{legacy}={value:?}" + ); + // A set DW name wins even when it is invalid and the legacy + // value is valid. + assert_eq!(refusals(&[(name, value), (legacy, "{}")]).len(), 1, "{name}={value:?}"); + } + // A valid DW name hides an invalid legacy value. + assert_eq!(refusals(&[(name, "{}"), (legacy, "not json")]), Vec::::new(), "{name}"); + } + // Native honors the external payload overrides; it refuses the other + // five unless they are empty, as before. + let policy = "{\"billing\":{\"max_bytes\":5}}"; + for (name, legacy) in &maps[..5] { + assert_eq!(refusals(&[(name, policy)]).len(), 1, "{name}"); + assert!(refusals(&[(legacy, policy)])[0].starts_with(&format!("{name} (read from {legacy}) must be unset or `{{}}`"))); + } + assert_eq!(refusals(&[(maps[5].0, policy)]), Vec::::new()); + assert_eq!( + json_object(|_| Some(policy.to_owned()), maps[5].0, Some(maps[5].1)).unwrap()["billing"]["max_bytes"], + 5 + ); + // The refusal never repeats the value. + let secret = "{\"secret-tenant-value\":"; + assert!(refusals(&[(maps[5].0, secret)]).iter().all(|refusal| !refusal.contains("secret-tenant-value"))); + } + #[test] fn refusals_never_repeat_the_value() { let secret = "principal-secret-value"; diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index a75eadc46..0e8378301 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -929,6 +929,90 @@ impl Runtime { } } } + /// `pollUpdateValidationTasks`. + pub(crate) async fn poll_update_validation(&self, body: Value) -> Result { + super::update_validation::poll_rules(&body)?; + let timeout = match body.get("timeout_seconds") { + None | Some(Value::Null) => self.worker_poll_wait(), + Some(value) => super::laravel_validation::integer(value).and_then(|value| u64::try_from(value).ok()).unwrap_or(0), + }; + let worker_id = super::text(&body, "worker_id")?; + let task_queue = super::text(&body, "task_queue")?; + let leased = match &*self.storage { + Storage::Sqlite(store) => store.update_validation_poll(worker_id, task_queue, timeout).await?, + Storage::Postgres(store) => store.update_validation_poll(worker_id, task_queue, timeout).await?, + Storage::MySql(store) => store.update_validation_poll(worker_id, task_queue, timeout).await?, + }; + let task = match leased { + Some(task_id) => Some(self.update_validation_task(&task_id).await?), + None => None, + }; + let status = if task.is_some() { "leased" } else { "empty" }; + Ok(json!({"task":task,"poll_status":status})) + } + /// `POST /worker/update-validation-tasks/{taskId}/approve` and `/reject`. + pub(crate) async fn complete_update_validation(&self, task_id: &str, body: Value, reject: bool) -> Result<(StatusCode, Value)> { + super::update_validation::completion_rules(&body, reject)?; + let failure = reject.then(|| body["failure"].clone()); + match &*self.storage { + Storage::Sqlite(store) => store.complete_update_validation(task_id, &body, failure.as_ref()).await, + Storage::Postgres(store) => store.complete_update_validation(task_id, &body, failure.as_ref()).await, + Storage::MySql(store) => store.complete_update_validation(task_id, &body, failure.as_ref()).await, + } + } + /// A leased validation task as a worker receives it, with the run's + /// replay bundle. + async fn update_validation_task(&self, task_id: &str) -> Result { + let mut task = match &*self.storage { + Storage::Sqlite(store) => store.update_validation_payload(task_id).await?, + Storage::Postgres(store) => store.update_validation_payload(task_id).await?, + Storage::MySql(store) => store.update_validation_payload(task_id).await?, + }; + if task.get("history_events").is_some() + && let (Some(workflow_id), Some(run_id)) = (task["workflow_id"].as_str(), task["run_id"].as_str()) + { + let export = self.history_bundle(&workflow_id.to_owned(), &run_id.to_owned(), false).await?; + task["history_export"] = export; + } + Ok(task) + } + /// `pollWorkflowTasks`: workflow tasks, validation tasks or both, as + /// `task_kinds` asks. Every task names its kind. + pub(crate) async fn poll_workflow_tasks(&self, body: Value, protocol_version: &str) -> Result { + let kinds = super::update_validation::task_kinds(&body)?; + let pages = self.history_pages(); + let wait = self.worker_poll_wait(); + let mark = |mut answer: Value| { + if let Some(task) = answer["task"].as_object_mut() { + task.insert("task_kind".into(), json!("workflow")); + } + answer + }; + if kinds == ["workflow"] { + return self.poll_with_pages(body, "workflow", protocol_version, pages, wait).await.map(mark); + } + let timeout = match body.get("timeout_seconds") { + None | Some(Value::Null) => wait, + Some(value) => super::laravel_validation::integer(value) + .filter(|seconds| (0..=60).contains(seconds)) + .and_then(|seconds| u64::try_from(seconds).ok()) + .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_poll_timeout"))?, + }; + let leased = match &*self.storage { + Storage::Sqlite(store) => store.multiplexed_poll(&body, &kinds, protocol_version, pages, timeout).await?, + Storage::Postgres(store) => store.multiplexed_poll(&body, &kinds, protocol_version, pages, timeout).await?, + Storage::MySql(store) => store.multiplexed_poll(&body, &kinds, protocol_version, pages, timeout).await?, + }; + match leased { + super::update_validation::Leased::Workflow(answer) => Ok(mark(answer)), + super::update_validation::Leased::Validation(task_id) => { + let mut task = self.update_validation_task(&task_id).await?; + // The multiplexed poll's history paging field. + task["next_history_page_token"] = Value::Null; + Ok(json!({"task":task,"poll_status":"leased","server_capabilities":super::store::capabilities()})) + } + } + } pub(crate) async fn poll_query(&self, body: Value) -> Result { match &*self.storage { Storage::Sqlite(store) => store.poll_query(body, &self.queries).await, @@ -1019,6 +1103,11 @@ impl Runtime { ); /// `HistoryController::export`: one run's replay bundle. pub(crate) async fn history_export(&self, workflow_id: &str, run_id: &str) -> Result { + self.history_bundle(workflow_id, run_id, true).await + } + /// One run's replay bundle, signed when asked and configured. + /// `HistoryExport::forRun` itself, as replay tasks carry it, is unsigned. + async fn history_bundle(&self, workflow_id: &str, run_id: &str, signed: bool) -> Result { let (mut rows, memo) = match &*self.storage { Storage::Sqlite(store) => store.history_export_rows(workflow_id, run_id).await?, Storage::Postgres(store) => store.history_export_rows(workflow_id, run_id).await?, @@ -1032,7 +1121,7 @@ impl Runtime { .map(|value| value.trim().to_owned()) .filter(|value| !value.is_empty()) }; - let signing = setting(["DW_V2_HISTORY_EXPORT_SIGNING_KEY", "WORKFLOW_V2_HISTORY_EXPORT_SIGNING_KEY"]).map(|key| { + let signing = setting(["DW_V2_HISTORY_EXPORT_SIGNING_KEY", "WORKFLOW_V2_HISTORY_EXPORT_SIGNING_KEY"]).filter(|_| signed).map(|key| { super::history_export::Signing { key, key_id: setting(["DW_V2_HISTORY_EXPORT_SIGNING_KEY_ID", "WORKFLOW_V2_HISTORY_EXPORT_SIGNING_KEY_ID"]), diff --git a/rust/src/runtime/external_config.rs b/rust/src/runtime/external_config.rs index e4992b199..070638fc7 100644 --- a/rust/src/runtime/external_config.rs +++ b/rust/src/runtime/external_config.rs @@ -122,11 +122,13 @@ fn member<'a>(value: &'a Value, key: &str) -> Option<&'a Value> { impl ExternalPayloadConfig { pub(super) fn from_env() -> Result { let completion = process_value("COMPLETION_MAX_BYTES", Value::Null)?; - let overrides = process_value("NAMESPACE_OVERRIDES", Value::String("{}".into()))?; - let overrides = overrides - .as_str() - .and_then(|value| serde_json::from_str(value).ok()) - .unwrap_or(Value::Null); + // Startup refuses a value that is not a JSON object (Server #705). + let overrides = crate::config_audit::json_object( + |name| env::var_os(name).map(|value| value.to_string_lossy().into_owned()), + "DW_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES", + Some("WORKFLOW_SERVER_EXTERNAL_PAYLOAD_NAMESPACE_OVERRIDES"), + ) + .map_err(|_| configuration_error())?; Ok(Self { completion_max_bytes: (!completion.is_null()) .then(|| integer(&completion).max(1) as u64), @@ -150,7 +152,7 @@ impl ExternalPayloadConfig { "HARD_MAX_OBJECTS_PER_NAMESPACE", Value::Null, )?, - namespace_overrides: overrides, + namespace_overrides: Value::Object(overrides), ..Self::default() }) } diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 3cc195f41..1c6614aa0 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -292,6 +292,15 @@ pub fn router(runtime: Runtime) -> Router { post(complete_query), ) .route("/api/worker/query-tasks/{task_id}/fail", post(fail_query)) + .route("/api/worker/update-validation-tasks/poll", post(poll_update_validation)) + .route( + "/api/worker/update-validation-tasks/{task_id}/approve", + post(approve_update_validation), + ) + .route( + "/api/worker/update-validation-tasks/{task_id}/reject", + post(reject_update_validation), + ) .fallback(route_fallback) .method_not_allowed_fallback(route_fallback) .layer(middleware::from_fn(empty_list_body)) @@ -751,6 +760,8 @@ fn worker_route(rest: &str) -> Option { | ["workflow-tasks" | "activity-tasks", "poll"] | ["query-tasks", "poll"] | ["query-tasks", _, "complete" | "fail" | "status"] + | ["update-validation-tasks", "poll"] + | ["update-validation-tasks", _, "approve" | "reject"] | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat"] | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease"] ) @@ -1820,12 +1831,46 @@ async fn poll_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { - let pages = runtime.history_pages(); - let wait = runtime.worker_poll_wait(); runtime - .poll_with_pages(body, "workflow", request_protocol(&headers)?, pages, wait) + .poll_workflow_tasks(body, request_protocol(&headers)?) .await .map(Json) + .map_err(worker_refusal) +} + +async fn poll_update_validation( + Extension(runtime): Extension, + headers: HeaderMap, + Json(body): Json, +) -> Result> { + request_protocol(&headers)?; + runtime + .poll_update_validation(body) + .await + .map(|body| Json(worker_response(body))) + .map_err(worker_refusal) +} + +async fn approve_update_validation( + Extension(runtime): Extension, + Path(task_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + request_protocol(&headers)?; + let (status, body) = runtime.complete_update_validation(&task_id, body, false).await.map_err(worker_refusal)?; + Ok((status, Json(worker_response(body)))) +} + +async fn reject_update_validation( + Extension(runtime): Extension, + Path(task_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + request_protocol(&headers)?; + let (status, body) = runtime.complete_update_validation(&task_id, body, true).await.map_err(worker_refusal)?; + Ok((status, Json(worker_response(body)))) } async fn poll_activity( Extension(runtime): Extension, diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index a38f42362..2e3df6dc3 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -130,6 +130,7 @@ mod terminal_updates; mod timer_summary; mod transaction; mod update_summary; +mod update_validation; mod updates; pub mod upgrade; mod visibility; diff --git a/rust/src/runtime/mysql_pool_tests.rs b/rust/src/runtime/mysql_pool_tests.rs index a4780a930..20b524bf7 100644 --- a/rust/src/runtime/mysql_pool_tests.rs +++ b/rust/src/runtime/mysql_pool_tests.rs @@ -221,3 +221,246 @@ async fn qualification_cancelled_pool_boundaries_preserve_fresh_reads_and_commit .await .expect("bounded real transaction-boundary qualification"); } + +const COM_STMT_CLOSE: u8 = 0x19; + +/// Forward clients to `upstream`, recording each TCP read chunk received from +/// a client. TCP is a byte stream: a chunk boundary shows bytes that arrived +/// separately, not a write call. +async fn record_client_chunks( + listener: TcpListener, + upstream: String, + writes: Arc>>>, +) { + loop { + let (client, _) = listener.accept().await.unwrap(); + let server = TcpStream::connect(&upstream).await.unwrap(); + let writes = writes.clone(); + tokio::spawn(async move { + let (mut client_read, mut client_write) = client.into_split(); + let (mut server_read, mut server_write) = server.into_split(); + let upstream = async { + let mut bytes = vec![0; 65536]; + loop { + let count = client_read.read(&mut bytes).await?; + if count == 0 { + break; + } + writes.lock().unwrap().push(bytes[..count].to_vec()); + server_write.write_all(&bytes[..count]).await?; + } + server_write.shutdown().await + }; + let downstream = async { + tokio::io::copy(&mut server_read, &mut client_write).await?; + client_write.shutdown().await + }; + let _: (std::io::Result<()>, std::io::Result<()>) = tokio::join!(upstream, downstream); + }); + } +} + +/// Each client command packet (sequence 0): its command byte, and whether a +/// received chunk ended with it. +fn recorded_commands(writes: &[Vec]) -> Vec<(u8, bool)> { + let mut stream = Vec::new(); + let mut write_ends = std::collections::BTreeSet::new(); + for write in writes { + stream.extend_from_slice(write); + write_ends.insert(stream.len()); + } + let mut commands = Vec::new(); + let mut at = 0; + while at + 4 <= stream.len() { + let length = usize::from(stream[at]) + | usize::from(stream[at + 1]) << 8 + | usize::from(stream[at + 2]) << 16; + let end = at + 4 + length; + if end > stream.len() { + break; + } + if stream[at + 3] == 0 && length > 0 { + commands.push((stream[at + 4], write_ends.contains(&end))); + } + at = end; + } + commands +} + +/// A statement close has no reply. Sent on its own, it leaves +/// unacknowledged data in flight, and Nagle's algorithm then holds the next +/// command until the server's delayed acknowledgement, about 40 ms per close. +/// The proxy then receives the close as a chunk of its own. Under the +/// transition lock those stalls added up to a late timer (#714). +#[tokio::test] +#[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] +async fn qualification_statement_close_leaves_with_the_next_command() { + tokio::time::timeout(Duration::from_secs(15), async { + let url = std::env::var("DW_TEST_MYSQL_URL").unwrap(); + let direct = MySqlConnectOptions::from_str(&url).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let writes = Arc::new(std::sync::Mutex::new(Vec::new())); + let proxy = tokio::spawn(record_client_chunks( + listener, + format!("{}:{}", direct.get_host(), direct.get_port()), + writes.clone(), + )); + // Plain text so the proxy can read packets; one cached statement, so + // each new statement evicts and closes the previous one. + let options = direct + .host("127.0.0.1") + .port(port) + .ssl_mode(sqlx::mysql::MySqlSslMode::Disabled) + .statement_cache_capacity(1); + let mut connection = MySqlConnection::connect_with(&options).await.unwrap(); + for round in 0..4_i64 { + for (sql, step) in [("SELECT ? + 1", 1), ("SELECT ? + 2", 2), ("SELECT ? + 3", 3)] { + let value: i64 = sqlx::query_scalar(sql).bind(round).fetch_one(&mut connection).await.unwrap(); + assert_eq!(value, round + step); + } + } + // An uncached statement is closed right after it executes. + let value: i64 = sqlx::query_scalar("SELECT ? + 4") + .bind(1_i64) + .persistent(false) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(value, 5); + let value: i64 = sqlx::query_scalar("SELECT ? + 5").bind(1_i64).fetch_one(&mut connection).await.unwrap(); + assert_eq!(value, 6); + connection.close().await.unwrap(); + proxy.abort(); + let commands = recorded_commands(&writes.lock().unwrap()); + let closes = commands.iter().filter(|(command, _)| *command == COM_STMT_CLOSE).count(); + assert!(closes >= 12, "eviction and uncached closes were exercised: {closes}"); + assert!( + commands.iter().all(|(command, ended)| *command != COM_STMT_CLOSE || !ended), + "a statement close arrived as a chunk of its own: {commands:?}" + ); + }) + .await + .expect("bounded statement-close chunk qualification"); +} + +/// A plaintext connection through a recording proxy. +async fn proxied_connection( + capacity: usize, +) -> (MySqlConnection, Arc>>>, tokio::task::JoinHandle<()>, MySqlConnectOptions) { + let url = std::env::var("DW_TEST_MYSQL_URL").unwrap(); + let direct = MySqlConnectOptions::from_str(&url).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let writes = Arc::new(std::sync::Mutex::new(Vec::new())); + let proxy = tokio::spawn(record_client_chunks( + listener, + format!("{}:{}", direct.get_host(), direct.get_port()), + writes.clone(), + )); + let options = direct + .clone() + .host("127.0.0.1") + .port(port) + .ssl_mode(sqlx::mysql::MySqlSslMode::Disabled) + .statement_cache_capacity(capacity); + (MySqlConnection::connect_with(&options).await.unwrap(), writes, proxy, direct) +} + +fn closes(writes: &std::sync::Mutex>>) -> usize { + recorded_commands(&writes.lock().unwrap()) + .iter() + .filter(|(command, _)| *command == COM_STMT_CLOSE) + .count() +} + +/// Wait, bounded, until the proxy has received `count` statement closes. +async fn closes_arrive(writes: &std::sync::Mutex>>, count: usize) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while closes(writes) < count { + assert!( + tokio::time::Instant::now() < deadline, + "{count} statement closes did not leave the client: {:?}", + recorded_commands(&writes.lock().unwrap()) + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// `Connection::flush` sends queued closes by itself: no later command is +/// needed, and the connection reports nothing left to flush. +#[tokio::test] +#[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] +async fn qualification_explicit_flush_sends_queued_statement_closes() { + use sqlx::Connection as _; + tokio::time::timeout(Duration::from_secs(15), async { + let (mut connection, writes, proxy, _) = proxied_connection(1).await; + // An uncached statement queues its close after it executes. + let value: i64 = sqlx::query_scalar("SELECT ? + 4") + .bind(1_i64) + .persistent(false) + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(value, 5); + assert!(connection.should_flush(), "the close is queued"); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(closes(&writes), 0, "the queued close waits for a command or flush"); + connection.flush().await.unwrap(); + assert!(!connection.should_flush(), "an explicit flush leaves nothing buffered"); + closes_arrive(&writes, 1).await; + connection.close().await.unwrap(); + proxy.abort(); + }) + .await + .expect("bounded explicit-flush qualification"); +} + +/// `clear_cached_statements` closes every cached statement on the server +/// before it returns, without a later command. +#[tokio::test] +#[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] +async fn qualification_clearing_the_statement_cache_closes_them_now() { + use sqlx::Connection as _; + tokio::time::timeout(Duration::from_secs(15), async { + let (mut connection, writes, proxy, direct) = proxied_connection(100).await; + // A separate session can see this connection's server-side + // statements where the performance schema is on (MySQL 8 default). + let thread: u64 = sqlx::query_scalar("SELECT CONNECTION_ID()").fetch_one(&mut connection).await.unwrap(); + connection.clear_cached_statements().await.unwrap(); + let before = closes(&writes); + for sql in ["SELECT ? + 1", "SELECT ? + 2", "SELECT ? + 3"] { + let _: i64 = sqlx::query_scalar(sql).bind(1_i64).fetch_one(&mut connection).await.unwrap(); + } + assert_eq!(connection.cached_statements_size(), 3); + let mut observer = MySqlConnection::connect_with(&direct).await.unwrap(); + let schema: i64 = sqlx::query_scalar("SELECT @@performance_schema").fetch_one(&mut observer).await.unwrap(); + let server_statements = async |observer: &mut MySqlConnection| -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM performance_schema.prepared_statements_instances p JOIN performance_schema.threads t ON t.THREAD_ID=p.OWNER_THREAD_ID WHERE t.PROCESSLIST_ID=?") + .bind(thread) + .fetch_one(observer) + .await + .unwrap() + }; + if schema == 1 { + let held = server_statements(&mut observer).await; + assert_eq!(held, 3, "the server holds this connection's three statements"); + } + connection.clear_cached_statements().await.unwrap(); + assert_eq!(connection.cached_statements_size(), 0); + assert!(!connection.should_flush(), "clearing leaves nothing buffered"); + closes_arrive(&writes, before + 3).await; + if schema == 1 { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while server_statements(&mut observer).await != 0 { + assert!(tokio::time::Instant::now() < deadline, "the server still holds the statements"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + observer.close().await.unwrap(); + connection.close().await.unwrap(); + proxy.abort(); + }) + .await + .expect("bounded statement-cache clear qualification"); +} diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 98d803b40..61a2773b9 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1124,11 +1124,10 @@ where let wait = open.clone().filter(|wait| { wait.get("signal_name").is_none() || (wait["signal_name"] == name && wait["signal_wait_id"] == wait_id) }); - if wait.is_none() - && !Self::declared_signals(tx, run_id) - .await? - .is_some_and(|names| names.contains(&name)) - { + let declared = Self::declared_signals(tx, run_id) + .await? + .is_some_and(|names| names.contains(&name)); + if wait.is_none() && !declared { let reason = if open.is_some() { "signal_wait_mismatch" } else { "signal_wait_not_open" }; return Err(refuse(StatusCode::CONFLICT, reason)); } @@ -1156,8 +1155,13 @@ where .bind(&command_id) .execute(&mut **tx) .await?; + // PHP's `applySignal` takes a sequence only from the signal's own + // wait, so a declared signal applied under a condition wait keeps no + // workflow sequence. Only an undeclared signal takes the condition + // wait's (`markConditionWaitSignalConsumed`). + let record_sequence = if direct || !declared { sequence } else { None }; Self::query("UPDATE workflow_signal_records SET status='applied',workflow_sequence=$1,applied_at=$2,closed_at=$3 WHERE id=$4") - .bind(DB::optional_integer(sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; + .bind(DB::optional_integer(record_sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; let run = Self::query( "SELECT workflow_instance_id,message_cursor_position FROM workflow_runs WHERE id=$1", ) @@ -1798,10 +1802,11 @@ where } } } - let (events, next) = match page { + let (mut events, next) = match page { Some(size) => Self::history_page_connection(&mut tx, &run_id, 0, size).await?, None => (Self::whole_history(&mut tx, &run_id).await?, None), }; + Self::with_signal_sequences(&mut tx, &run_id, &mut events).await?; let budget = Self::workflow_history_budget( &mut tx, &run_id, @@ -1865,7 +1870,8 @@ where } Self::workflow_task_guard(&mut tx, &task, &body).await?; let run_id = DB::string(&task, "workflow_run_id")?; - let (events, next) = Self::history_page_connection(&mut tx, &run_id, after, size).await?; + let (mut events, next) = Self::history_page_connection(&mut tx, &run_id, after, size).await?; + Self::with_signal_sequences(&mut tx, &run_id, &mut events).await?; let run = Self::query("SELECT last_history_sequence FROM workflow_runs WHERE id=$1") .bind(&run_id) .fetch_one(&mut *tx) @@ -3565,6 +3571,33 @@ where /// A poll's whole history, read in bounded pages: published PHP sends /// every event unless the worker asks for pages. + /// `WorkflowTaskPoller::historyEventsWithSignalArgumentEnvelopes`: the + /// history a worker reads shows a consumed signal's workflow sequence on + /// its `SignalReceived`, from the signal's record. The run's own history + /// route shows the events as recorded. + async fn with_signal_sequences(connection: &mut DB::Connection, run_id: &str, events: &mut [Value]) -> Result<()> { + if !events.iter().any(|event| event["event_type"] == "SignalReceived") { + return Ok(()); + } + let rows = Self::query("SELECT id,workflow_sequence FROM workflow_signal_records WHERE workflow_run_id=$1 AND workflow_sequence IS NOT NULL") + .bind(run_id) + .fetch_all(&mut *connection) + .await?; + let mut sequences = std::collections::HashMap::new(); + for row in rows { + sequences.insert(DB::string(&row, "id")?, DB::number(&row, "workflow_sequence")?); + } + for event in events.iter_mut().filter(|event| event["event_type"] == "SignalReceived") { + let sequence = event["payload"]["signal_id"].as_str().and_then(|id| sequences.get(id)).copied(); + if let (Some(sequence), Some(payload)) = (sequence, event["payload"].as_object_mut()) { + if payload.get("workflow_sequence").is_none_or(Value::is_null) { + payload.insert("workflow_sequence".into(), json!(sequence)); + } + } + } + Ok(()) + } + async fn whole_history(connection: &mut DB::Connection, run_id: &str) -> Result> { let mut events = Vec::new(); let mut after = 0; @@ -3961,6 +3994,7 @@ pub(crate) fn capabilities() -> Value { "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "update_validation_tasks": true, "synchronous_update_validation": super::update_validation::capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); value["runtime_external_payload_transport"] = super::payload_discovery::transport_manifest(); diff --git a/rust/src/runtime/update_validation.rs b/rust/src/runtime/update_validation.rs new file mode 100644 index 000000000..b79de1e6f --- /dev/null +++ b/rust/src/runtime/update_validation.rs @@ -0,0 +1,1010 @@ +//! Published PHP 2.11.1's synchronous update validation +//! (`WorkflowUpdateValidationTaskBroker` and +//! `ValidatedExternalWorkflowUpdateAdmission`). +//! +//! An update whose durable contract declares a validator is not admitted until +//! a validator-capable worker approves it. The caller waits up to +//! `DW_UPDATE_VALIDATION_TIMEOUT` seconds. A worker claims the validation task +//! for `DW_UPDATE_VALIDATION_LEASE_TIMEOUT` seconds and approves or rejects it. +//! Every outcome stays on the task row, so a retry with the same `request_id` +//! and arguments gets the same answer. +use super::{Result, backend::Backend, refuse, store::{Store, id, now}, text, wire}; +use axum::http::StatusCode; +use serde_json::{Map, Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::sync::OnceLock; +use std::time::Duration; + +pub(super) const CAPABILITY: &str = "update_validation_tasks"; +const CONTRACT_FIELD: &str = "update_validators"; +const TERMINAL: [&str; 4] = ["approved", "rejected", "failed", "timed_out"]; +pub(super) const VALIDATOR_REJECTED: &str = "update_validator_rejected"; +const TIMEOUT_MESSAGE: &str = "Synchronous update validation did not complete before the acceptance deadline."; +const NOT_ADVERTISED: &str = "Worker registration does not advertise synchronous update validation."; + +/// `server.update_validation`: how long a caller waits for a validator, and +/// how long a worker's claim lasts. +pub(super) fn timeouts() -> (i64, i64) { + static TIMEOUTS: OnceLock<(i64, i64)> = OnceLock::new(); + *TIMEOUTS.get_or_init(|| crate::config_audit::update_validation_timeouts(|key| std::env::var(key).ok())) +} + +/// `WorkerProtocol::serverCapabilities()['synchronous_update_validation']`. +pub(super) fn capability() -> Value { + let (timeout, lease) = timeouts(); + json!({"supported":true,"minimum_protocol_version":"1.13","acceptance_boundary":"validator_approved", + "worker_capability":CAPABILITY,"workflow_contract_field":CONTRACT_FIELD, + "validator_execution":"replay_without_handler_or_state_commit","request_id_required":true, + "duplicate_completion":"typed_rejection","stale_completion":"typed_rejection", + "missing_contract":"typed_rejection","unsupported_worker":"typed_rejection", + "task_poll":{"strategy":"multiplexed","endpoint":"/worker/workflow-tasks/poll","request_field":"task_kinds", + "task_kinds":["workflow","update_validation"],"default_task_kinds":["workflow"], + "response_discriminator":"task.task_kind","poll_request_id_binding":"normalized_task_kind_set", + "poll_request_id_conflict_reason":"poll_request_task_kinds_conflict"}, + "completion":{"approve_endpoint":"/worker/update-validation-tasks/{taskId}/approve", + "reject_endpoint":"/worker/update-validation-tasks/{taskId}/reject", + "fence_fields":["lease_owner","update_validation_attempt"], + "typed_failure_reasons":["update_validation_task_not_found","duplicate_update_validation_completion", + "update_validation_task_not_leased","update_validation_lease_owner_mismatch", + "stale_update_validation_completion","update_validation_lease_expired","update_validator_worker_lost"]}, + "timeout_seconds":timeout,"lease_timeout_seconds":lease, + "sdk_surfaces":{"sdk-python":"validator_capable","sdk-php":"validators_not_exposed","sdk-rust":"validators_not_exposed"}}) +} + +/// The 409 a poll gets from a registration without the capability. +pub(super) fn not_advertised() -> super::RuntimeError { + super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task":null,"poll_status":"unsupported", + "reason":"update_validation_capability_not_advertised","error":NOT_ADVERTISED}), + } +} + +/// `stringValue`: a trimmed, non-empty string. +fn string_value(value: &Value) -> Option<&str> { + value.as_str().map(str::trim).filter(|text| !text.is_empty()) +} + +/// `stringList`: trimmed, unique, non-empty strings. +fn string_list(value: &Value) -> Vec { + let mut list: Vec = value + .as_array() + .map(|items| items.iter().filter_map(string_value).map(str::to_owned).collect()) + .unwrap_or_default(); + list.sort(); + list.dedup(); + list +} + +/// `validationErrors`: field names mapped to their non-empty string messages. +pub(super) fn validation_errors(value: &Value) -> Map { + let fields: Vec<(String, &Value)> = match value { + Value::Object(map) => map.iter().map(|(field, messages)| (field.clone(), messages)).collect(), + Value::Array(list) => list.iter().enumerate().map(|(field, messages)| (field.to_string(), messages)).collect(), + _ => vec![], + }; + let mut errors = Map::new(); + for (field, messages) in fields { + let Some(messages) = messages.as_array() else { continue }; + let kept: Vec = messages + .iter() + .filter(|message| message.as_str().is_some_and(|text| !text.is_empty())) + .cloned() + .collect(); + if !kept.is_empty() { + errors.insert(field, Value::Array(kept)); + } + } + errors +} + +fn sha256(text: &str) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(text.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// `ValidatedExternalWorkflowUpdateAdmission::validationFailure`. +pub(super) fn admission_failure( + workflow_id: &str, + run_id: &str, + name: &str, + reason: &str, + message: &str, + retryable: bool, + validation_errors: Option, +) -> Value { + let mut body = json!({"workflow_id":workflow_id,"run_id":run_id,"update_name":name, + "outcome":"update_validation_failed","reason":reason,"rejection_reason":reason,"message":message}); + if let Some(errors) = validation_errors.filter(|errors| errors.as_object().is_some_and(|map| !map.is_empty())) { + body["validation_errors"] = errors; + } + body["retryable"] = json!(retryable); + body +} + +/// `WorkflowUpdateValidationTaskBroker::failure`. +#[allow(clippy::too_many_arguments)] +fn broker_failure( + workflow_id: &str, + run_id: &str, + name: &str, + reason: &str, + message: &str, + errors: Map, + update_id: &str, + retryable: bool, +) -> Value { + let rejected = reason == VALIDATOR_REJECTED; + let mut body = json!({"approved":false,"workflow_id":workflow_id,"run_id":run_id,"update_name":name,"update_id":update_id}); + if rejected { + body["update_status"] = json!("rejected"); + } + body["outcome"] = json!(if rejected { "rejected_invalid_arguments" } else { "update_validation_failed" }); + body["reason"] = json!(reason); + body["rejection_reason"] = json!(reason); + body["message"] = json!(message); + if !errors.is_empty() { + body["validation_errors"] = Value::Object(errors); + } + body["retryable"] = json!(retryable); + body +} + +/// `staleValidationFailure`: the run moved on between validation and commit. +pub(super) fn stale_failure(workflow_id: &str, run_id: &str, name: &str, update_id: &str, message: &str) -> (StatusCode, Value) { + let mut body = json!({"workflow_id":workflow_id}); + if !run_id.is_empty() { + body["run_id"] = json!(run_id); + } + body["update_name"] = json!(name); + body["update_id"] = json!(update_id); + body["outcome"] = json!("update_validation_failed"); + body["reason"] = json!("stale_update_validation_completion"); + body["rejection_reason"] = json!("stale_update_validation_completion"); + body["message"] = json!(message); + body["retryable"] = json!(true); + (StatusCode::CONFLICT, body) +} + +/// `RunCommandContract::forRun` from durable history: the start event holds +/// every declared contract field, and each declared query and update has a +/// handler contract. +pub(super) fn durable_contract(started: &Value) -> bool { + const FIELDS: [&str; 9] = ["declared_queries", "declared_query_contracts", "declared_signals", + "declared_signal_contracts", "declared_updates", "declared_update_contracts", "declared_entry_method", + "declared_entry_mode", "declared_entry_declaring_class"]; + if !FIELDS.iter().all(|field| started.get(*field).is_some()) { + return false; + } + let lists = ["declared_queries", "declared_query_contracts", "declared_signals", "declared_signal_contracts", + "declared_updates", "declared_update_contracts"]; + if !lists.iter().all(|field| started[*field].is_array()) { + return false; + } + if started["declared_entry_method"] != "handle" + || started["declared_entry_mode"] != "canonical" + || string_value(&started["declared_entry_declaring_class"]).is_none() + { + return false; + } + let covered = |names: &str, contracts: &str| { + let handled: Vec<&str> = started[contracts] + .as_array() + .map(|items| items.iter().filter_map(|item| string_value(&item["name"])).collect()) + .unwrap_or_default(); + string_list(&started[names]).iter().all(|name| handled.contains(&name.as_str())) + }; + covered("declared_queries", "declared_query_contracts") && covered("declared_updates", "declared_update_contracts") +} + +/// What one update's validation decided. +pub(super) enum Validation { + /// Admit the update under the task's id. + Approved { task_id: String, input_hash: String }, + /// The validator refused it; the rejection is recorded durably. + Rejected { failure: Value, validation_errors: Value }, + /// Anything else: answer with this status and body, recording nothing. + Failed(StatusCode, Value), +} + +/// The update a caller asked to validate. +pub(super) struct ValidationRequest<'a> { + pub(super) workflow_id: &'a str, + pub(super) run_id: &'a str, + pub(super) workflow_type: &'a str, + pub(super) task_queue: &'a str, + pub(super) compatibility: Option<&'a str>, + pub(super) fingerprint: Option<&'a str>, + pub(super) name: &'a str, + pub(super) request_id: &'a str, + pub(super) arguments: &'a str, + pub(super) context: Option<&'a Value>, +} + +/// `inputHash`: the codec and serialized arguments. +pub(super) fn input_hash(arguments: &str) -> String { + sha256(&format!("avro\0{arguments}")) +} + +/// One registration as validation routing reads it. +struct Registration { + worker_id: String, + fresh: bool, + capable: bool, + workflow_types: Vec, + contracts: Value, + build_id: Option, + fingerprints: Value, +} + +impl Registration { + /// `workerMatchesRun` and the type, contract, build and fingerprint + /// parts of `workerMatchesTask`. + fn serves(&self, workflow_type: &str, name: &str, compatibility: Option<&str>, fingerprint: Option<&str>) -> bool { + self.workflow_types.iter().any(|kind| kind == workflow_type) + && self.contracts[workflow_type].is_object() + && string_list(&self.contracts[workflow_type][CONTRACT_FIELD]).iter().any(|declared| declared == name) + && compatibility.is_none_or(|required| self.build_id.as_deref() == Some(required)) + && fingerprint.is_none_or(|recorded| self.fingerprints[workflow_type].as_str() == Some(recorded)) + } +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + fn registration(&self, row: &DB::Row) -> Result { + let fresh_after = now() - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1)); + // `WorkerPollFence::isFresh`: active, with a recent heartbeat. + let fresh = DB::optional_string(row, "status")?.as_deref() == Some("active") + && DB::optional_instant(row, "last_heartbeat_at")?.is_some_and(|heartbeat| heartbeat >= fresh_after); + let capabilities = DB::optional_document_row(row, "capabilities")?.unwrap_or(Value::Null); + Ok(Registration { + worker_id: DB::string(row, "worker_id")?, + fresh, + capable: capabilities.as_array().is_some_and(|names| names.iter().any(|name| name == CAPABILITY)), + workflow_types: string_list(&DB::optional_document_row(row, "supported_workflow_types")?.unwrap_or(Value::Null)), + contracts: DB::optional_document_row(row, "workflow_command_contracts")?.unwrap_or(Value::Null), + build_id: DB::optional_string(row, "build_id")?.filter(|build| !build.is_empty()), + fingerprints: DB::optional_document_row(row, "workflow_definition_fingerprints")?.unwrap_or(Value::Null), + }) + } + + /// Whether this registration may claim validation tasks at all. + pub(super) async fn worker_validates(&self, worker_id: &str) -> Result { + let row = Self::query("SELECT * FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + Ok(match row { + Some(row) => self.registration(&row)?.capable, + None => false, + }) + } + + /// `route`: the first missing link between the queue's workers and this + /// run's validator, with its message and whether a retry may help. + async fn validation_route(&self, request: &ValidationRequest<'_>) -> Result> { + let rows = Self::query("SELECT * FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2 AND status='active'") + .bind(self.namespace.as_ref()) + .bind(request.task_queue) + .fetch_all(&self.pool) + .await?; + let mut workers = Vec::with_capacity(rows.len()); + for row in &rows { + let worker = self.registration(row)?; + if worker.fresh { + workers.push(worker); + } + } + if workers.is_empty() { + return Ok(Some(("update_validator_worker_unavailable", + "No active worker is available to perform synchronous update validation.", true))); + } + workers.retain(|worker| worker.capable); + if workers.is_empty() { + return Ok(Some(("update_validation_capability_unsupported", + "Active workers do not advertise synchronous pre-accept update validation.", false))); + } + if !workers.iter().any(|worker| worker.serves(request.workflow_type, request.name, request.compatibility, request.fingerprint)) { + return Ok(Some(("update_validator_worker_incompatible", + "Active workers do not advertise a validator-capable implementation matching this workflow run.", true))); + } + Ok(None) + } + + /// `findOrCreateTask`: one task per run, update name and request id. + async fn validation_task(&self, request: &ValidationRequest<'_>, input_hash: &str) -> Result { + let key = sha256(&[self.namespace.as_ref(), request.run_id, request.name, request.request_id].join("\0")); + let mut tx = self.begin().await?; + let existing = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE idempotency_key=$1") + .bind(&key) + .fetch_optional(&mut *tx) + .await?; + if let Some(existing) = existing { + tx.commit().await?; + return Ok(existing); + } + let task_id = id(); + let at = now(); + Self::query("INSERT INTO workflow_update_validation_tasks(id,idempotency_key,namespace,workflow_instance_id,workflow_run_id,workflow_type,task_queue,compatibility,workflow_definition_fingerprint,update_name,request_id,input_hash,payload_codec,arguments,command_context,status,attempt_count,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,'avro',$13,$14,'pending',0,$15,$16)") + .bind(&task_id).bind(&key).bind(self.namespace.as_ref()).bind(request.workflow_id).bind(request.run_id) + .bind(request.workflow_type).bind(request.task_queue).bind(request.compatibility).bind(request.fingerprint) + .bind(request.name).bind(request.request_id).bind(input_hash).bind(request.arguments) + .bind(request.context.map(DB::document)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + let row = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE id=$1") + .bind(&task_id) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + Ok(row) + } + + /// `WorkflowUpdateValidationTaskBroker::validate`. + pub(super) async fn validate_update(&self, request: &ValidationRequest<'_>) -> Result { + let hash = input_hash(request.arguments); + let task = self.validation_task(request, &hash).await?; + let task_id = DB::string(&task, "id")?; + if DB::string(&task, "input_hash")? != hash { + return Ok(Validation::Failed(StatusCode::CONFLICT, broker_failure( + request.workflow_id, request.run_id, request.name, "update_validation_idempotency_conflict", + "The update request ID was already used with different arguments.", Map::new(), &task_id, false))); + } + if !TERMINAL.contains(&DB::string(&task, "status")?.as_str()) { + if let Some((reason, message, retryable)) = self.validation_route(request).await? { + return Ok(Validation::Failed(StatusCode::CONFLICT, broker_failure( + request.workflow_id, request.run_id, request.name, reason, message, Map::new(), &task_id, retryable))); + } + // Pollers on this node wake now; other nodes find the task on + // their next read. + self.wake.notify_waiters(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(timeouts().0 as u64); + loop { + let wake = self.wake.notified(); + let status = Self::query("SELECT status FROM workflow_update_validation_tasks WHERE id=$1") + .bind(&task_id) + .fetch_one(&self.pool) + .await?; + if TERMINAL.contains(&DB::string(&status, "status")?.as_str()) { + break; + } + if tokio::time::Instant::now() >= deadline { + self.time_out_validation(&task_id).await?; + break; + } + tokio::select! { + _ = wake => {}, + _ = tokio::time::sleep(Duration::from_millis(50)) => {}, + _ = tokio::time::sleep_until(deadline) => {}, + } + } + } + let task = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE id=$1") + .bind(&task_id) + .fetch_one(&self.pool) + .await?; + self.validation_result(&task, request) + } + + /// `result`: an approval, or the stored failure with its HTTP status. + fn validation_result(&self, task: &DB::Row, request: &ValidationRequest<'_>) -> Result { + let task_id = DB::string(task, "id")?; + let status = DB::string(task, "status")?; + if status == "approved" { + return Ok(Validation::Approved { task_id, input_hash: DB::string(task, "input_hash")? }); + } + let reason = DB::optional_string(task, "rejection_reason")?.unwrap_or_else(|| "update_validation_failed".into()); + let message = DB::optional_string(task, "rejection_message")?.unwrap_or_else(|| "Synchronous update validation failed.".into()); + let errors = validation_errors(&DB::optional_document_row(task, "validation_errors")?.unwrap_or(Value::Null)); + let mut failure = broker_failure(request.workflow_id, request.run_id, request.name, &reason, &message, errors.clone(), &task_id, false); + if let Some(kind) = DB::optional_string(task, "failure_type")? { + failure["failure_type"] = json!(kind); + } + failure["update_validation_attempt"] = json!(DB::number(task, "attempt_count")?); + if status == "rejected" && reason == VALIDATOR_REJECTED { + return Ok(Validation::Rejected { failure, validation_errors: Value::Object(errors) }); + } + let http = match status.as_str() { + "rejected" => StatusCode::UNPROCESSABLE_ENTITY, + "timed_out" => StatusCode::GATEWAY_TIMEOUT, + _ => StatusCode::CONFLICT, + }; + Ok(Validation::Failed(http, failure)) + } + + /// `markTimedOut`: the reason names what was still missing. + async fn time_out_validation(&self, task_id: &str) -> Result<()> { + let mut tx = self.begin().await?; + let task = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE id=$1") + .bind(task_id) + .fetch_one(&mut *tx) + .await?; + let status = DB::string(&task, "status")?; + if TERMINAL.contains(&status.as_str()) { + tx.commit().await?; + return Ok(()); + } + let owner = DB::optional_string(&task, "lease_owner")?; + let mut lost = false; + if status == "leased" { + let worker = match owner.as_deref() { + Some(owner) => Self::query("SELECT * FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(DB::string(&task, "namespace")?) + .bind(owner) + .fetch_optional(&mut *tx) + .await?, + None => None, + }; + lost = match worker { + Some(worker) => !self.registration(&worker)?.fresh, + None => true, + }; + } + let (reason, message) = if lost { + ("update_validator_worker_lost", "The validator worker was lost before returning a validation result.") + } else if status == "pending" { + ("update_validation_task_not_claimed", TIMEOUT_MESSAGE) + } else { + ("update_validation_execution_timeout", TIMEOUT_MESSAGE) + }; + let at = now(); + Self::query("UPDATE workflow_update_validation_tasks SET status='timed_out',rejection_reason=$1,rejection_message=$2,timed_out_at=$3,lease_expires_at=NULL,updated_at=$4 WHERE id=$5") + .bind(reason).bind(message).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(task_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// `approvedTask`: the approval still stands for this run and input. + pub(super) async fn approved_validation( + tx: &mut super::transaction::Transaction<'_, DB>, + task_id: &str, + run_id: &str, + input_hash: &str, + ) -> Result { + Ok(Self::query("SELECT id FROM workflow_update_validation_tasks WHERE id=$1 AND workflow_run_id=$2 AND input_hash=$3 AND status='approved'") + .bind(task_id).bind(run_id).bind(input_hash) + .fetch_optional(&mut **tx) + .await? + .is_some()) + } + + /// `claim`: the oldest claimable task this worker can validate, leased + /// to it for the lease timeout. + pub(super) async fn claim_update_validation(&self, worker_id: &str, task_queue: &str) -> Result> { + let mut tx = self.begin().await?; + let Some(worker) = Self::query("SELECT * FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await? + else { + tx.commit().await?; + return Ok(None); + }; + let worker = self.registration(&worker)?; + if !worker.fresh || !worker.capable { + tx.commit().await?; + return Ok(None); + } + let at = now(); + let tasks = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE namespace=$1 AND task_queue=$2 AND (status='pending' OR (status='leased' AND lease_expires_at<=$3)) ORDER BY created_at,id LIMIT 25") + .bind(self.namespace.as_ref()) + .bind(task_queue) + .bind(DB::bind_time(at)) + .fetch_all(&mut *tx) + .await?; + for task in tasks { + let compatibility = DB::optional_string(&task, "compatibility")?; + let fingerprint = DB::optional_string(&task, "workflow_definition_fingerprint")?; + if !worker.serves(&DB::string(&task, "workflow_type")?, &DB::string(&task, "update_name")?, + compatibility.as_deref(), fingerprint.as_deref()) + { + continue; + } + let task_id = DB::string(&task, "id")?; + let attempt = DB::number(&task, "attempt_count")?.saturating_add(1); + Self::query("UPDATE workflow_update_validation_tasks SET status='leased',attempt_count=$1,lease_owner=$2,lease_expires_at=$3,updated_at=$4 WHERE id=$5") + .bind(attempt).bind(&worker.worker_id) + .bind(DB::bind_time(at + chrono::Duration::seconds(timeouts().1))) + .bind(DB::bind_time(at)).bind(&task_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + return Ok(Some(task_id)); + } + tx.commit().await?; + Ok(None) + } + + /// `taskPayload` without its history export, which the runtime adds. + pub(super) async fn update_validation_payload(&self, task_id: &str) -> Result { + let task = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE id=$1") + .bind(task_id) + .fetch_one(&self.pool) + .await?; + let run_id = DB::string(&task, "workflow_run_id")?; + let mut payload = json!({}); + let mut connection = self.pool.acquire().await?; + if let Some(run) = Self::query("SELECT workflow_class,arguments,status,last_history_sequence FROM workflow_runs WHERE id=$1") + .bind(&run_id) + .fetch_optional(&mut *connection) + .await? + { + // `workflowReplayPayload`: the whole history, unpaged. + let mut history = Vec::new(); + let mut after = 0; + loop { + let (page, next) = Self::history_page_connection(&mut connection, &run_id, after, 1000).await?; + after = page.last().and_then(|event| event["sequence"].as_i64()).unwrap_or(after); + history.extend(page); + if next.is_none() { + break; + } + } + payload = json!({"workflow_class":DB::optional_string(&run, "workflow_class")?, + "workflow_arguments":DB::optional_string(&run, "arguments")?.as_deref().map(wire), + "run_status":DB::string(&run, "status")?, + "last_history_sequence":DB::optional_number(&run, "last_history_sequence")?.unwrap_or(0), + "history_events":history}); + } + let fields = json!({"task_kind":"update_validation","update_validation_task_id":task_id,"update_id":task_id, + "update_validation_attempt":DB::number(&task, "attempt_count")?, + "workflow_id":DB::string(&task, "workflow_instance_id")?,"run_id":run_id, + "workflow_type":DB::string(&task, "workflow_type")?,"compatibility":DB::optional_string(&task, "compatibility")?, + "update_name":DB::string(&task, "update_name")?,"update_arguments":wire(&DB::string(&task, "arguments")?), + "payload_codec":DB::string(&task, "payload_codec")?,"task_queue":DB::string(&task, "task_queue")?, + "lease_owner":DB::optional_string(&task, "lease_owner")?, + "lease_expires_at":super::task_queues::json_time(DB::optional_instant(&task, "lease_expires_at")?)}); + payload.as_object_mut().expect("payload").extend(fields.as_object().expect("fields").clone()); + Ok(payload) + } + + /// `guardCompletion`, then `approve` or `reject`. + pub(super) async fn complete_update_validation(&self, task_id: &str, body: &Value, failure: Option<&Value>) -> Result<(StatusCode, Value)> { + let owner = text(body, "lease_owner")?; + let attempt = super::laravel_validation::integer(&body["update_validation_attempt"]) + .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "validation_failed"))?; + let mut tx = self.begin().await?; + let task = Self::query("SELECT * FROM workflow_update_validation_tasks WHERE id=$1") + .bind(task_id) + .fetch_optional(&mut *tx) + .await? + .filter(|task| DB::string(task, "namespace").ok().as_deref() == Some(self.namespace.as_ref())); + let refusal = |reason: &str, message: &str, status: StatusCode, current: Option| { + let mut body = json!({"update_validation_task_id":task_id,"update_id":task_id}); + if let Some(current) = current { + body["update_validation_attempt"] = json!(current); + } + body["outcome"] = json!("rejected"); + body["reason"] = json!(reason); + body["error"] = json!(message); + body["status"] = json!(status.as_u16()); + Ok((status, body)) + }; + let Some(task) = task else { + tx.commit().await?; + return refusal("update_validation_task_not_found", "Update validation task not found.", StatusCode::NOT_FOUND, None); + }; + let status = DB::string(&task, "status")?; + let current = DB::number(&task, "attempt_count")?; + let lease_owner = DB::optional_string(&task, "lease_owner")?; + let guard = if TERMINAL.contains(&status.as_str()) { + Some(if current != attempt || lease_owner.as_deref() != Some(owner) { + ("stale_update_validation_completion", "Update validation completion belongs to a stale delivery attempt.") + } else { + ("duplicate_update_validation_completion", "The update validation task already has a terminal result.") + }) + } else if status != "leased" { + Some(("update_validation_task_not_leased", "Update validation task is not leased.")) + } else if lease_owner.as_deref() != Some(owner) { + Some(("update_validation_lease_owner_mismatch", "Update validation lease is owned by another worker.")) + } else if current != attempt { + Some(("stale_update_validation_completion", "Update validation completion belongs to a stale delivery attempt.")) + } else if DB::optional_instant(&task, "lease_expires_at")?.is_none_or(|expires| expires <= now()) { + Some(("update_validation_lease_expired", "Update validation task lease has expired.")) + } else { + let worker = Self::query("SELECT * FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(self.namespace.as_ref()) + .bind(owner) + .fetch_optional(&mut *tx) + .await?; + let fresh = match worker { + Some(worker) => self.registration(&worker)?.fresh, + None => false, + }; + (!fresh).then_some(("update_validator_worker_lost", "The validator worker is no longer active.")) + }; + if let Some((reason, message)) = guard { + tx.commit().await?; + return refusal(reason, message, StatusCode::CONFLICT, Some(current)); + } + let at = now(); + let mut answer = json!({"update_validation_task_id":task_id,"update_id":task_id,"update_validation_attempt":attempt}); + match failure { + None => { + Self::query("UPDATE workflow_update_validation_tasks SET status='approved',approved_at=$1,lease_expires_at=NULL,updated_at=$2 WHERE id=$3") + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(task_id) + .execute(&mut *tx) + .await?; + answer["outcome"] = json!("approved"); + } + Some(failure) => { + let reason = string_value(&failure["reason"]).unwrap_or(VALIDATOR_REJECTED).to_owned(); + let rejected = reason == VALIDATOR_REJECTED; + let message = string_value(&failure["message"]).map(str::to_owned).unwrap_or_else(|| { + if rejected { "The update validator rejected the request." } else { "Update validation failed." }.to_owned() + }); + let errors = Value::Object(validation_errors(&failure["validation_errors"])); + let sql = if rejected { + "UPDATE workflow_update_validation_tasks SET status='rejected',rejection_reason=$1,rejection_message=$2,failure_type=$3,validation_errors=$4,rejected_at=$5,failed_at=NULL,lease_expires_at=NULL,updated_at=$6 WHERE id=$7" + } else { + "UPDATE workflow_update_validation_tasks SET status='failed',rejection_reason=$1,rejection_message=$2,failure_type=$3,validation_errors=$4,rejected_at=NULL,failed_at=$5,lease_expires_at=NULL,updated_at=$6 WHERE id=$7" + }; + Self::query(sql) + .bind(reason.as_str()).bind(message).bind(string_value(&failure["type"])) + .bind(DB::document(&errors)).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(task_id) + .execute(&mut *tx) + .await?; + answer["outcome"] = json!(if rejected { "rejected" } else { "failed" }); + answer["reason"] = json!(reason); + } + } + answer["status"] = json!(200); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::OK, answer)) + } +} + +/// The rules `pollUpdateValidationTasks` applies. +pub(super) fn poll_rules(body: &Value) -> Result<()> { + use super::laravel_validation::{Rule, failure_messages, rule_errors}; + let rules: &[(&str, &[Rule])] = &[ + ("worker_id", &[Rule::Required, Rule::String]), + ("task_queue", &[Rule::Required, Rule::String]), + ("timeout_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(0), Rule::Max(60)]), + ]; + failure_messages(rule_errors(body, rules, |_, _| None)).map_or(Ok(()), Err) +} + +/// The rules `approveUpdateValidationTask` and `rejectUpdateValidationTask` +/// apply, nested failure fields named as Laravel names them. +pub(super) fn completion_rules(body: &Value, reject: bool) -> Result<()> { + use super::laravel_validation::{Rule, failure_messages, rule_errors}; + let mut flat = Map::new(); + let mut rules: Vec<(String, Vec)> = vec![ + ("lease_owner".into(), vec![Rule::Required, Rule::String]), + ("update_validation_attempt".into(), vec![Rule::Required, Rule::Integer, Rule::Min(1)]), + ]; + for field in ["lease_owner", "update_validation_attempt"] { + if let Some(value) = body.get(field) { + flat.insert(field.into(), value.clone()); + } + } + if reject { + if let Some(failure) = body.get("failure") { + flat.insert("failure".into(), failure.clone()); + } + rules.push(("failure".into(), vec![Rule::Required, Rule::Array])); + let failure = body.get("failure").filter(|failure| failure.is_object()); + let nested = |field: &str| failure.and_then(|failure| failure.get(field)); + for (field, field_rules) in [ + ("message", vec![Rule::Required, Rule::String]), + ("reason", vec![Rule::Nullable, Rule::String]), + ("type", vec![Rule::Nullable, Rule::String]), + ("stack_trace", vec![Rule::Nullable, Rule::String]), + ("validation_errors", vec![Rule::Nullable, Rule::Array]), + ] { + let name = format!("failure.{field}"); + if let Some(value) = nested(field) { + flat.insert(name.clone(), value.clone()); + } + rules.push((name, field_rules)); + } + if let Some(errors) = nested("validation_errors").and_then(Value::as_object) { + for (field, messages) in errors { + let name = format!("failure.validation_errors.{field}"); + flat.insert(name.clone(), messages.clone()); + rules.push((name.clone(), vec![Rule::Array])); + for (index, message) in messages.as_array().into_iter().flatten().enumerate() { + let item = format!("{name}.{index}"); + flat.insert(item.clone(), message.clone()); + rules.push((item, vec![Rule::String])); + } + } + } + } + let borrowed: Vec<(&str, &[Rule])> = rules.iter().map(|(field, rules)| (field.as_str(), rules.as_slice())).collect(); + failure_messages(rule_errors(&Value::Object(flat), &borrowed, |_, _| None)).map_or(Ok(()), Err) +} + +/// `task_kinds`: `nullable|array|min:1|max:2`, each a distinct `workflow` or +/// `update_validation`. Absent or null means workflow tasks only. +pub(super) fn task_kinds(body: &Value) -> Result> { + use super::laravel_validation::{Rule, failure_messages, rule_errors}; + let Some(kinds) = body.get("task_kinds").filter(|kinds| !kinds.is_null()) else { + return Ok(vec!["workflow"]); + }; + let mut errors = rule_errors(body, &[("task_kinds", &[Rule::Nullable, Rule::Array, Rule::Min(1), Rule::Max(2)])], |_, _| None); + let items: Vec<(String, Value)> = match kinds { + Value::Array(items) => items.iter().enumerate().map(|(index, item)| (index.to_string(), item.clone())).collect(), + Value::Object(items) => items.iter().map(|(key, item)| (key.clone(), item.clone())).collect(), + _ => vec![], + }; + for (key, item) in &items { + let field = format!("task_kinds.{key}"); + let mut messages = Vec::new(); + if !item.is_string() { + messages.push(format!("The {field} field must be a string.")); + } + if items.iter().filter(|(_, other)| other == item).count() > 1 { + messages.push(format!("The {field} field has a duplicate value.")); + } + if !matches!(item.as_str(), Some("workflow" | "update_validation")) { + messages.push(format!("The selected {field} is invalid.")); + } + errors.push((field, messages)); + } + if let Some(error) = failure_messages(errors) { + return Err(error); + } + let mut requested = Vec::new(); + for kind in ["workflow", "update_validation"] { + if items.iter().any(|(_, item)| item == kind) { + requested.push(kind); + } + } + Ok(requested) +} + +/// What a multiplexed poll leased. +pub(super) enum Leased { + /// A workflow poll's answer, leased or not. + Workflow(Value), + /// A validation task's id, leased to the polling worker. + Validation(String), +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `resolveRegisteredWorker` for a validation poll, then its capability + /// check. Answers the registration's supported workflow types. + pub(super) async fn validation_poller(&self, worker_id: &str, task_queue: &str) -> Result> { + let Some(row) = Self::query("SELECT * FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + else { + return Err(super::RuntimeError::Protocol { + status: StatusCode::PRECONDITION_FAILED, + response: json!({"error":"Worker must be registered before polling. Call POST /worker/register first.", + "reason":"worker_not_registered","worker_id":worker_id}), + }); + }; + let registered = DB::string(&row, "task_queue")?; + if registered != task_queue { + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"error":format!("Worker [{worker_id}] is registered for task queue [{registered}], not [{task_queue}]."), + "reason":"task_queue_mismatch","worker_id":worker_id,"registered_task_queue":registered, + "requested_task_queue":task_queue}), + }); + } + self.refuse_draining_worker(worker_id, task_queue).await?; + Ok(self.registration(&row)?.workflow_types) + } + + /// A dedicated validation poll: the registration, its capability, then + /// the wait for a claim. + pub(super) async fn update_validation_poll(&self, worker_id: &str, task_queue: &str, timeout: u64) -> Result> { + self.validation_poller(worker_id, task_queue).await?; + if !self.worker_validates(worker_id).await? { + return Err(not_advertised()); + } + self.poll_update_validation(worker_id, task_queue, timeout).await + } + + /// A workflow poll that asked for validation tasks: the registration, + /// a workflow worker, the capability, then both kinds. + pub(super) async fn multiplexed_poll( + &self, + body: &Value, + kinds: &[&str], + protocol_version: &str, + pages: super::history_pages::HistoryPages, + timeout: u64, + ) -> Result { + let worker_id = text(body, "worker_id")?; + let types = self.validation_poller(worker_id, text(body, "task_queue")?).await?; + if types.is_empty() { + return Ok(Leased::Workflow(json!({"task":null,"poll_status":"no_workflow_capability"}))); + } + if !self.worker_validates(worker_id).await? { + return Err(not_advertised()); + } + self.poll_task_kinds(body, kinds, protocol_version, pages, timeout).await + } + + /// Whether a claimable validation task waits on this queue. + async fn validation_waiting(&self, task_queue: &str) -> Result { + Ok(Self::query("SELECT id FROM workflow_update_validation_tasks WHERE namespace=$1 AND task_queue=$2 AND (status='pending' OR (status='leased' AND lease_expires_at<=$3)) LIMIT 1") + .bind(self.namespace.as_ref()) + .bind(task_queue) + .bind(DB::bind_time(now())) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// `poll`: claim a validation task, waiting up to `timeout` seconds. + pub(super) async fn poll_update_validation(&self, worker_id: &str, task_queue: &str, timeout: u64) -> Result> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout); + let mut pace = super::poll_readiness::ReadPace::new(); + let mut first = true; + loop { + let wake = self.wake.notified(); + if (std::mem::take(&mut first) || self.validation_waiting(task_queue).await?) + && let Some(task_id) = self.claim_update_validation(worker_id, task_queue).await? + { + return Ok(Some(task_id)); + } + if tokio::time::Instant::now() >= deadline { + return Ok(None); + } + pace.wait(wake, deadline).await; + } + } + + /// `WorkflowTaskKindSelector`'s cursor: which kind this queue tries first. + async fn first_task_kind(&self, task_queue: &str) -> Result { + Ok(Self::query("SELECT next_task_kind FROM workflow_task_poll_cursors WHERE namespace=$1 AND task_queue=$2") + .bind(self.namespace.as_ref()) + .bind(task_queue) + .fetch_optional(&self.pool) + .await? + .map(|row| DB::string(&row, "next_task_kind")) + .transpose()? + .unwrap_or_else(|| "update_validation".into())) + } + + /// After a lease, the queue tries the other kind first. + async fn turn_task_kind(&self, task_queue: &str, leased: &str) -> Result<()> { + let next = if leased == "workflow" { "update_validation" } else { "workflow" }; + let at = now(); + let mut tx = self.begin().await?; + // MySQL counts only changed rows, so look before writing. + let cursor = Self::query("SELECT next_task_kind FROM workflow_task_poll_cursors WHERE namespace=$1 AND task_queue=$2") + .bind(self.namespace.as_ref()).bind(task_queue) + .fetch_optional(&mut *tx) + .await?; + if cursor.is_some() { + Self::query("UPDATE workflow_task_poll_cursors SET next_task_kind=$1,updated_at=$2 WHERE namespace=$3 AND task_queue=$4") + .bind(next).bind(DB::bind_time(at)).bind(self.namespace.as_ref()).bind(task_queue) + .execute(&mut *tx) + .await?; + } else { + Self::query("INSERT INTO workflow_task_poll_cursors(namespace,task_queue,next_task_kind,created_at,updated_at) VALUES ($1,$2,$3,$4,$5)") + .bind(self.namespace.as_ref()).bind(task_queue).bind(next).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// `WorkflowTaskPoller` with both kinds requested: the queue's cursor + /// picks the kind tried first, and a lease turns it to the other kind. + /// When neither kind has work, the workflow poll's answer stands. + pub(super) async fn poll_task_kinds( + &self, + body: &Value, + kinds: &[&str], + protocol_version: &str, + pages: super::history_pages::HistoryPages, + timeout: u64, + ) -> Result { + let worker_id = text(body, "worker_id")?.to_owned(); + let task_queue = text(body, "task_queue")?.to_owned(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout); + let mut probe = body.clone(); + probe["timeout_seconds"] = json!(0); + let mut pace = super::poll_readiness::ReadPace::new(); + loop { + let wake = self.wake.notified(); + let first = self.first_task_kind(&task_queue).await?; + let mut order: Vec<&str> = kinds.to_vec(); + order.sort_by_key(|kind| *kind != first); + let mut workflow = None; + for kind in order { + if kind == "workflow" { + let answer = self.poll_with_pages(probe.clone(), "workflow", protocol_version, pages, 0).await?; + if !answer["task"].is_null() { + if kinds.len() > 1 { + self.turn_task_kind(&task_queue, "workflow").await?; + } + return Ok(Leased::Workflow(answer)); + } + workflow = Some(answer); + } else if let Some(task_id) = self.claim_update_validation(&worker_id, &task_queue).await? { + if kinds.len() > 1 { + self.turn_task_kind(&task_queue, "update_validation").await?; + } + return Ok(Leased::Validation(task_id)); + } + } + if tokio::time::Instant::now() >= deadline { + return Ok(Leased::Workflow(workflow.unwrap_or_else(|| { + json!({"task":null,"poll_status":"empty","server_capabilities":super::store::capabilities()}) + }))); + } + pace.wait(wake, deadline).await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validation_errors_keep_non_empty_string_messages() { + let errors = validation_errors(&json!({"v":["too small", "", 3],"w":"not a list","x":[]})); + assert_eq!(Value::Object(errors), json!({"v":["too small"]})); + let listed = validation_errors(&json!([["first"]])); + assert_eq!(Value::Object(listed), json!({"0":["first"]})); + } + + #[test] + fn broker_failures_mark_only_validator_rejections_rejected() { + let rejected = broker_failure("w", "r", "u", VALIDATOR_REJECTED, "no", Map::new(), "t", false); + assert_eq!(rejected["update_status"], "rejected"); + assert_eq!(rejected["outcome"], "rejected_invalid_arguments"); + let failed = broker_failure("w", "r", "u", "validator_error", "no", Map::new(), "t", false); + assert!(failed.get("update_status").is_none()); + assert_eq!(failed["outcome"], "update_validation_failed"); + assert_eq!(failed["approved"], false); + } + + #[test] + fn registrations_serve_only_runs_their_contract_validates() { + let worker = Registration { + worker_id: "w".into(), + fresh: true, + capable: true, + workflow_types: vec!["t".into()], + contracts: json!({"t":{"update_validators":["check"]}}), + build_id: Some("b1".into()), + fingerprints: json!({"t":"sha256:f"}), + }; + assert!(worker.serves("t", "check", None, None)); + assert!(worker.serves("t", "check", Some("b1"), Some("sha256:f"))); + assert!(!worker.serves("t", "other", None, None)); + assert!(!worker.serves("u", "check", None, None)); + assert!(!worker.serves("t", "check", Some("b2"), None)); + assert!(!worker.serves("t", "check", None, Some("sha256:g"))); + } +} diff --git a/rust/src/runtime/updates.rs b/rust/src/runtime/updates.rs index 2521ddf90..7a78a7c1b 100644 --- a/rust/src/runtime/updates.rs +++ b/rust/src/runtime/updates.rs @@ -30,6 +30,44 @@ struct UpdateRejection<'a, R> { validation_errors: Value, /// A structural ceiling keeps no arguments and records its limit. exceeded: Option, + /// A validated update's rejection keeps its validation task's id. + update_id: Option, +} + +/// A validator's approval of one update on one run. +struct Approval { + task_id: String, + run_id: String, + input_hash: String, +} + +/// The request's own `wait_timeout_seconds`, as PHP echoes it. +fn request_body_timeout(body: &Value) -> Value { + body.get("wait_timeout_seconds").cloned().unwrap_or(Value::Null) +} + +/// A validation answer as the control plane sends it: the requester and the +/// update contract, with the wait the request asked for. +fn validation_response(mut body: Value, wait_for: &str, context: Option<&Value>, name: &str) -> Value { + if body.get("wait_for").is_none() { + body["wait_for"] = json!(wait_for); + } + if let Some(principal) = context.and_then(principal) { + body["principal"] = principal; + } + super::control_plane::Operation::Update(name).response(body) +} + +/// `principalPayload`: the requester's non-empty type, id and label. +fn principal(context: &Value) -> Option { + let principal: serde_json::Map = ["type", "id", "label"] + .iter() + .filter_map(|field| { + context["principal"][*field].as_str().filter(|value| !value.trim().is_empty()) + .map(|value| ((*field).to_owned(), json!(value))) + }) + .collect(); + (!principal.is_empty()).then_some(Value::Object(principal)) } /// `DefaultWorkflowControlPlane::updateStatus` for a rejected update. @@ -294,6 +332,7 @@ where for<'r> i64: Decode<'r, DB>, usize: ColumnIndex, { + #[allow(clippy::too_many_arguments)] pub(crate) async fn update_workflow( &self, workflow_id: &str, @@ -305,6 +344,26 @@ where codec: Arc, limits: super::memo_limits::StructuralLimits, external: Option<&super::external_resolution::PreparedExternal>, + ) -> Result<(StatusCode, Value)> { + self.admit_update(workflow_id, selected_run, name, body, prepared, context, codec, limits, external, None) + .await + } + + /// One update admission. A validator's approval admits the update under + /// the validation task's id, once its run is still the current run. + #[allow(clippy::too_many_arguments)] + async fn admit_update( + &self, + workflow_id: &str, + selected_run: Option<&str>, + name: &str, + body: Value, + prepared: PreparedControlArguments, + context: Option, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + external: Option<&super::external_resolution::PreparedExternal>, + approval: Option, ) -> Result<(StatusCode, Value)> { // PHP's update_name column holds 255 bytes, one fewer than its // validation allows; native refuses what it cannot store. @@ -347,6 +406,13 @@ where "historical_run_command_rejected", )); } + if let Some(approval) = &approval + && approval.run_id != run_id + { + let (status, failure) = super::update_validation::stale_failure(workflow_id, &approval.run_id, name, &approval.task_id, + "The workflow advanced to a different run before validation could be committed."); + return Ok((status, validation_response(failure, wait_for, context.as_ref(), name))); + } let existing = if let Some(request_id) = request_id { Self::query("SELECT u.id,u.update_name,u.workflow_run_id FROM workflow_updates u JOIN workflow_commands c ON c.id=u.workflow_command_id WHERE c.workflow_instance_id=$1 AND c.command_type='update' AND c.request_id=$2") .bind(workflow_id).bind(request_id).fetch_optional(&mut *tx).await? @@ -386,23 +452,98 @@ where .bind(&run_id).fetch_one(&mut *tx).await .map_err(|error| at_update_lookup(error.into(), UpdateLookup::StartContract))?; let contract = DB::document_row(&started, "payload")?; + let declared = contract["declared_updates"] + .as_array() + .is_some_and(|names| names.iter().any(|value| value == name)); + if let Some(approval) = &approval + && (!active || !declared || !Self::approved_validation(&mut tx, &approval.task_id, &run_id, &approval.input_hash).await?) + { + let message = if active && declared { + "The validator approval no longer matches this update admission attempt." + } else { + "The validated update could not be committed to the current workflow run." + }; + let (status, failure) = super::update_validation::stale_failure(workflow_id, &run_id, name, &approval.task_id, message); + return Ok((status, validation_response(failure, wait_for, context.as_ref(), name))); + } + // `ValidatedExternalWorkflowUpdateAdmission::preAdmissionValidation`: + // a live run's declared update on a durable contract checks its + // validator declaration before anything is recorded. + if active && declared && approval.is_none() && super::update_validation::durable_contract(&contract) { + let Some(validators) = contract.get("declared_update_validators") else { + let failure = super::update_validation::admission_failure(workflow_id, &run_id, name, + "update_validator_contract_missing", + "The durable workflow contract does not state whether this update has a validator.", false, None); + return Ok((StatusCode::CONFLICT, validation_response(failure, wait_for, context.as_ref(), name))); + }; + if super::query_route::strings(validators).iter().any(|validator| validator == name) { + match validate_arguments(&contract, "declared_update_contracts", name, &prepared.values, + "unsupported_update_contract", "invalid_update_arguments") + { + Ok(()) => {} + Err(super::RuntimeError::Refused { reason: "invalid_update_arguments", .. }) => { + let failure = super::update_validation::admission_failure(workflow_id, &run_id, name, + "invalid_update_arguments", &format!("Workflow update [{name}] argument validation failed."), + false, Some(update_argument_errors(&contract, name, &prepared.values))); + return Ok((StatusCode::UNPROCESSABLE_ENTITY, validation_response(failure, wait_for, context.as_ref(), name))); + } + Err(error) => return Err(error), + } + let Some(request_id) = request_id else { + let failure = super::update_validation::admission_failure(workflow_id, &run_id, name, + "update_validation_request_id_required", + "Updates with declared validators require request_id so validation and application remain idempotent across retries.", + true, None); + return Ok((StatusCode::UNPROCESSABLE_ENTITY, validation_response(failure, wait_for, context.as_ref(), name))); + }; + // The caller waits for the validator without a transaction. + drop(tx); + let queue = DB::optional_string(&run, "queue")?.filter(|queue| !queue.trim().is_empty()); + let compatibility = DB::optional_string(&run, "compatibility")?.filter(|build| !build.is_empty()); + let fingerprint = super::query_route::trimmed(contract["workflow_definition_fingerprint"].as_str()); + let workflow_type = DB::string(&run, "workflow_type")?; + let request = super::update_validation::ValidationRequest { + workflow_id, + run_id: &run_id, + workflow_type: &workflow_type, + task_queue: queue.as_deref().unwrap_or("default"), + compatibility: compatibility.as_deref(), + fingerprint, + name, + request_id, + arguments: &prepared.arguments, + context: context.as_ref(), + }; + return match self.validate_update(&request).await? { + super::update_validation::Validation::Approved { task_id, input_hash } => { + let mut context = context.clone(); + if let Some(context) = context.as_mut().filter(|context| context["server"]["metadata"].is_object()) { + context["server"]["metadata"]["update_validation_task_id"] = json!(task_id); + context["server"]["metadata"]["update_validation_input_hash"] = json!(input_hash); + } + let approval = Approval { task_id, run_id: run_id.clone(), input_hash }; + Box::pin(self.admit_update(workflow_id, selected_run, name, body, prepared, context, codec, + limits, external, Some(approval))) + .await + } + super::update_validation::Validation::Rejected { failure, validation_errors } => { + let timeout = super::laravel_validation::integer(&request_body_timeout(&body)); + self.record_validator_rejection(workflow_id, &run_id, name, &prepared, request_id, + context.clone(), codec, failure, validation_errors, wait_for, timeout) + .await + } + super::update_validation::Validation::Failed(status, failure) => { + Ok((status, validation_response(failure, wait_for, context.as_ref(), name))) + } + }; + } + } // `WorkflowStub::submitUpdate`'s order: a closed run, an unknown // update, then invalid arguments. Each records a rejection. let mut rejection = if !active { Some(("run_not_active", Some("rejected_not_active"), json!([]))) - } else if !contract["declared_updates"] - .as_array() - .is_some_and(|names| names.iter().any(|value| value == name)) - { + } else if !declared { Some(("unknown_update", Some("rejected_unknown_update"), json!([]))) - } else if contract["declared_update_validators"] - .as_array() - .is_some_and(|names| names.iter().any(|value| value == name)) - { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "update_validation_not_implemented", - )); } else { match validate_arguments( &contract, @@ -458,6 +599,7 @@ where outcome, validation_errors, exceeded, + update_id: approval.as_ref().map(|approval| approval.task_id.clone()), }, ) .await?; @@ -489,7 +631,9 @@ where .bind(&run_id).fetch_one(&mut *tx).await .map_err(|error| at_update_lookup(error.into(), UpdateLookup::Inventory))?; let command_id = id(); - let update_id = id(); + // `WorkflowUpdateValidationObserver`: an approved update keeps + // its validation task's id. + let update_id = approval.as_ref().map_or_else(id, |approval| approval.task_id.clone()); let command_sequence = DB::number(&run, "last_command_sequence")? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; @@ -587,6 +731,82 @@ where } } + /// `recordValidatorRejection`: the validator's refusal, recorded as a + /// rejected command and update under the validation task's id. + #[allow(clippy::too_many_arguments)] + async fn record_validator_rejection( + &self, + workflow_id: &str, + run_id: &str, + name: &str, + prepared: &PreparedControlArguments, + request_id: &str, + context: Option, + codec: Arc, + mut failure: Value, + validation_errors: Value, + wait_for: &str, + timeout: Option, + ) -> Result<(StatusCode, Value)> { + let task_id = failure["update_id"].as_str().unwrap_or_default().to_owned(); + let mut tx = self.begin().await?; + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + let Some(instance) = instance.filter(|instance| DB::string(instance, "current_run_id").ok().as_deref() == Some(run_id)) else { + let (status, failure) = super::update_validation::stale_failure(workflow_id, run_id, name, &task_id, + "The workflow run changed before the validator rejection could be recorded."); + return Ok((status, validation_response(failure, wait_for, context.as_ref(), name))); + }; + let recorded = Self::query("SELECT workflow_command_id FROM workflow_updates WHERE id=$1") + .bind(&task_id).fetch_optional(&mut *tx).await?; + let command_id = match recorded { + Some(recorded) => DB::string(&recorded, "workflow_command_id")?, + None => { + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(run_id).fetch_one(&mut *tx).await?; + Self::reject_update( + &mut tx, + codec, + UpdateRejection { + workflow_id, + instance: &instance, + run: &run, + name, + prepared, + request_id: Some(request_id), + context: context.clone(), + reason: super::update_validation::VALIDATOR_REJECTED, + outcome: Some("rejected_invalid_arguments"), + validation_errors, + exceeded: None, + update_id: Some(task_id.clone()), + }, + ) + .await?; + let recorded = Self::query("SELECT workflow_command_id FROM workflow_updates WHERE id=$1") + .bind(&task_id).fetch_one(&mut *tx).await?; + DB::string(&recorded, "workflow_command_id")? + } + }; + tx.commit().await?; + // `rejectionResponse`: the validator's failure with the rejection + // receipt's fields. + for (field, value) in [ + ("accepted", json!(false)), + ("command_id", json!(command_id)), + ("command_status", json!("rejected")), + ("outcome", json!("rejected_invalid_arguments")), + ("update_id", json!(task_id)), + ("update_status", json!("rejected")), + ("rejection_reason", json!(super::update_validation::VALIDATOR_REJECTED)), + ("wait_for", json!(wait_for)), + ("wait_timed_out", json!(false)), + ("wait_timeout_seconds", json!(timeout)), + ] { + failure[field] = value; + } + Ok((StatusCode::UNPROCESSABLE_ENTITY, validation_response(failure, wait_for, context.as_ref(), name))) + } + /// `WorkflowStub::rejectUpdateCommand`: a rejected command and update, /// and an `UpdateRejected` event on the run. Nothing is delivered. async fn reject_update( @@ -606,6 +826,7 @@ where outcome, validation_errors, exceeded, + update_id, } = rejection; let run_id = DB::string(run, "id")?; let (values, diagnostics, arguments) = match &exceeded { @@ -626,7 +847,7 @@ where ) .await?; let command_id = id(); - let update_id = id(); + let update_id = update_id.unwrap_or_else(id); let command_sequence = DB::number(run, "last_command_sequence")? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; @@ -743,7 +964,7 @@ where .filter(|errors| !errors.is_null()) .unwrap_or_else(|| json!([])); let mut response = json!({ - "accepted":!rejected,"workflow_id":DB::string(update,"workflow_instance_id")?,"run_id":DB::string(update,"workflow_run_id")?, + "workflow_id":DB::string(update,"workflow_instance_id")?,"run_id":DB::string(update,"workflow_run_id")?, "resolved_run_id":DB::string(update,"resolved_workflow_run_id")?,"requested_run_id":null,"target_scope":"instance", "workflow_type":DB::string(update,"workflow_type")?,"command_id":DB::string(update,"workflow_command_id")?, "command_sequence":DB::number(update,"command_sequence")?,"command_status":if rejected {"rejected"} else {"accepted"},"command_source":"control_plane", @@ -757,17 +978,15 @@ where "wait_for":wait_for,"wait_timed_out":timed_out,"wait_timeout_seconds":timeout, "rejected_at":DB::optional_instant(update,"rejected_at")? }); + // `ControlPlaneResultMapper` drops `accepted` and puts it back only + // when it is false. + if rejected { + response["accepted"] = json!(false); + } // The requester recorded on the command, as `principalPayload`. let context = DB::optional_document_row(update, "command_context")?.unwrap_or(Value::Null); - let principal: serde_json::Map = ["type", "id", "label"] - .iter() - .filter_map(|field| { - context["principal"][*field].as_str().filter(|value| !value.trim().is_empty()) - .map(|value| ((*field).to_owned(), json!(value))) - }) - .collect(); - if !principal.is_empty() { - response["principal"] = Value::Object(principal); + if let Some(principal) = principal(&context) { + response["principal"] = principal; } let name = response["update_name"].as_str().unwrap_or_default().to_owned(); Ok(super::control_plane::Operation::Update(&name).response(response)) diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 2a83a3117..5685a92be 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -339,18 +339,24 @@ where "status":DB::string(task,"status")?,"attempt_count":DB::number(task,"attempt_count")?, "repair_count":DB::number(task,"repair_count")?,"queue":DB::string(task,"queue")?, "connection":DB::PHP_CONNECTION}); + // `WorkflowHistoryEvent::taskSnapshot` keeps every field that is set. for field in [ "available_at", "leased_at", "lease_expires_at", + "last_dispatch_attempt_at", + "last_dispatched_at", + "last_claim_failed_at", "repair_available_at", ] { if let Some(value) = DB::optional_instant(task, field)? { snapshot[field] = json!(value); } } - if let Some(owner) = DB::optional_string(task, "lease_owner")? { - snapshot["lease_owner"] = json!(owner); + for field in ["lease_owner", "compatibility", "last_dispatch_error", "last_claim_error"] { + if let Some(value) = DB::optional_string(task, field)? { + snapshot[field] = json!(value); + } } Ok(snapshot) } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index c2e4b4415..2a65c6e23 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod rejected_updates; mod continue_messages; #[path = "execution/update_wait.rs"] mod update_wait; +#[path = "execution/update_validation.rs"] +mod update_validation; #[path = "execution/payload_limits.rs"] mod payload_limits; #[path = "execution/start_definition.rs"] @@ -111,6 +113,8 @@ mod parent_close_policies; mod history_pages; #[path = "execution/declared_signals.rs"] mod declared_signals; +#[path = "execution/worker_history_presentation.rs"] +mod worker_history_presentation; #[path = "execution/poll_waits.rs"] mod poll_waits; #[path = "execution/child_cascade.rs"] diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index 82237259c..0038b91ed 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -70,7 +70,7 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { // An update resolves the reference to its bytes and keeps them inline. waiting_run(&app, "external-update", json!({"worker_id":"update-worker","task_queue":"external-update", "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":["workflow_updates"], - "workflow_command_contracts":{"echo":{"updates":["u"],"update_contracts":[{"name":"u","parameters":string_parameter()}]}}})).await; + "workflow_command_contracts":{"echo":{"updates":["u"],"update_validators":[],"update_contracts":[{"name":"u","parameters":string_parameter()}]}}})).await; let blob = argument("update"); let update = upload(&app, &blob).await; let (status, body) = json_request(&app, "POST", "/api/workflows/external-update/update/u", "operator-token", diff --git a/rust/tests/execution/rejected_updates.rs b/rust/tests/execution/rejected_updates.rs index 7c14dfbd9..90a17d058 100644 --- a/rust/tests/execution/rejected_updates.rs +++ b/rust/tests/execution/rejected_updates.rs @@ -80,7 +80,8 @@ async fn refused_updates_are_recorded_like_php() { // delivered to the waiting run. let (status, admitted) = update(&app, "rejected-update", "set", input(vec![Payload::Long(5)])).await; assert_eq!(status, StatusCode::ACCEPTED, "{admitted}"); - assert_eq!(admitted["accepted"], true, "{admitted}"); + // PHP's result mapper sends `accepted` only when it is false. + assert!(admitted.get("accepted").is_none(), "{admitted}"); assert!(admitted["command_sequence"].as_i64().unwrap() > unknown["command_sequence"].as_i64().unwrap()); let task = poll(&app, "update-a", "workflow").await; assert!(task["task_id"].is_string(), "{task}"); diff --git a/rust/tests/execution/update_validation.rs b/rust/tests/execution/update_validation.rs new file mode 100644 index 000000000..239222edd --- /dev/null +++ b/rust/tests/execution/update_validation.rs @@ -0,0 +1,310 @@ +use super::*; +use std::time::Duration; + +// Server 2.11.1's synchronous update validation, as observed against the +// published image: a declared validator decides an update before it is +// accepted, through validation tasks a capable worker claims. + +fn int_contract(name: &str) -> Value { + json!({"name":name,"parameters":[{"name":"value","type":"int","required":true,"variadic":false, + "default_available":false,"allows_null":false}]}) +} + +async fn validating_worker(app: &Router, worker: &str, queue: &str, validators: Option, capable: bool) { + let updates = ["approve_me", "plain", "reject_me"]; + let mut contract = json!({"signals":["finish"],"signal_contracts":[int_contract("finish")],"queries":[], + "updates":updates,"update_contracts":updates.iter().map(|name| int_contract(name)).collect::>()}); + if let Some(validators) = validators { + contract["update_validators"] = validators; + } + let mut capabilities = vec!["workflow_updates"]; + if capable { + capabilities.push("update_validation_tasks"); + } + let (status, body) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":queue, + "runtime":"python","supported_workflow_types":["echo"],"supported_activity_types":[], + "capabilities":capabilities,"workflow_command_contracts":{"echo":contract}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +/// A run on `queue` waiting for a signal, so updates find it idle. +async fn waiting_run(app: &Router, workflow: &str, queue: &str, worker: &str) -> Value { + let (status, started) = request(app, "POST", "/api/workflows", json!({"workflow_id":workflow,"workflow_type":"echo", + "task_queue":queue,"input":envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CREATED, "{started}"); + let (status, polled) = request(app, "POST", "/api/worker/workflow-tasks/poll", + json!({"worker_id":worker,"task_queue":queue,"timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{polled}"); + let task = &polled["task"]; + assert_eq!(task["task_kind"], "workflow", "{polled}"); + assert_eq!(finish_task(app, task, json!([{"type":"open_signal_wait","signal_name":"finish"}])).await.0, StatusCode::OK); + started +} + +fn update_body(value: i64, request_id: Option<&str>) -> Value { + let mut body = json!({"input":envelope(Payload::Array(vec![Payload::Long(value)]))}); + if let Some(request_id) = request_id { + body["request_id"] = json!(request_id); + } + body +} + +/// An update request that waits for its validator, answered later. +fn send_update(app: &Router, workflow: &str, name: &str, body: Value) -> tokio::task::JoinHandle<(StatusCode, Value)> { + let app = app.clone(); + let path = format!("/api/workflows/{workflow}/update/{name}"); + tokio::spawn(async move { request(&app, "POST", &path, body).await }) +} + +async fn poll_validation(app: &Router, worker: &str, queue: &str) -> Value { + let (status, body) = request(app, "POST", "/api/worker/update-validation-tasks/poll", + json!({"worker_id":worker,"task_queue":queue,"timeout_seconds":5})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["poll_status"], "leased", "{body}"); + body["task"].clone() +} + +async fn complete(app: &Router, task: &str, verdict: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/worker/update-validation-tasks/{task}/{verdict}"), body).await +} + +fn events<'a>(history: &'a Value, kind: &str) -> Vec<&'a Value> { + history.as_array().unwrap().iter().filter(|event| event["event_type"] == kind).collect() +} + +#[tokio::test] +async fn validators_approve_and_reject_updates_before_acceptance() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + validating_worker(&app, "validator", "vq", Some(json!(["approve_me", "reject_me"])), true).await; + let started = waiting_run(&app, "validated", "vq", "validator").await; + let history = || run_history(&app, &started["workflow_id"], &started["run_id"]); + let before = history().await; + let started_event = events(&before, "WorkflowStarted")[0]; + assert_eq!(started_event["payload"]["declared_update_validators"], json!(["approve_me", "reject_me"])); + + // Refusals before validation record nothing. + let (status, body) = request(&app, "POST", "/api/workflows/validated/update/approve_me", update_body(1, None)).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["reason"], "update_validation_request_id_required"); + assert_eq!(body["outcome"], "update_validation_failed"); + assert_eq!(body["retryable"], true); + assert_eq!(body["wait_for"], "accepted"); + assert!(body.get("update_id").is_none() && body.get("approved").is_none(), "{body}"); + let mut wrong = json!({"input":envelope(Payload::Array(vec![Payload::String("x".into())])),"request_id":"bad"}); + let (status, body) = request(&app, "POST", "/api/workflows/validated/update/approve_me", wrong.take()).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["reason"], "invalid_update_arguments"); + assert_eq!(body["message"], "Workflow update [approve_me] argument validation failed."); + assert_eq!(body["validation_errors"], json!({"value":["The value argument must be of type int."]})); + assert_eq!(history().await, before); + + // A rejection through the multiplexed poll. + let pending = send_update(&app, "validated", "reject_me", update_body(2, Some("r2"))); + let (status, polled) = request(&app, "POST", "/api/worker/workflow-tasks/poll", json!({"worker_id":"validator", + "task_queue":"vq","timeout_seconds":5,"task_kinds":["workflow","update_validation"]})).await; + assert_eq!(status, StatusCode::OK, "{polled}"); + let task = polled["task"].clone(); + assert_eq!(task["task_kind"], "update_validation", "{polled}"); + assert_eq!(task["update_name"], "reject_me"); + assert_eq!(task["update_validation_attempt"], 1); + assert_eq!(task["update_id"], task["update_validation_task_id"]); + assert_eq!(task["lease_owner"], "validator"); + assert!(task["next_history_page_token"].is_null() && task.get("next_history_page_token").is_some()); + assert_eq!(task["update_arguments"], update_body(2, None)["input"]); + assert!(task["history_events"].as_array().is_some_and(|events| !events.is_empty()), "{task}"); + assert!(task["history_export"].is_object(), "{task}"); + let id = task["update_validation_task_id"].as_str().unwrap().to_owned(); + let (status, body) = complete(&app, &id, "reject", json!({"lease_owner":"validator","update_validation_attempt":1})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"], json!({"failure":["The failure field is required."],"failure.message":["The failure.message field is required."]})); + let (status, body) = complete(&app, &id, "reject", json!({"lease_owner":"validator","update_validation_attempt":1, + "failure":{"message":"too small","validation_errors":{"v":["too small"]}}})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["outcome"], "rejected"); + assert_eq!(body["reason"], "update_validator_rejected"); + let (status, rejected) = pending.await.unwrap(); + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{rejected}"); + for (field, expected) in [ + ("approved", json!(false)), + ("accepted", json!(false)), + ("outcome", json!("rejected_invalid_arguments")), + ("reason", json!("update_validator_rejected")), + ("update_status", json!("rejected")), + ("command_status", json!("rejected")), + ("message", json!("too small")), + ("validation_errors", json!({"v":["too small"]})), + ("update_validation_attempt", json!(1)), + ("update_id", json!(id)), + ("wait_timed_out", json!(false)), + ] { + assert_eq!(rejected[field], expected, "{field}: {rejected}"); + } + let (status, again) = complete(&app, &id, "reject", json!({"lease_owner":"validator","update_validation_attempt":1, + "failure":{"message":"again"}})).await; + assert_eq!(status, StatusCode::CONFLICT, "{again}"); + assert_eq!(again["reason"], "duplicate_update_validation_completion"); + + // An approval through the dedicated poll. + let pending = send_update(&app, "validated", "approve_me", update_body(3, Some("r1"))); + let task = poll_validation(&app, "validator", "vq").await; + let id = task["update_validation_task_id"].as_str().unwrap().to_owned(); + assert!(task.get("next_history_page_token").is_none(), "{task}"); + let (status, body) = complete(&app, &id, "approve", json!({"lease_owner":"someone","update_validation_attempt":1})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("update_validation_lease_owner_mismatch"))); + let (status, body) = complete(&app, &id, "approve", json!({"lease_owner":"validator","update_validation_attempt":2})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("stale_update_validation_completion"))); + let (status, body) = complete(&app, &id, "approve", json!({"lease_owner":"validator","update_validation_attempt":1})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, json!({"update_validation_task_id":id,"update_id":id,"update_validation_attempt":1,"outcome":"approved", + "status":200,"protocol_version":body["protocol_version"],"server_capabilities":body["server_capabilities"]})); + let (status, approved) = pending.await.unwrap(); + assert_eq!(status, StatusCode::ACCEPTED, "{approved}"); + assert_eq!(approved["update_id"], json!(id)); + assert_eq!(approved["update_status"], "accepted"); + let (status, unknown) = complete(&app, "01ZZZZZZZZZZZZZZZZZZZZZZZZ", "approve", + json!({"lease_owner":"validator","update_validation_attempt":1})).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{unknown}"); + assert!(unknown.get("update_validation_attempt").is_none(), "{unknown}"); + + // Retries return the recorded receipts; an undeclared validator is not asked. + let (status, retained) = request(&app, "POST", "/api/workflows/validated/update/approve_me", update_body(99, Some("r1"))).await; + assert_eq!((status, retained["update_id"].clone()), (StatusCode::ACCEPTED, json!(id))); + let (status, retained) = request(&app, "POST", "/api/workflows/validated/update/reject_me", update_body(2, Some("r2"))).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{retained}"); + assert_eq!(retained["reason"], "update_validator_rejected"); + assert_eq!(retained["validation_errors"], json!({"v":["too small"]})); + let (status, plain) = request(&app, "POST", "/api/workflows/validated/update/plain", update_body(4, Some("p1"))).await; + assert_eq!(status, StatusCode::ACCEPTED, "{plain}"); + + let after = history().await; + let rejected_events = events(&after, "UpdateRejected"); + assert_eq!(rejected_events.len(), 1, "{after}"); + assert_eq!(rejected_events[0]["payload"]["update_id"], rejected["update_id"]); + assert_eq!(rejected_events[0]["payload"]["validation_errors"], json!({"v":["too small"]})); + let accepted = events(&after, "UpdateAccepted"); + assert_eq!(accepted.len(), 2, "{after}"); + assert_eq!(accepted[0]["payload"]["update_id"], json!(id)); + assert_eq!(accepted[0]["payload"]["request_id"], "r1"); +} + +#[tokio::test] +async fn validation_routes_time_out_and_fence_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + // A worker contract without validators makes every remote update refuse. + validating_worker(&app, "silent", "q-missing", None, true).await; + waiting_run(&app, "missing", "q-missing", "silent").await; + let (status, body) = request(&app, "POST", "/api/workflows/missing/update/plain", update_body(1, None)).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "update_validator_contract_missing"); + assert_eq!(body["retryable"], false); + + // No capable worker, or none matching the run. + validating_worker(&app, "incapable", "q-cap", Some(json!(["approve_me"])), false).await; + waiting_run(&app, "incapable-run", "q-cap", "incapable").await; + let (status, body) = request(&app, "POST", "/api/workflows/incapable-run/update/approve_me", update_body(1, Some("c1"))).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!((body["reason"].clone(), body["retryable"].clone()), (json!("update_validation_capability_unsupported"), json!(false))); + assert_eq!(body["approved"], false); + assert!(body["update_id"].is_string(), "{body}"); + let (status, body) = request(&app, "POST", "/api/worker/update-validation-tasks/poll", + json!({"worker_id":"incapable","task_queue":"q-cap","timeout_seconds":0})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("update_validation_capability_not_advertised"))); + let (status, body) = request(&app, "POST", "/api/worker/workflow-tasks/poll", + json!({"worker_id":"incapable","task_queue":"q-cap","timeout_seconds":0,"task_kinds":["update_validation"]})).await; + assert_eq!((status, body["poll_status"].clone()), (StatusCode::CONFLICT, json!("unsupported"))); + validating_worker(&app, "changed", "q-changed", Some(json!(["approve_me"])), true).await; + waiting_run(&app, "changed-run", "q-changed", "changed").await; + validating_worker(&app, "changed", "q-changed", Some(json!([])), true).await; + let (status, body) = request(&app, "POST", "/api/workflows/changed-run/update/approve_me", update_body(1, Some("i1"))).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!((body["reason"].clone(), body["retryable"].clone()), (json!("update_validator_worker_incompatible"), json!(true))); + let (status, body) = request(&app, "POST", "/api/worker/workflow-tasks/poll", + json!({"worker_id":"changed","task_queue":"q-changed","timeout_seconds":0,"task_kinds":["workflow","activity"]})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"], json!({"task_kinds.1":["The selected task_kinds.1 is invalid."]})); + + // Nobody claims: the request waits out the timeout, and retries replay it. + validating_worker(&app, "idle", "q-idle", Some(json!(["approve_me"])), true).await; + waiting_run(&app, "idle-run", "q-idle", "idle").await; + let unclaimed = send_update(&app, "idle-run", "approve_me", update_body(5, Some("r3"))); + + // A claimed task whose worker leaves. + validating_worker(&app, "leaver", "q-lost", Some(json!(["approve_me"])), true).await; + waiting_run(&app, "lost-run", "q-lost", "leaver").await; + let lost = send_update(&app, "lost-run", "approve_me", update_body(6, Some("l1"))); + let task = poll_validation(&app, "leaver", "q-lost").await; + let lost_id = task["update_validation_task_id"].as_str().unwrap().to_owned(); + assert_eq!(request(&app, "DELETE", "/api/workers/leaver", Value::Null).await.0, StatusCode::OK); + let (status, body) = complete(&app, &lost_id, "approve", json!({"lease_owner":"leaver","update_validation_attempt":1})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("update_validator_worker_lost"))); + + let (status, body) = unclaimed.await.unwrap(); + assert_eq!(status, StatusCode::GATEWAY_TIMEOUT, "{body}"); + assert_eq!(body["reason"], "update_validation_task_not_claimed"); + assert_eq!(body["update_validation_attempt"], 0); + let (status, again) = request(&app, "POST", "/api/workflows/idle-run/update/approve_me", update_body(5, Some("r3"))).await; + assert_eq!((status, again["update_id"].clone()), (StatusCode::GATEWAY_TIMEOUT, body["update_id"].clone())); + let (status, conflict) = request(&app, "POST", "/api/workflows/idle-run/update/approve_me", update_body(6, Some("r3"))).await; + assert_eq!((status, conflict["reason"].clone()), (StatusCode::CONFLICT, json!("update_validation_idempotency_conflict"))); + let (status, body) = lost.await.unwrap(); + assert_eq!(status, StatusCode::GATEWAY_TIMEOUT, "{body}"); + assert_eq!(body["reason"], "update_validator_worker_lost"); + + // With no worker left on the queue. + let (status, body) = request(&app, "POST", "/api/workflows/lost-run/update/approve_me", update_body(7, Some("u1"))).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("update_validator_worker_unavailable"))); +} + +#[tokio::test] +async fn expired_validation_leases_are_claimed_again() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + validating_worker(&app, "slow", "q-slow", Some(json!(["approve_me", "reject_me"])), true).await; + waiting_run(&app, "slow-run", "q-slow", "slow").await; + let pending = send_update(&app, "slow-run", "approve_me", update_body(8, Some("r4"))); + let first = poll_validation(&app, "slow", "q-slow").await; + let id = first["update_validation_task_id"].as_str().unwrap().to_owned(); + tokio::time::sleep(Duration::from_secs(6)).await; + let (status, body) = complete(&app, &id, "approve", json!({"lease_owner":"slow","update_validation_attempt":1})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("update_validation_lease_expired"))); + let second = poll_validation(&app, "slow", "q-slow").await; + assert_eq!((second["update_validation_task_id"].clone(), second["update_validation_attempt"].clone()), (json!(id), json!(2))); + let (status, body) = complete(&app, &id, "approve", json!({"lease_owner":"slow","update_validation_attempt":1})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("stale_update_validation_completion"))); + assert_eq!(complete(&app, &id, "approve", json!({"lease_owner":"slow","update_validation_attempt":2})).await.0, StatusCode::OK); + let (status, body) = pending.await.unwrap(); + assert_eq!((status, body["update_id"].clone()), (StatusCode::ACCEPTED, json!(id))); + + // A failure other than a validator rejection records no update. + let pending = send_update(&app, "slow-run", "reject_me", update_body(9, Some("r5"))); + let task = poll_validation(&app, "slow", "q-slow").await; + let id = task["update_validation_task_id"].as_str().unwrap().to_owned(); + let (status, body) = complete(&app, &id, "reject", json!({"lease_owner":"slow","update_validation_attempt":1, + "failure":{"message":"validator crashed","reason":"validator_error","type":"RuntimeError","stack_trace":"trace"}})).await; + assert_eq!((status, body["outcome"].clone()), (StatusCode::OK, json!("failed"))); + let (status, failed) = pending.await.unwrap(); + assert_eq!(status, StatusCode::CONFLICT, "{failed}"); + assert_eq!((failed["reason"].clone(), failed["failure_type"].clone()), (json!("validator_error"), json!("RuntimeError"))); + assert!(failed.get("update_status").is_none(), "{failed}"); + + // The multiplexed poll alternates kinds after each lease. + assert_eq!(request(&app, "POST", "/api/workflows/slow-run/update/plain", update_body(10, Some("p2"))).await.0, StatusCode::ACCEPTED); + let pending = send_update(&app, "slow-run", "approve_me", update_body(11, Some("r6"))); + tokio::time::sleep(Duration::from_millis(300)).await; + let mut kinds = vec![]; + for _ in 0..2 { + let (status, body) = request(&app, "POST", "/api/worker/workflow-tasks/poll", json!({"worker_id":"slow", + "task_queue":"q-slow","timeout_seconds":1,"task_kinds":["update_validation","workflow"]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = &body["task"]; + kinds.push(task["task_kind"].clone()); + if task["task_kind"] == "update_validation" { + let id = task["update_validation_task_id"].as_str().unwrap(); + assert_eq!(complete(&app, id, "approve", json!({"lease_owner":"slow","update_validation_attempt":1})).await.0, StatusCode::OK); + } + } + assert_eq!(kinds, vec![json!("update_validation"), json!("workflow")]); + assert_eq!(pending.await.unwrap().0, StatusCode::ACCEPTED); +} diff --git a/rust/tests/execution/worker_history_presentation.rs b/rust/tests/execution/worker_history_presentation.rs new file mode 100644 index 000000000..a5cafa602 --- /dev/null +++ b/rust/tests/execution/worker_history_presentation.rs @@ -0,0 +1,97 @@ +use super::*; + +// Server 2.11.1 `WorkflowTaskPoller::historyEventsWithSignalArgumentEnvelopes` +// and `WorkflowHistoryEvent::taskSnapshot`: the history a worker reads shows a +// consumed signal's workflow sequence, and task snapshots keep their dispatch +// stamps. + +fn wait(key: &str) -> Value { + json!([{"type":"open_condition_wait","condition_key":key, + "condition_definition_fingerprint":format!("sha256:{}", "b".repeat(64))}]) +} + +fn received(history: &Value) -> Vec { + history.as_array().unwrap().iter().filter(|event| event["event_type"] == "SignalReceived") + .map(|event| event["payload"].clone()).collect() +} + +#[tokio::test] +async fn a_worker_reads_the_sequence_that_consumed_a_signal() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + // A run started before any worker declared a contract admits any signal. + let started = start(&app, "presented-signals").await; + let worker = json!({"worker_id":"presenter","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[]}); + assert_eq!(request(&app, "POST", "/api/worker/register", worker).await.0, StatusCode::CREATED); + let task = poll(&app, "presenter", "workflow").await; + assert_eq!(finish_task(&app, &task, wait("first")).await.0, StatusCode::OK); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let opened = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ConditionWaitOpened").unwrap(); + let sequence = opened["payload"]["sequence"].clone(); + + let (status, body) = request(&app, "POST", "/api/workflows/presented-signals/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "presenter", "workflow").await; + assert!(received(&task["history_events"])[0].get("workflow_sequence").is_none(), "not consumed yet: {task}"); + assert_eq!(finish_task(&app, &task, wait("second")).await.0, StatusCode::OK); + + let (status, body) = request(&app, "POST", "/api/workflows/presented-signals/signal/again", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "presenter", "workflow").await; + let signals = received(&task["history_events"]); + assert_eq!(signals[0]["workflow_sequence"], sequence, "the consuming wait's sequence: {signals:?}"); + assert!(signals[1].get("workflow_sequence").is_none(), "{signals:?}"); + // A history page reads the same. + let (status, page) = request(&app, "POST", &format!("/api/worker/workflow-tasks/{}/history", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"]})).await; + assert_eq!(status, StatusCode::OK, "{page}"); + assert_eq!(received(&page["history_events"])[0]["workflow_sequence"], sequence, "{page}"); + // The run's own history shows the event as recorded. + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(received(&history)[0].get("workflow_sequence").is_none(), "{history}"); + + // The completing task was dispatched when the signal created it, and its + // snapshot keeps the dispatch stamps. + let done = json!([{"type":"complete_workflow","result":envelope(Payload::Null)}]); + assert_eq!(finish_task(&app, &task, done).await.0, StatusCode::OK); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let completed = history.as_array().unwrap().iter().find(|event| event["event_type"] == "WorkflowCompleted").unwrap(); + for stamp in ["last_dispatch_attempt_at", "last_dispatched_at"] { + assert!(completed["payload"]["task"][stamp].is_string(), "{stamp}: {completed}"); + } + assert!(completed["payload"]["task"].get("last_claim_error").is_none(), "unset fields stay out: {completed}"); +} + +// Workflow 2.10.2 `DefaultWorkflowTaskBridge::applySignal` takes a workflow +// sequence only from the signal's own wait. A signal the run's contract +// declares is applied without one even while a condition wait is open, so +// `markConditionWaitSignalConsumed` finds it applied and its record keeps no +// sequence: the history a worker reads matches the run's history. +#[tokio::test] +async fn a_declared_signal_consumed_by_a_condition_wait_keeps_no_sequence() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + update_worker(&app, "declared-presenter").await; + let started = start(&app, "declared-signals").await; + let task = poll(&app, "declared-presenter", "workflow").await; + assert_eq!(finish_task(&app, &task, wait("first")).await.0, StatusCode::OK); + + let finish = json!({"input":envelope(Payload::Array(vec![Payload::Long(7)]))}); + let (status, body) = request(&app, "POST", "/api/workflows/declared-signals/signal/finish", finish.clone()).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "declared-presenter", "workflow").await; + assert_eq!(finish_task(&app, &task, wait("second")).await.0, StatusCode::OK); + let (status, body) = request(&app, "POST", "/api/workflows/declared-signals/signal/finish", finish).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "declared-presenter", "workflow").await; + + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let polled = received(&task["history_events"]); + assert_eq!(polled.len(), 2, "{task}"); + assert!(polled.iter().all(|signal| signal.get("workflow_sequence").is_none()), "{polled:?}"); + let recorded = received(&history); + assert_eq!(polled[0]["signal_id"], recorded[0]["signal_id"]); + let (_, described) = request(&app, "GET", "/api/workflows/declared-signals", Value::Null).await; + assert_eq!(described["status"], "waiting", "{described}"); +} diff --git a/rust/vendor/sqlx-mysql/DURABLE-PATCH.md b/rust/vendor/sqlx-mysql/DURABLE-PATCH.md index f08664b46..f4792828c 100644 --- a/rust/vendor/sqlx-mysql/DURABLE-PATCH.md +++ b/rust/vendor/sqlx-mysql/DURABLE-PATCH.md @@ -1,7 +1,7 @@ -# SQLx MySQL hard-close patch +# SQLx MySQL driver patches -This directory contains the official `sqlx-mysql` 0.9.0 crate with one scoped -driver correction. Its published archive has SHA-256 +This directory contains the official `sqlx-mysql` 0.9.0 crate with two scoped +driver corrections. Its published archive has SHA-256 `90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27` and VCS commit `003b698e99e024f3621b8043a2426fde5b741171`. @@ -19,10 +19,49 @@ The original and flush-before-gate TLS controls fail with the published driver; both pass with this correction. Qualification and original evidence are in [Server #429](https://github.com/durable-workflow/server/pull/429#issuecomment-6135933370). -All upstream source is retained. The only source-code change is in -`src/connection/mod.rs`. The packaged license-reference files are replaced with +## Statement closes leave with the next command + +`COM_STMT_CLOSE` has no server reply. The published driver flushes it on its +own write: when its statement cache evicts a statement, and right after +executing a statement it does not cache. The next command is then a second +small write while the close is still unacknowledged, so Nagle's algorithm +holds it until the server's delayed acknowledgement, about 40 ms. SQLx does not +set `TCP_NODELAY`; PHP's mysqlnd and libmysqlclient do. + +Inside the native transition lock, a transaction issues 30 to 48 statements, +so these stalls added up to holds of 0.3 to 1 s under load. A due timer +waiting behind them fired more than the 1 s maximum late (Server #714). + +`MySqlStream::queue_packet` now buffers each close with the reset sequence +number that `send_packet` would use. On the path before the next command +(`wait_until_ready`), a buffer that holds only such closes is not flushed, so +they leave in the same write as that command. The server receives the same +packets in the same order. A connection dropped before its next command leaves +the close unsent; the server releases that connection's prepared statements +when it closes. + +The public lifecycle is unchanged. `Connection::flush` sends every buffered +byte, queued closes included, and leaves `should_flush` false. +`clear_cached_statements` sends its closes before it returns. Neither needs a +later command. + +Three tests in `rust/src/runtime/mysql_pool_tests.rs` record the TCP read +chunks a proxy receives from the client: +- `qualification_statement_close_leaves_with_the_next_command` fails with the + published driver, where each close arrives as a chunk of its own, and passes + with this correction; +- `qualification_explicit_flush_sends_queued_statement_closes` and + `qualification_clearing_the_statement_cache_closes_them_now` prove the two + lifecycle calls without a later command. Where the performance schema is on, + the second also sees the server's statements for the connection drop from + three to none. + +All upstream source is retained. The source-code changes are in +`src/connection/mod.rs`, `src/connection/stream.rs` and +`src/connection/executor.rs`. The packaged license-reference files are replaced with the full MIT and Apache license texts from the same upstream commit. This file records the patch; no upstream release is represented as containing it. Remove this override when a reviewed published driver provides equivalent -hard-close behavior and passes the same database and TLS qualification. +hard-close behavior and statement-close batching (or `TCP_NODELAY`), and passes +the same database and TLS qualification. diff --git a/rust/vendor/sqlx-mysql/src/connection/executor.rs b/rust/vendor/sqlx-mysql/src/connection/executor.rs index ee59d03d0..6c12d29fc 100755 --- a/rust/vendor/sqlx-mysql/src/connection/executor.rs +++ b/rust/vendor/sqlx-mysql/src/connection/executor.rs @@ -92,8 +92,7 @@ impl MySqlConnection { { self.inner .stream - .send_packet(StmtClose { statement: id }) - .await?; + .queue_packet(StmtClose { statement: id })?; } Ok((id, metadata)) @@ -168,7 +167,7 @@ impl MySqlConnection { }) .await?; - self.inner.stream.send_packet(StmtClose { statement: id }).await?; + self.inner.stream.queue_packet(StmtClose { statement: id })?; MySqlValueFormat::Binary } @@ -337,8 +336,7 @@ impl<'c> Executor<'c> for &'c mut MySqlConnection { self.inner .stream - .send_packet(StmtClose { statement: id }) - .await?; + .queue_packet(StmtClose { statement: id })?; metadata }; @@ -367,8 +365,7 @@ impl<'c> Executor<'c> for &'c mut MySqlConnection { self.inner .stream - .send_packet(StmtClose { statement: id }) - .await?; + .queue_packet(StmtClose { statement: id })?; let columns = (*metadata.columns).clone(); diff --git a/rust/vendor/sqlx-mysql/src/connection/mod.rs b/rust/vendor/sqlx-mysql/src/connection/mod.rs index 110ae4f1e..2513cf3cd 100755 --- a/rust/vendor/sqlx-mysql/src/connection/mod.rs +++ b/rust/vendor/sqlx-mysql/src/connection/mod.rs @@ -94,7 +94,11 @@ impl Connection for MySqlConnection { #[doc(hidden)] fn flush(&mut self) -> impl Future> + Send + '_ { - self.inner.stream.wait_until_ready() + async move { + // `Connection::flush` sends everything, queued closes included. + self.inner.stream.flush_queued().await?; + self.inner.stream.wait_until_ready().await + } } fn cached_statements_size(&self) -> usize { @@ -105,13 +109,13 @@ impl Connection for MySqlConnection { while let Some((statement_id, _)) = self.inner.cache_statement.remove_lru() { self.inner .stream - .send_packet(StmtClose { + .queue_packet(StmtClose { statement: statement_id, - }) - .await?; + })?; } - Ok(()) + // The server closes them now, not with some later command. + self.inner.stream.flush_queued().await } #[doc(hidden)] diff --git a/rust/vendor/sqlx-mysql/src/connection/stream.rs b/rust/vendor/sqlx-mysql/src/connection/stream.rs index e6aa8b48c..c16a21eb0 100755 --- a/rust/vendor/sqlx-mysql/src/connection/stream.rs +++ b/rust/vendor/sqlx-mysql/src/connection/stream.rs @@ -19,6 +19,9 @@ pub struct MySqlStream> { pub(crate) sequence_id: u8, pub(crate) waiting: VecDeque, pub(crate) is_tls: bool, + // The write buffer holds only commands without a reply, queued to leave + // with the next command. + pub(crate) queued_unanswered: bool, } #[derive(Debug, PartialEq, Eq)] @@ -56,13 +59,27 @@ impl MySqlStream { sequence_id: 0, socket: BufferedSocket::new(socket), is_tls: false, + queued_unanswered: false, } } - pub(crate) async fn wait_until_ready(&mut self) -> Result<(), Error> { + /// Send every buffered byte now, including queued commands without a + /// reply. Explicit flushes and statement cache clears use this; only the + /// path before the next command leaves queued closes to share its write. + pub(crate) async fn flush_queued(&mut self) -> Result<(), Error> { if !self.socket.write_buffer().is_empty() { self.socket.flush().await?; } + self.queued_unanswered = false; + Ok(()) + } + + pub(crate) async fn wait_until_ready(&mut self) -> Result<(), Error> { + // Queued commands without a reply wait for the next command; nothing + // pending needs them to be sent first. + if !self.socket.write_buffer().is_empty() && !self.queued_unanswered { + self.socket.flush().await?; + } while !self.waiting.is_empty() { while self.waiting.front() == Some(&Waiting::Row) { @@ -108,10 +125,26 @@ impl MySqlStream { Ok(()) } + /// Buffer a command that has no server reply (`COM_STMT_CLOSE`) so it + /// leaves in the same write as the next command. Flushing it alone leaves + /// unacknowledged data in flight, and Nagle's algorithm then holds the next + /// command until the server's delayed acknowledgement (about 40 ms). + pub(crate) fn queue_packet<'en, T>(&mut self, payload: T) -> Result<(), Error> + where + T: ProtocolEncode<'en, Capabilities>, + { + let unanswered = self.socket.write_buffer().is_empty() || self.queued_unanswered; + self.sequence_id = 0; + self.write_packet(payload)?; + self.queued_unanswered = unanswered; + Ok(()) + } + pub(crate) fn write_packet<'en, T>(&mut self, payload: T) -> Result<(), Error> where T: ProtocolEncode<'en, Capabilities>, { + self.queued_unanswered = false; self.socket .write_with(Packet(payload), (self.capabilities, &mut self.sequence_id)) } @@ -213,6 +246,7 @@ impl MySqlStream { sequence_id: self.sequence_id, waiting: self.waiting, is_tls: self.is_tls, + queued_unanswered: self.queued_unanswered, } } } diff --git a/rust/vendor/sqlx-mysql/src/connection/tls.rs b/rust/vendor/sqlx-mysql/src/connection/tls.rs index 9034fbd63..9ac71f676 100755 --- a/rust/vendor/sqlx-mysql/src/connection/tls.rs +++ b/rust/vendor/sqlx-mysql/src/connection/tls.rs @@ -97,6 +97,7 @@ impl WithSocket for MapStream { sequence_id: self.sequence_id, waiting: self.waiting, is_tls: true, + queued_unanswered: false, } } } diff --git a/scripts/conformance/server-parity.mjs b/scripts/conformance/server-parity.mjs index f0da20bc4..b8a599a9d 100644 --- a/scripts/conformance/server-parity.mjs +++ b/scripts/conformance/server-parity.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -import {mkdtempSync, readFileSync, readdirSync, rmSync} from 'node:fs'; +import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {createHash} from 'node:crypto'; import {fileURLToPath} from 'node:url'; import {basename, dirname, join, resolve} from 'node:path'; @@ -14,6 +14,7 @@ import {checkSdkChildDirectionArtifacts, checkSdkChildDirectionPackages} from '. import {checkSdkTimerAuthorArtifacts, checkSdkTimerAuthorPackages} from './server-parity/sdk-timer-author-contract.mjs'; import {checkSdkSignalDirectionArtifacts, checkSdkSignalDirectionPackages} from './server-parity/sdk-signal-direction-contract.mjs'; import {captureProbe} from './server-parity/probe-capture.mjs'; +import {selectFixturePaths} from './server-parity/fixture-selection.mjs'; import {writeRecord} from './server-parity/record-file.mjs'; import {checkCorrectedMarkerArtifacts, checkCorrectedMarkerPackages} from './server-parity/corrected-marker-contract.mjs'; import {checkCorrectedRangeArtifacts, checkCorrectedRangePackages} from './server-parity/corrected-range-contract.mjs'; @@ -28,6 +29,7 @@ const {values, positionals} = parseArgs({allowPositionals: true, options: { fixture: {type: 'string', multiple: true}, 'database-resource-limits': {type: 'string'}, 'include-fixture': {type: 'string', multiple: true}, + 'exclude-fixture': {type: 'string', multiple: true}, }}); const [command, ...files] = positionals; if (values.help) { @@ -50,16 +52,16 @@ selected records with the same --fixture paths so current source bytes remain authoritative; selected candidates do not enlarge the default passing corpus. --include-fixture PATH adds an explicit candidate to the reviewed default set; use that same option when comparing the combined recordings. It cannot be -combined with --fixture and does not change default fixture discovery.`); +combined with --fixture and does not change default fixture discovery. +--exclude-fixture PATH removes one fixture of that set because a separate, +explicitly selected record covers it. It must name a selected fixture, the +comparison must use the same option, and it cannot be combined with --fixture.`); process.exit(0); } const json = path => JSON.parse(readFileSync(path, 'utf8')); try { const directory = resolve(root, 'tests/Fixtures/ServerParity'); - if (values.fixture?.length && values['include-fixture']?.length) throw new Error('Choose either --fixture or --include-fixture'); - const paths = values.fixture?.length ? values.fixture.map(path => resolve(root, path)) - : [...readdirSync(directory).filter(name => name !== 'php-baseline.json' && name.endsWith('.json')).map(name => resolve(directory, name)), - ...(values['include-fixture'] ?? []).map(path => resolve(root, path))]; + const paths = selectFixturePaths({root, directory, fixture: values.fixture, include: values['include-fixture'], exclude: values['exclude-fixture']}); if (paths.some(path => !path.endsWith('.json')) || new Set(paths.map(path => basename(path))).size !== paths.length) throw new Error('Selected fixtures require distinct JSON filenames'); const byName = Object.fromEntries(paths.map(path => [basename(path), path])); const fixtures = Object.keys(byName).sort(); @@ -101,7 +103,7 @@ try { for (const flag of ['url', 'application-root']) { if (values[flag]) arguments_.push(`--${flag}`, values[flag]); } - const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : 45000, + const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : 45000, env: {...process.env, DW_PARITY_FAILURE_EVIDENCE_DIR: dirname(resolve(values.output))}, preserveStdout: join(sourceDirectory, name)}); const item = {fixture_id: fixture.id, fixture, outcome: 'runner-blocked'}; diff --git a/scripts/conformance/server-parity/activity-lease-renewal-probe.php b/scripts/conformance/server-parity/activity-lease-renewal-probe.php index 08a9bf8b0..57350cf9e 100644 --- a/scripts/conformance/server-parity/activity-lease-renewal-probe.php +++ b/scripts/conformance/server-parity/activity-lease-renewal-probe.php @@ -4,6 +4,7 @@ use DurableWorkflow\Client; use GuzzleHttp\Client as LeaseRenewalHttp; +use GuzzleHttp\HandlerStack; // Runtime activity lease renewal (POST /api/worker/activity-tasks/{task}/renew-lease) // over raw worker HTTP in a fresh namespace. Three runs each claim one @@ -17,8 +18,12 @@ function finishHttpActivityLeaseRenewal(Client $admin, array $fixture, string $w $namespace = strtolower($workflowId.'-lease'); $admin->createNamespace($namespace, 'activity lease renewal λ', 30); $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $http = new LeaseRenewalHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, - 'connect_timeout' => 1, 'timeout' => 30]); + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); $raw = static function (string $method, string $path, ?array $body = [], string $protocol = '1.20') use ($http, $token, $namespace): array { $options = ['headers' => array_filter([ 'Authorization' => $token === null ? null : 'Bearer '.$token, diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 69b947698..3d1356df6 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -21,6 +21,7 @@ import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; import {checkClusterInfo} from './cluster-info-contract.mjs'; import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; +import {checkUpdateValidation} from './update-validation-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -159,6 +160,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const clusterInfo = fixture.cluster_info ? checkClusterInfo(fixture, observation, workflowId) : null; const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; + const updateValidation = fixture.update_validation ? checkUpdateValidation(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -714,6 +716,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(clusterInfo ? {cluster_info: clusterInfo} : {}), ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), + ...(updateValidation ? {update_validation: updateValidation} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/fixture-selection.mjs b/scripts/conformance/server-parity/fixture-selection.mjs new file mode 100644 index 000000000..d70ebc7e3 --- /dev/null +++ b/scripts/conformance/server-parity/fixture-selection.mjs @@ -0,0 +1,19 @@ +import {readdirSync} from 'node:fs'; +import {resolve} from 'node:path'; + +// The fixture files one record or comparison covers. +// fixture replaces the reviewed corpus with an explicit set. include adds +// candidates to the corpus. exclude removes members of the corpus that a +// separate, explicitly selected record covers (Server #723). +export function selectFixturePaths({root, directory, fixture = [], include = [], exclude = []}) { + if (fixture.length && include.length) throw new Error('Choose either --fixture or --include-fixture'); + if (fixture.length && exclude.length) throw new Error('Choose either --fixture or --exclude-fixture'); + const selected = fixture.length ? fixture.map(path => resolve(root, path)) + : [...readdirSync(directory).filter(name => name !== 'php-baseline.json' && name.endsWith('.json')).map(name => resolve(directory, name)), + ...include.map(path => resolve(root, path))]; + const excluded = exclude.map(path => resolve(root, path)); + for (const path of excluded) { + if (!selected.includes(path)) throw new Error(`Excluded fixture is not in the selected set: ${path}`); + } + return selected.filter(path => !excluded.includes(path)); +} diff --git a/scripts/conformance/server-parity/php-maintenance-window.sh b/scripts/conformance/server-parity/php-maintenance-window.sh new file mode 100644 index 000000000..ec55e95a7 --- /dev/null +++ b/scripts/conformance/server-parity/php-maintenance-window.sh @@ -0,0 +1,65 @@ +# Sourced by SQLite-job scripts that record with PHP's maintenance loop stopped +# (Server #721, #723). +# +# run_without_maintenance SCHEDULE -- COMMAND... +# +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown + return 0 +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + [[ "$(maintenance_state)" == running ]] || return 1 + sleep "$maintenance_poll_seconds" + done + [[ "$(maintenance_state)" == running ]] +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then + return 0 + fi + done + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 + return 1 +} + +run_without_maintenance() { + local schedule="$1" + shift + local separator="$1" + [[ "$separator" == -- ]] && shift + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + [[ "$maintenance_container" == "$RESOURCE_SCOPE-php-maintenance" ]] + echo "before: $(maintenance_state)" > "$schedule" + grep -qx 'before: running' "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 4df73a65f..78f46978b 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -439,6 +439,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $clusterInfo = isset($fixture['cluster_info']) ? finishHttpClusterInfo($client, $fixture, $workflowId, $url) : null; $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; + $updateValidation = isset($fixture['update_validation']) ? finishHttpUpdateValidation($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -547,6 +548,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'cluster_info' => $clusterInfo, 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, + 'update_validation' => $updateValidation, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -859,6 +861,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/cluster-info-probe.php'; require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; + require __DIR__.'/update-validation-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1044,6 +1047,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } + if ($mode === 'embedded' && isset($fixture['update_validation'])) { + $observation['update_validation'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_update_validation']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/prometheus-metrics-probe.php b/scripts/conformance/server-parity/prometheus-metrics-probe.php index 3fbbe285c..2192b7dc5 100644 --- a/scripts/conformance/server-parity/prometheus-metrics-probe.php +++ b/scripts/conformance/server-parity/prometheus-metrics-probe.php @@ -4,6 +4,7 @@ use DurableWorkflow\Client; use GuzzleHttp\Client as PrometheusMetricsHttp; +use GuzzleHttp\HandlerStack; // The Prometheus summary of a fresh namespace before and after a known // sequence over raw HTTP: a worker with declared slots on the first queue; a @@ -19,8 +20,12 @@ function finishHttpPrometheusMetrics(Client $admin, array $fixture, string $work $worker = $namespace.'-worker'; $admin->createNamespace($namespace, 'prometheus metrics λ', 30); $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $http = new PrometheusMetricsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, - 'connect_timeout' => 1, 'timeout' => 30]); + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); $raw = static function (string $method, string $path, ?array $body = null, bool $worker = false) use ($http, $token, $namespace): array { $response = $http->request($method, $path, ['headers' => array_filter([ 'Authorization' => $token === null ? null : 'Bearer '.$token, diff --git a/scripts/conformance/server-parity/update-validation-contract.mjs b/scripts/conformance/server-parity/update-validation-contract.mjs new file mode 100644 index 000000000..56007ac98 --- /dev/null +++ b/scripts/conformance/server-parity/update-validation-contract.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; + +// Synchronous update validation, as published Server 2.11.1 serves it. Every +// recorded step must equal the reviewed fixture after three normalizations: +// - identifiers become labels in the order they first appear, walking object +// keys in sorted order so either server's key order reads alike; a reused +// identifier (an update keeping its validation task's id, a retry returning +// its receipt) must be reused, and a new one must be new; +// - timestamps become "