diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index a27f44844..f9faa48d8 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -78,6 +78,9 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore the patch cohort's maintenance loop on every path + run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs + - name: Run the required checks for merge queue groups run: node --test tests/Unit/MergeGroupTriggersTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 5bac4b4dd..a1e7d4a02 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -40,6 +40,7 @@ on: - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityActivityCancellationAckContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs - tests/Unit/ServerParitySystemMetricsContractTest.mjs - tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs @@ -48,6 +49,7 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1593,7 +1595,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md new file mode 100644 index 000000000..b96d9b562 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md @@ -0,0 +1,21 @@ +# October 23, optional condition wait fields + +- **The gap.** Native refused an `open_condition_wait` command without a + `condition_key`, or without a `condition_definition_fingerprint` of the form + `sha256:` followed by 64 hex digits. Published PHP accepts both. +- **How PHP 2.11.1 does it.** `normalizeOpenConditionWaitCommand` treats the key + and the fingerprint as optional strings of any form. It trims them and leaves + them out when empty. `applyOpenConditionWait` records only the fields that + remain, on `ConditionWaitOpened` and on a condition timeout's + `TimerScheduled`. Run summaries name a keyless wait "Waiting for condition", + without a key. Observed live against the published image: a bare + `open_condition_wait`, a key of `" ready "` with the fingerprint + `"any form"`, and blank values all behave this way. +- **What native does now.** The same. The open wait projection stores no + target name for a keyless wait, and the run summary reads such a wait as + PHP does. A key or fingerprint that is not a string is still refused by the + completion transport rules. +- **How it was checked.** A new integration test covers a bare wait, a padded + key with an arbitrary fingerprint, blank values with a timeout, and a key + that is not a string. The 63 existing condition, timed-condition and + observation tests still pass. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md new file mode 100644 index 000000000..c369f493f --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md @@ -0,0 +1,64 @@ +# October 23, message streams + +- **The gap.** All three published SDKs declare `message-streams` from + worker protocol 1.15. Native refused a worker that asked for them, did not + serve the append and diagnostics routes, and ignored + `message_stream_cursors` and `message_stream_waits` on a completion. +- **How PHP 2.11.1 does it.** Learned from its source, then observed live + against the published image: + - Each accepted message is delivered to the current run as the + runtime-reserved signal `__durable_workflow_message_stream`, in the + append's transaction. The signal skips the declared-signal and argument + checks but not the pending-signal and payload ceilings. + - SDK receives open a condition wait keyed by the stream cursor. A stream + message wakes that wait. + - When a signal wakes a condition wait, PHP consumes it without + `SignalApplied`, unless the run's durable contract declares that signal. + This holds for any undeclared signal, not only stream messages. + - The stream cursor's `MessageCursorAdvanced` is recorded after the + completion, with the completed task's snapshot. + - On continue-as-new, the next run receives a payload-free cursor + checkpoint for each acknowledged stream, then every unconsumed message + again, by append order. The old run rejects the signals it had not + consumed. +- **What native does now.** + - `POST /api/workflows/{id}/message-streams/{stream}/messages`, with PHP's + validation order, refusals and diagnostics of malformed input. + - `GET /api/workflows/{id}/message-streams` and + `GET /api/workflows/{id}/message-streams/{stream}`. + - Completion cursors and waits: validated before anything changes, + recorded in the completion's transaction, handed over on continue-as-new. + - The worker capability is PHP's contract with `supported: true`, and + cluster info publishes `message_streams_contract`. + - Continue-as-new no longer refuses a run with message streams. +- **Native changes found on the way.** + - A signal consumed by a condition wait no longer records `SignalApplied` + unless it is declared, and keeps the wait's sequence. + - The instance-level `MessageCursorAdvanced` carries the task snapshot, as + PHP's does. + - A completion reports the run's next message task in + `created_task_ids`, as PHP does. +- **How it was checked.** + - The `message-streams` fixture: 15 appends, 8 refused completions, the + deliveries each worker read, both runs' full event sequences, and the + diagnostics at each step. + - Published PHP 2.11.1 and native pass it on SQLite. The probe's output + is the same size on both, and both runs' event sequences are identical. + - The contract test refuses 50 changed observations, among them a + `SignalApplied` for a consumed message, a missing next message task, a + redelivered consumed message and a missing cursor checkpoint. +- **Not done yet** (Server #325 follow-ups): + - **External payload references.** Native refuses a reference as stream + input with 501 `message_stream_external_payload_unavailable`. PHP keeps + the reference and resolves it when history reaches a worker. + - **Cleanup and upgrade.** Retention cleanup of streams + (`MessageStreamRetentionCleanup`) is not ported. Takeover of PHP data + with streams is still refused. + - **Cluster `capabilities` map.** Native has none yet, so + `capabilities.message_streams` is absent; that map is the cluster info + work. + - **History presentation, outside streams.** Task snapshots lack PHP's + `last_dispatch_attempt_at` and `last_dispatched_at`. A consumed signal's + `SignalReceived` lacks `workflow_sequence` in poll history. + - **Condition fingerprint.** Native requires `condition_definition_fingerprint` + on `open_condition_wait`; PHP accepts a wait without one. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md new file mode 100644 index 000000000..1a82cab36 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md @@ -0,0 +1,81 @@ +# October 23, operator metrics after PHP redispatches an overdue task + +- **Found by.** On PR #673's parity run 39995620421, the Shared Server + fixtures failed on PostgreSQL 17 and MySQL 8.0. The diff came before + any PHP pass line, so published PHP 2.11.1 failed the + `operator-metrics` contract, not native. The failing stage was + `waiting_after_redispatch`. +- **The race in published PHP.** PHP's repair pass redispatches the + overdue ready task in two steps: + - `TaskWatchdog::recoverExistingTask` raises the task's + `repair_count` and projects the run summary, in one transaction. The + run is marked `repair_needed`. + - `TaskDispatcher::markDispatched` then stamps the dispatch, and + `refreshRunSummary` clears `repair_needed` in a separate step. + + `OperatorMetrics::snapshot` reads runs, then tasks, then activities, + then the backlog, with no shared snapshot. `HealthCheck` takes its own + snapshot after the metrics request. So one response can mix states. +- **The four PHP states.** In order, the overdue task is: + 0. overdue and unrepaired; + 1. overdue, repaired, run marked `repair_needed`; + 2. dispatched, run still marked `repair_needed`; + 3. dispatched, run summary refreshed. + + The old contract accepted states 0, 1 and 3 read consistently. It + refused state 2 (the MySQL 8.0 shape) and a backlog read after the + redispatch while the tasks were read before it (the PostgreSQL 17 + shape). +- **First model, refused in review.** The first correction (head + `3fa5cd5288`) let each section of the response show one state, with + sections never going backwards. Security's review (#708 comment + 6373392835) showed this was not enough. Published Workflow `aca7928484` + reads each value with its own query, even inside one section. So a + forward-only change can split the `tasks` section: `dispatch_overdue` + at state 0, `max_repair_count` at state 1, and `unhealthy` at state 2. + The timestamps are read before the counts, so a timestamp can also + outlive its count. +- **Contract decision.** For published PHP only, the + `waiting_after_redispatch` stage follows PHP's actual query order. The + reads that depend on the repair state are, in order: + - runs: `oldest_repair_needed_at`, then `repair_needed`; + - tasks: `oldest_dispatch_overdue_since` (with `oldest_unhealthy_at`, + which PHP builds from the same timestamps), then `dispatch_overdue`, + then `max_repair_count`, then `unhealthy`, counted again; + - backlog: `unhealthy_tasks`, then `repair_needed_runs`. + + The response passes when some sequence of states that never goes + backwards in that order produces exactly its counts and timestamps. The + health request reads its own snapshot afterwards, starting at or after + the response's last read. Each health warning follows the field it + reads: + - `task_transport` follows `tasks.unhealthy`; + - `routing_health` follows `tasks.dispatch_overdue`; + - `durable_resume_paths` follows `backlog.repair_needed_runs`. + + Native runs no repair loop, so native must still report the overdue + task consistently, with each timestamp present exactly when its count + is. A native response that shows a redispatch must show it as one + settled state. +- **How it was checked.** + - Security's counterexample passes for PHP and fails for native. So + does the timestamp kept from before a redispatch. + - New tests refuse each backward step: a recount that is overdue after + the dispatch count was not; a backlog that is overdue after the + recount was not; an overdue count after its timestamp showed the + redispatch; a run timestamp from a repair the later reads have not + reached; and health whose recount is overdue after its dispatch count + was not. + - An exhaustive test covers all 256 combinations of the eight values + that depend on the state. The 37 combinations that some forward-only + order produces pass; the other 219 fail. + - Two earlier tests assumed one read per section. They now accept + health whose backlog was read after the run was projected again, and + whose dispatch count was read before the redispatch. A native + response still fails both. + - The original CI shapes (MySQL 8.0 state 2, PostgreSQL 17 backlog + after the redispatch) still pass. The server-parity unit tests pass: + 5,502 of 5,504, with one intended skip and one test that needs Docker + inside its container. +- **Not changed.** The fixture, its stages and the other stages' expected + values. Native's behaviour is unchanged. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md new file mode 100644 index 000000000..f139ee749 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -0,0 +1,79 @@ +# October 23, task snapshots on history events and described commands + +- **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` + snapshot to every event it records with the task model in hand. Native + added snapshots event by event and missed several. Driving one workflow + through a side effect, an activity, a timer, a child, a signal wait and a + condition wait against both servers showed that native had no snapshot on: + - `TimerFired`, `ChildWorkflowScheduled`, `ChildRunStarted`; + - `SignalWaitOpened`, `ConditionWaitOpened`, `ConditionWaitSatisfied`; + - `SignalApplied` and the instance `MessageCursorAdvanced`. + + Native's own snapshots on `TimerScheduled`, `ActivityStarted`, + `ActivityCompleted` and `ActivityFailed` also left out fields that PHP + keeps, such as `queue`, `connection`, `available_at` and `repair_count`. +- **What native does now.** + - When an event of a type PHP records with the task model is appended with + a task and no snapshot of its own, native adds the task's snapshot. The + list of those event types comes from the Workflow package's source. + - `TimerScheduled` uses the full snapshot. The activity events' snapshots + keep PHP's fields. + - As `RunTimerTask` does, a timer task is leased to itself while its timer + fires: status leased, `leased_at`, `lease_owner` set to the task id, a + workflow-task lease, and one more attempt. Its lease is cleared once the + timer has fired. `TimerFired` therefore carries the leased timer task. +- **How it was checked.** The same probe now finds the same events with + snapshots on both servers, with the same fields. A new integration test + checks the timer, child, signal-wait, signal and condition-wait events. +- **Described commands.** A field-by-field diff of the same workflow's + description on both servers found two more gaps. Each command PHP + describes names the run's `workflow_class` (`WorkflowCommand::record` keeps + it), and says whether its payload preview is omitted + (`payload_preview_omitted`, true only for a stored external payload over the + size limit). Native now sends both. With fresh workers on both sides, the + descriptions then differ only in `request_route_name` and the + `request_fingerprint` hashed from it. +- **Security review corrections.** + - The size check read only PHP's `external_storage` envelope. Native's + stored envelope names its reference `external_payload`, so the flag was + always false. It now reads both, which also corrects the run's input and + output previews. As PHP does, an omitted preview also omits the + command's `target_name`. + - The query read any command payload starting with `dw-`, and SQLite's + `LIKE` and MySQL's case-insensitive collation ignore case. It also read the + wrong column for signals. Native keeps a signal's stored input reference + in its signal record (`workflow_signal_records.arguments`) and a start's + in the command row (`workflow_commands.payload`). Other commands keep + none. The query now reads only that column for each of the two command + types, and only when its first 30 characters equal native's stored prefix + exactly: `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, + and PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + 10.11 the binary cast refuses the upper-case prefix that the column's + collation would match. + - A task snapshot is read only from a task of the event's own run. + - Tests: + - a real uploaded reference on a signal, found in the signal record and + not in the command row; + - that signal record rewritten with the same reference at the 2 MiB limit + (not omitted) and one byte over (omitted, with no target name). The + described command never includes the reference; + - counterfactuals that are not native references: an upper-case or v2 + prefix, the prefix with no valid reference after it, and a large inline + payload starting with `dw-`. An oversized reference in the signal's + command row is not counted either; + - the same sizes and counterfactuals in a start command's row; + - on SQLite, the query itself returns only the exact prefix in the + command type's own column; + - an event recorded on one run with another run's task carries no + snapshot. + + Each test fails without its correction. +- **Left as they are.** + - PHP's command snapshots carry `request_route_name`, a name Laravel + generates at random for the deployment's cached routes. Native has no + such name. + - Native stamps a dispatch on every new task, so the snapshot of a task + created inside a completion keeps `last_dispatch_attempt_at` and + `last_dispatched_at`, which PHP's does not (Server #325 follow-up). + - A message stream's `SignalApplied` and the condition wait it satisfies + follow the message streams change (#709). diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 773b6b4e4..eec7b7b9b 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -14,87 +14,83 @@ qualify a later tree. ## Current state -Updated October 22, 2026. - -- **Merged since October 20.** - - **Backup-first takeover** on SQLite, PostgreSQL and MySQL/MariaDB: - [#523](https://github.com/durable-workflow/server/pull/523), - [#529](https://github.com/durable-workflow/server/pull/529), - [#532](https://github.com/durable-workflow/server/pull/532), - [#533](https://github.com/durable-workflow/server/pull/533), - [#536](https://github.com/durable-workflow/server/pull/536), - [#538](https://github.com/durable-workflow/server/pull/538) and - [#539](https://github.com/durable-workflow/server/pull/539). This covers the - fingerprint, the layouts and WAL preparation. It also covers a PHP-accepted - signal and an expired PHP lease resumed under native, and the refusal of - unsupported features. - - **Multi-node and API parity.** - - [#549](https://github.com/durable-workflow/server/pull/549): the - multi-node kill and restart qualification. - - [#548](https://github.com/durable-workflow/server/pull/548): ownership - guard refusals. - - [#550](https://github.com/durable-workflow/server/pull/550) and - [#551](https://github.com/durable-workflow/server/pull/551): the search - attribute registry and namespace update. - - [#594](https://github.com/durable-workflow/server/pull/594): history - export. - - [#595](https://github.com/durable-workflow/server/pull/595) and - [#596](https://github.com/durable-workflow/server/pull/596): retention and - activity timeouts. - - **The October 21-22 trains.** - [#629](https://github.com/durable-workflow/server/pull/629), - [#638](https://github.com/durable-workflow/server/pull/638), - [#641](https://github.com/durable-workflow/server/pull/641), - [#649](https://github.com/durable-workflow/server/pull/649), - [#659](https://github.com/durable-workflow/server/pull/659) and - [#665](https://github.com/durable-workflow/server/pull/665) covered: - - input envelopes, run repair, legacy settings, activity deadlines, - runtime credentials and principals; - - query and update admission, signals rejected after a run closes, start - fingerprints and all-of barriers; - - cooperative waits, child cancellation and parent-close policies, - continue-as-new messages, history pages, declared signals and build IDs. - - **GHSA-6q48-6c7w-9gw2 trust remediation.** - [#671](https://github.com/durable-workflow/server/pull/671) moves the Rust - locks to the fixed webpki roots. - [#672](https://github.com/durable-workflow/server/pull/672) pins the image's - TLS toolchains and checks every trust store. -- **Open.** - - [#675](https://github.com/durable-workflow/server/pull/675): train of - [#667](https://github.com/durable-workflow/server/pull/667) (definition - routing), [#669](https://github.com/durable-workflow/server/pull/669) (poll - waits), [#670](https://github.com/durable-workflow/server/pull/670) (worker - admission) and [#674](https://github.com/durable-workflow/server/pull/674) - (schedule fixture probe). - - [#676](https://github.com/durable-workflow/server/pull/676): accept - published PHP's health read during or after its own task repair. - - [#673](https://github.com/durable-workflow/server/pull/673): pip, Composer - and Cargo trust in CI and conformance, in Security review. - [#677](https://github.com/durable-workflow/server/issues/677) owns the - remaining clients, with an explicit non-completion boundary. -- **Shared Server fixtures on main.** Embedded passes. Native differs on two - fixtures: - - **`signals-before-worker`:** native records `SignalRejected` after a - terminal event. - - **`worker-listing`:** native reports `_worker_protocol_version` in - `process_metrics`. - - In both, native follows published Server 2.10.0 while the fixtures still - expect the frozen 2.5.14 reference. Separately, the published PHP reference - sometimes differs from itself on `operator-metrics` (postgres-17) because of - its own repair window, which #676 handles. Moving the reference needs a - recorded contract decision, not an edited expectation. -- **Parity surface.** Native serves 97 of published PHP's 175 API routes. That - count compares `php artisan route:list --path=api` on this tree with the - native router. The missing routes are listed below. -- **Next action.** - 1. Verify the Rust suites on #675 and #676, then merge them. - 2. Record the reference decision for Server 2.10.0 and re-record the affected - fixtures from that published release, so main's Shared Server fixtures run - green. - 3. Port the missing families below. - - Packaging the native binary into the server image remains an open gate. +Updated October 24, 2026. + +- **Merged October 24.** Train + [#729](https://github.com/durable-workflow/server/pull/729) (merge + `61219609b`): + - MySQL statement closes are sent with the next command + ([#717](https://github.com/durable-workflow/server/pull/717)), ending the + MySQL-family stalls of [#714](https://github.com/durable-workflow/server/issues/714); + - policy maps that are not JSON objects are refused, like PHP #705, and a + namespace quota map that is not UTF-8 is refused under either name + ([#719](https://github.com/durable-workflow/server/pull/719)); + - worker history shows consumed signal sequences, hydrating only each page's + own signals, plus full task snapshots + ([#710](https://github.com/durable-workflow/server/pull/710)); + - a newer push cancels a superseded Shared Server fixtures run + ([#724](https://github.com/durable-workflow/server/pull/724)). +- **Merged October 20-23.** These are summarized in the checkpoint history: + - backup-first takeover on every database; + - multi-node kill and restart; + - ownership guards, search attributes, namespace update, history export, + retention and activity timeouts; + - the October 21-23 trains; + - the move to the published 2.11.1 reference; + - the GHSA-6q48-6c7w-9gw2 fixes. + + Each train's PR holds its evidence. +- **In review or qualifying, in merge order.** + 1. CI speed and the merge queue: + - [#741](https://github.com/durable-workflow/server/pull/741) runs the + SDK direction profiles in parallel lanes, about 14 minutes off every + database job, and fixes a timestamp-comparison test flake; + - [#742](https://github.com/durable-workflow/server/pull/742) adds the + merge-queue prerequisites (`merge_group` triggers, one fixtures + result, and fixture cells skipped when a change cannot affect them). + 2. This train: operator metrics read across PHP's redispatch steps + ([#708](https://github.com/durable-workflow/server/pull/708)), + query-authority clients that wait for the acknowledgement + ([#720](https://github.com/durable-workflow/server/pull/720)), and the + machine-checked route list below. + 3. Condition waits without a key, task snapshots, the SQLite patch cohort and + message streams (#713, #718, #722, #709). + 4. Update validation with PHP's poll request ID contract + ([#712](https://github.com/durable-workflow/server/pull/712)), then the + held-poll registration fence + ([#739](https://github.com/durable-workflow/server/pull/739), #734) and + worker sessions + ([#740](https://github.com/durable-workflow/server/pull/740), #736), + stacked in that order. + 5. Activity cancellation acknowledgement, and pip, Composer and Cargo trust + in CI ([#673](https://github.com/durable-workflow/server/pull/673)). +- **Shared Server fixtures.** Native, embedded and published PHP 2.11.1 are + compared on SQLite, MySQL 8.0, MariaDB 10.11 and PostgreSQL 17. Where the + published reference itself misbehaves, an explicit boundary separates it + from native, and the PHP owner has an issue for each case: + - `operator-metrics` across PHP's redispatch window (#708); + - the published-SDK patch cohort's SQLite poll refusal + ([#721](https://github.com/durable-workflow/server/issues/721), #722); + - update validation under SQLite write locks + ([#726](https://github.com/durable-workflow/server/issues/726), fixed in + PHP by #730); + - `worker-sdk-persistent-pressure`'s shutdown window + ([#728](https://github.com/durable-workflow/server/issues/728)); + - a retried activity's cooperative cancellation + ([Workflow #923](https://github.com/durable-workflow/workflow/issues/923)); + - a reacquired worker session keeping its old failure reason + ([#737](https://github.com/durable-workflow/server/issues/737)). + + No expectation is edited to make native pass. +- **Parity surface.** Native serves 126 of the 175 API route methods in + published 2.11.1's route table. An execution test sends each one through + the native router, and the ones still answering + `501 rust_capability_not_implemented` must match + [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). + The work in review serves 12 more: message streams (3), update validation + (3), worker sessions (5) and activity cancellation acknowledgement (1). +- **Next action.** Land the items above in order, keep porting the unserved + routes fixture first, and package the native binary into the server image. ## API surface still missing @@ -148,7 +144,10 @@ any other unknown path or method answers PHP's own 404 or 405 - **Run operations.** - Imports from embedded v2 and Waterline v1. - **Other families.** - - Message streams. + - Message streams are served + ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), + except external payload references as input, stream retention cleanup + and takeover of PHP data with streams. - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every @@ -275,9 +274,12 @@ contract fixture before its correction. Add a fixture for each before changing them. - **Retention of runs with external payloads or message streams.** Published PHP deletes a pruned run's external payload objects and releases - its message stream items. Native does not clean either up yet. It skips - such a run with `external_payload_cleanup_unavailable` or - `message_stream_cleanup_unavailable` and keeps its details. + its message stream items. It also compacts its instance's consumed inbound + stream items, and removes the instance's streams with its last run. Native + does not clean any of these up yet. It skips such a run, including any run + whose instance has inbound streams, with + `external_payload_cleanup_unavailable` or + `message_stream_cleanup_unavailable`, and keeps its details. - **Deeply nested JSON bodies.** Published PHP decodes JSON up to 512 levels deep. Native checks every body as PHP does before any route, but its handlers parse with serde_json's 128-level limit. A body nested 129 to 512 @@ -601,7 +603,9 @@ October 21, operator metrics and health: - PHP's maintenance loop redispatches an overdue ready task, after which PHP reports it repaired, not overdue. Native runs no repair loop, so the task stays overdue until a worker claims it. The fixture accepts either - PHP state. + PHP state. Since October 23 it also accepts PHP's reads that straddle + the redispatch, provided they never go backwards; see + [the redispatch-states checkpoint](rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md). - The operator dashboards, capacity evidence, and the repair, retention, activity-timeout and payload-cleanup pass endpoints remain. diff --git a/rust/src/runtime/activity_cancellation_ack.rs b/rust/src/runtime/activity_cancellation_ack.rs new file mode 100644 index 000000000..f89e65662 --- /dev/null +++ b/rust/src/runtime/activity_cancellation_ack.rs @@ -0,0 +1,250 @@ +//! An activity worker's stop acknowledgement after a cooperative +//! cancellation reached its attempt, as published Server 2.11 serves it +//! (`ActivityTaskController::acknowledgeCancellation` over Workflow 2.10's +//! `ActivityCancellationAcknowledgement::recordStopped`), and the receipt +//! the status route shows for it. An acknowledgement records one event with +//! the stop's evidence; it never changes the attempt, its lease or the +//! cleanup deadline, and a repeat returns the original event. +use super::activity_lease_renewal::{micros, worker}; +use super::laravel_rules::{self, Rule}; +use super::transaction::Transaction; +use super::{ + Result, RuntimeError, + store::{capabilities, now}, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; + +/// Published PHP's validation rules for the request, in their order. +const RULES: &laravel_rules::Rules = &[ + ("activity_attempt_id", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("lease_owner", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("request_id", &[Rule::Required, Rule::String, Rule::Max(255)]), +]; + +const REMEDIATION: &str = "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer."; + +/// Whether a request's worker protocol is 1.20 or newer. +fn protocol_supported(protocol: &str) -> bool { + super::cooperative::supports(&json!(["cooperative_cancellation"]), protocol) +} + +store_methods! { + pub(crate) async fn acknowledge_activity_cancellation(&self, task_id: &str, body: Value, protocol: &str) -> Result { + // The protocol gate comes before validation, as in PHP. + if !protocol_supported(protocol) { + return Err(worker(StatusCode::UNPROCESSABLE_ENTITY, json!({"acknowledged":false, + "reason":"activity_cancellation_acknowledgement_not_supported","minimum_protocol_version":"1.20", + "unavailable":["worker_protocol"]}))); + } + let body = laravel_rules::normalize(&body); + if let Err(RuntimeError::Protocol { status, response }) = laravel_rules::validate(&body, RULES) { + return Err(worker(status, response)); + } + let attempt_id = body["activity_attempt_id"].as_str().unwrap_or_default(); + let owner = body["lease_owner"].as_str().unwrap_or_default(); + let request_id = body["request_id"].as_str().unwrap_or_default(); + let refusal = |status, error: &str, reason: &str| { + worker(status, json!({"task_id":task_id,"activity_attempt_id":attempt_id,"error":error,"reason":reason})) + }; + let outcome = |acknowledged: bool, duplicate: bool, reason: Option<&str>, event: Option| { + json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner,"request_id":request_id, + "acknowledged":acknowledged,"duplicate":duplicate,"reason":reason,"history_event_id":event, + "heartbeat_recorded":false}) + }; + // PHP answers a refusal of the recorder with 404 only for a missing + // attempt, otherwise 409. + let refused = |reason: &'static str| { + let status = if reason == "activity_attempt_not_found" { StatusCode::NOT_FOUND } else { StatusCode::CONFLICT }; + worker(status, outcome(false, false, Some(reason), None)) + }; + let mut tx = self.begin().await?; + // PHP's ownership guard, in its order: the task in this namespace, + // then the attempt by its identity, its task and its owner. + let task = Self::query("SELECT id FROM workflow_tasks WHERE id=$1 AND namespace=$2") + .bind(task_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| refusal(StatusCode::NOT_FOUND, "Activity task not found.", "task_not_found"))?; + let Some(rows) = Self::attempt_rows(&mut tx, attempt_id).await? else { + return Err(refusal(StatusCode::NOT_FOUND, "Activity attempt not found.", "attempt_not_found")); + }; + let observed = Self::attempt_status(attempt_id, &rows, None)?; + if observed["workflow_task_id"].as_str() != Some(DB::string(&task, "id")?.as_str()) { + return Err(refusal(StatusCode::CONFLICT, "Activity attempt is not leased for this task.", "task_mismatch")); + } + if observed["lease_owner"].as_str() != Some(owner) { + let mut error = json!({"task_id":task_id,"activity_attempt_id":attempt_id, + "error":"Activity attempt lease is owned by another worker.","reason":"lease_owner_mismatch"}); + error["lease_owner"] = observed["lease_owner"].clone(); + return Err(worker(StatusCode::CONFLICT, error)); + } + let (Some(activity), Some(run), Some(task)) = (&rows.activity, &rows.run, &rows.task) else { + return Err(refused("activity_attempt_not_found")); + }; + // The original claim must come from a cooperative worker on 1.20, + // checked before the request or its delivery. + if !Self::claim_supports_cancellation(task)? { + return Err(worker(StatusCode::CONFLICT, json!({"acknowledged":false, + "reason":"activity_cancellation_acknowledgement_not_supported","required_capability":"cooperative_cancellation", + "minimum_protocol_version":"1.20","worker_id":owner,"unavailable":["worker_claim_capability"], + "remediation":REMEDIATION}))); + } + let run_id = DB::string(run, "id")?; + let execution_id = DB::string(activity, "id")?; + // `recordStopped`'s fence: the execution's current attempt, on this + // run, leased to this owner on both the attempt and its task. + if DB::optional_string(activity, "current_attempt_id")?.as_deref() != Some(attempt_id) + || DB::string(activity, "workflow_run_id")? != run_id + || DB::string(task, "workflow_run_id")? != run_id + || DB::optional_string(&rows.attempt, "lease_owner")?.as_deref() != Some(owner) + || DB::optional_string(task, "lease_owner")?.as_deref() != Some(owner) + { + return Err(refused("activity_cancellation_acknowledgement_fence_mismatch")); + } + if DB::optional_string(run, "cancellation_request_command_id")?.as_deref() != Some(request_id) { + return Err(refused("cancellation_request_mismatch")); + } + let Some((cancellation_event, cancellation)) = + Self::activity_cancellation_event(&mut tx, &run_id, request_id, &execution_id, attempt_id).await? + else { + return Err(refused("activity_cancellation_not_recorded")); + }; + let snapshot = &cancellation["activity_attempt"]; + if snapshot["id"] != attempt_id || snapshot["status"] != "cancelled" || snapshot["lease_owner"] != owner + || snapshot["task_id"] != task_id || snapshot["activity_execution_id"] != execution_id.as_str() + { + return Err(refused("activity_cancellation_snapshot_mismatch")); + } + let Some(context) = Self::cooperative_context(&mut tx, &run_id, request_id).await? else { + return Err(refused("cancellation_context_not_recorded")); + }; + if let Some((event, _)) = Self::acknowledgement_event(&mut tx, &run_id, &cancellation_event, attempt_id).await? { + tx.commit().await?; + return Ok(with_protocol(outcome(true, true, None, Some(event)))); + } + if DB::string(&rows.attempt, "status")? != "cancelled" || DB::string(activity, "status")? != "cancelled" + || DB::string(task, "status")? != "cancelled" + { + return Err(refused("activity_cancellation_not_fenced")); + } + let at = now(); + let acknowledged_at = micros(at); + let mut payload = json!({"sequence":DB::number(activity,"sequence")?,"activity_execution_id":execution_id, + "activity_attempt_id":attempt_id,"lease_owner":owner,"cancellation_history_event_id":cancellation_event, + "request_id":request_id,"root_request_id":context.root_request_id,"cleanup_deadline_at":micros(context.deadline), + "callback_state":"stopped","evidence_source":"activity_worker","acknowledged_at":acknowledged_at, + "received_after_deadline":at >= context.deadline}); + payload["task"] = Self::task_snapshot(task)?; + Self::append_by_command_id(&mut tx, &run_id, "ActivityCancellationAcknowledged", payload, Some(task_id), request_id).await?; + Self::query("UPDATE workflow_runs SET last_progress_at=$1 WHERE id=$2") + .bind(DB::bind_time(at)).bind(&run_id).execute(&mut *tx).await?; + let event = Self::query("SELECT h.id FROM workflow_history_events h JOIN workflow_runs r ON r.id=h.workflow_run_id AND r.last_history_sequence=h.sequence WHERE h.workflow_run_id=$1") + .bind(&run_id).fetch_one(&mut *tx).await?; + let event = DB::string(&event, "id")?; + tx.summary_changed(&run_id); + tx.commit().await?; + Ok(with_protocol(outcome(true, false, None, Some(event)))) + } + + /// The status route's `cancellation_acknowledgement` receipt: present once + /// the run's cooperative request cancelled this attempt. + pub(super) async fn activity_cancellation_receipt( + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + execution_id: &str, + attempt_id: &str, + ) -> Result> { + let Some(request_id) = DB::optional_string(run, "cancellation_request_command_id")? else { + return Ok(None); + }; + let run_id = DB::string(run, "id")?; + let Some((cancellation_event, _)) = + Self::activity_cancellation_event(tx, &run_id, &request_id, execution_id, attempt_id).await? + else { + return Ok(None); + }; + let Some(context) = Self::cooperative_context(tx, &run_id, &request_id).await? else { + return Ok(None); + }; + let mut receipt = json!({"request_id":request_id,"root_request_id":context.root_request_id, + "cleanup_deadline_at":micros(context.deadline),"cancellation_history_event_id":cancellation_event, + "callback_state":"unknown","history_event_id":null,"acknowledged_at":null,"received_after_deadline":null}); + if let Some((event, payload)) = Self::acknowledgement_event(tx, &run_id, &cancellation_event, attempt_id).await? { + receipt["callback_state"] = payload["callback_state"].clone(); + receipt["history_event_id"] = json!(event); + receipt["acknowledged_at"] = payload["acknowledged_at"].clone(); + receipt["received_after_deadline"] = payload["received_after_deadline"].clone(); + } + Ok(Some(receipt)) + } + + /// The `ActivityCancelled` event the request recorded for this attempt. + async fn activity_cancellation_event( + tx: &mut Transaction<'_, DB>, + run_id: &str, + request_id: &str, + execution_id: &str, + attempt_id: &str, + ) -> Result> { + let events = Self::query("SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='ActivityCancelled' ORDER BY sequence") + .bind(run_id).bind(request_id).fetch_all(&mut **tx).await?; + for event in events { + let payload = DB::document_row(&event, "payload")?; + if payload["activity_execution_id"] == execution_id && payload["activity_attempt_id"] == attempt_id { + return Ok(Some((DB::string(&event, "id")?, payload))); + } + } + Ok(None) + } + + /// An earlier acknowledgement of the same cancellation by this attempt. + async fn acknowledgement_event( + tx: &mut Transaction<'_, DB>, + run_id: &str, + cancellation_event: &str, + attempt_id: &str, + ) -> Result> { + let events = Self::query("SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='ActivityCancellationAcknowledged' ORDER BY sequence") + .bind(run_id).fetch_all(&mut **tx).await?; + for event in events { + let payload = DB::document_row(&event, "payload")?; + if payload["cancellation_history_event_id"] == cancellation_event && payload["activity_attempt_id"] == attempt_id { + return Ok(Some((DB::string(&event, "id")?, payload))); + } + } + Ok(None) + } + + /// The request's root and original cleanup deadline, from its + /// `CooperativeCancellationRequested` event. + async fn cooperative_context(tx: &mut Transaction<'_, DB>, run_id: &str, request_id: &str) -> Result> { + let Some(requested) = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='CooperativeCancellationRequested' ORDER BY sequence LIMIT 1") + .bind(run_id).bind(request_id).fetch_optional(&mut **tx).await? + else { + return Ok(None); + }; + let cancellation = &DB::document_row(&requested, "payload")?["cancellation"]; + let Some(deadline) = cancellation["cleanup_deadline_at"].as_str() + .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok()) + .map(|value| value.with_timezone(&chrono::Utc)) + else { + return Ok(None); + }; + let root_request_id = cancellation["lineage"].as_array().and_then(|lineage| lineage.first()) + .and_then(|root| root["request_id"].as_str()).unwrap_or(request_id).to_owned(); + Ok(Some(CooperativeContext { root_request_id, deadline })) + } +} + +struct CooperativeContext { + root_request_id: String, + deadline: chrono::DateTime, +} + +fn with_protocol(mut response: Value) -> Value { + response["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + response["server_capabilities"] = capabilities(); + response +} diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 062f7720a..61b89964c 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -430,7 +430,8 @@ where return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":reported, - "recorded":false,"reason":"stale_attempt","next_task_id":null}), + "recorded":false,"reason":"stale_attempt","next_task_id":null, + "protocol_version":super::WORKER_PROTOCOL_VERSION,"server_capabilities":super::store::capabilities()}), }); } Self::closed_run_outcome(&mut tx, &task, &body, reported).await?; @@ -540,7 +541,9 @@ where } payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":count}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":count,"repair_count":DB::number(&task,"repair_count")?,"queue":DB::string(&task,"queue")?, + "connection":DB::PHP_CONNECTION}); let failed_event = payload.clone(); Self::append( &mut tx, diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 51c16efc6..24412be8b 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -128,12 +128,20 @@ where (true, false, None) }; // Observation preserves the existing lease and never records progress. - let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, + let mut response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, "can_continue":can_continue,"cancel_requested":cancel_requested,"reason":reason,"heartbeat_recorded":false, - "lease_expires_at":expires,"last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?), - "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?, + // PHP formats each time in UTC with microseconds. + "lease_expires_at":expires.map(super::cooperative::timestamp), + "last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?).map(super::cooperative::timestamp), + "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?.map(super::cooperative::timestamp), "activity_status":activity_status,"attempt_status":attempt_status,"task_status":task_status, "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); + // A cooperative request that reached this attempt shows its receipt. + if cancel_requested + && let Some(receipt) = Self::activity_cancellation_receipt(tx, &run, &DB::string(&activity, "id")?, attempt_id).await? + { + response["cancellation_acknowledgement"] = receipt; + } Ok((response, Self::cancellation_request(&run)?.is_none())) } @@ -193,11 +201,13 @@ where other => other, })?; // Published PHP answers with this structured refusal for ordinary - // expired authority, and for any closed run, whose activity is told - // it was cancelled. This read never records progress or renews a - // lease; a concurrently closed attempt remains fenced. + // expired authority, for any closed run, whose activity is told it + // was cancelled, and for an attempt a cooperative cancellation + // reached. This read never records progress or renews a lease; a + // concurrently closed attempt remains fenced. let closed = !observed["run_closed_reason"].is_null(); - if observed["can_continue"] == false && (closed || ordinary && observed["cancel_requested"] == false) { + let delivered = observed["cancel_requested"] == true && observed["reason"] == "attempt_closed"; + if observed["can_continue"] == false && (closed || delivered || ordinary && observed["cancel_requested"] == false) { let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], "cancel_requested":observed["cancel_requested"],"can_continue":false,"reason":observed["reason"], "run_closed_reason":observed["run_closed_reason"],"run_closed_at":observed["run_closed_at"], @@ -301,7 +311,9 @@ where Ok( json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], "cancel_requested":false,"can_continue":true,"reason":null,"run_closed_reason":null,"run_closed_at":null, - "heartbeat_recorded":true,"lease_expires_at":expires,"last_heartbeat_at":at,"worker_session":null, + // PHP formats each time in UTC with microseconds, as status reads do. + "heartbeat_recorded":true,"lease_expires_at":super::cooperative::timestamp(expires), + "last_heartbeat_at":super::cooperative::timestamp(at),"worker_session":null, "protocol_version":protocol,"server_capabilities":super::store::capabilities()}), ) } diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs index ac1a84f9e..f5d6d3cb6 100644 --- a/rust/src/runtime/activity_lease_renewal.rs +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -20,7 +20,7 @@ const RULES: &laravel_rules::Rules = &[ ]; /// A worker-protocol response with PHP's protocol fields. -fn worker(status: StatusCode, mut response: Value) -> RuntimeError { +pub(super) fn worker(status: StatusCode, mut response: Value) -> RuntimeError { response["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); response["server_capabilities"] = capabilities(); RuntimeError::Protocol { status, response } @@ -35,16 +35,16 @@ fn outcome_status(reason: Option<&str>) -> StatusCode { } } -fn micros(at: DateTime) -> String { +pub(super) fn micros(at: DateTime) -> String { at.to_rfc3339_opts(SecondsFormat::Micros, true) } /// The rows an attempt's authority is read from. -struct Rows { - attempt: R, - activity: Option, - run: Option, - task: Option, +pub(super) struct Rows { + pub(super) attempt: R, + pub(super) activity: Option, + pub(super) run: Option, + pub(super) task: Option, } store_methods! { @@ -157,7 +157,7 @@ store_methods! { Ok(status["can_continue"] == true && status["workflow_task_id"] == task_id && status["lease_owner"] == owner) } - async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { + pub(super) async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { let Some(attempt) = Self::query("SELECT * FROM activity_attempts WHERE id=$1") .bind(attempt_id) .fetch_optional(&mut **tx) @@ -188,7 +188,7 @@ store_methods! { /// Workflow's `attemptStatus` over its continuation state. `refused` /// replaces the state with a refusal that cannot continue. - fn attempt_status(attempt_id: &str, rows: &Rows, refused: Option<&'static str>) -> Result { + pub(super) fn attempt_status(attempt_id: &str, rows: &Rows, refused: Option<&'static str>) -> Result { let (can_continue, cancel_requested, reason) = match refused { Some(reason) => (false, false, Some(reason)), None => Self::continuation(rows, now())?, diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 52c26eb99..d4764e814 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -35,6 +35,12 @@ pub(super) trait Backend: Database { const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; /// The history sequence of a signal's `SignalReceived` event. const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; + // PostgreSQL compares text case-sensitively. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "substr(s.arguments,1,30)='dw-native-external-payload-v1:'", + "substr(c.payload,1,30)='dw-native-external-payload-v1:'" + ); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -185,6 +191,11 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "substr(s.arguments,1,30) COLLATE BINARY='dw-native-external-payload-v1:'", + "substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'" + ); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -457,6 +468,12 @@ impl Backend for MySql { const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; + // The payload column's collation ignores case; compare bytes. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)='dw-native-external-payload-v1:'", + "CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'" + ); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 7a5d4d62c..e8590924a 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -105,17 +105,15 @@ where } // Published PHP continues a run whatever it has open. Its activities, // timers, children and waits stay with the closed predecessor, and no - // parent-close policy applies. Streams, service calls and live - // queries are not transferred by native yet. + // parent-close policy applies. Message streams pass to the next run + // when the completion records its stream acknowledgements. Durable + // output streams, service calls and live queries are not transferred + // by native yet. for (sql, selected) in [ ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), ), - ( - "SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index df633552d..b3aa73d38 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -105,6 +105,75 @@ pub(super) async fn encode_command(value: Value, semaphore: Arc) -> R /// PHP's default `command_batch_size` structural limit. const PARALLEL_SPAN_LIMIT: i64 = 1000; +/// The reasons `CooperativeCancellationDelivery::validate` returns. +/// `DefaultWorkflowTaskBridge::deliverCancellation` answers each with its +/// full delivery body under 409: the task and run, no delivery. +const VALIDATION_REASONS: [&str; 8] = [ + "cancellation_request_mismatch", + "cancellation_request_history_missing", + "cancellation_delivery_mismatch", + "cancellation_delivery_history_missing", + "cancellation_delivery_sequence_mismatch", + "cancellation_delivery_shape_mismatch", + "cancellation_delivery_not_eligible", + "cancellation_delivery_requires_scope", +]; + +fn validation_refusal(task_id: &str, run_id: &str, error: RuntimeError) -> RuntimeError { + match error { + RuntimeError::Refused { status: StatusCode::CONFLICT, reason } if VALIDATION_REASONS.contains(&reason) => { + RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"delivered":false,"task_id":task_id,"workflow_run_id":run_id,"request_id":null, + "sequence":null,"call_kind":null,"sequence_span":null,"operation_sequence":null, + "operation_sequence_span":null,"reason":reason}), + } + } + other => other, + } +} + +/// `WorkflowStepHistory::WORKFLOW_STEP_EVENT_TYPES` without activity starts +/// and heartbeats: each follows its own `ActivityScheduled` at the same +/// sequence, so neither changes a boundary's shape, and heartbeats are +/// unbounded. +const STEP_EVENTS: [&str; 30] = [ + "CancellationScopeOpened", "ActivityScheduled", "ActivityRetryScheduled", "ActivityCompleted", "ActivityFailed", + "ActivityCancelled", "ActivityTimedOut", "ChildWorkflowScheduled", "ChildRunStarted", "ChildRunCompleted", + "ChildRunFailed", "ChildRunCancelled", "ChildRunTerminated", "ServiceCallStarted", "ServiceCallCompleted", + "ServiceCallFailed", "ServiceCallCancelled", "WorkflowContinuedAsNew", "ConditionWaitOpened", + "ConditionWaitSatisfied", "ConditionWaitTimedOut", "SignalWaitOpened", "SignalApplied", "MemoUpserted", + "SearchAttributesUpserted", "SideEffectRecorded", "VersionMarkerRecorded", "TimerScheduled", "TimerFired", + "TimerCancelled", +]; + +/// `CooperativeCancellationDelivery::RESOLUTION_TYPES`. +const RESOLUTION_EVENTS: [&str; 18] = [ + "ActivityCompleted", "ActivityFailed", "ActivityCancelled", "ActivityTimedOut", "TimerFired", "TimerCancelled", + "ConditionWaitSatisfied", "ConditionWaitTimedOut", "SignalApplied", "ChildRunCompleted", "ChildRunFailed", + "ChildRunCancelled", "ChildRunTerminated", "ServiceCallCompleted", "ServiceCallFailed", "ServiceCallCancelled", + "SelectionResolved", "SelectionOperationCancelled", +]; + +/// `WorkflowStepHistory::eventMatchesShape` for an activity or timer call. +fn matches_call_shape(call_kind: &str, event_type: &str, payload: &Value) -> bool { + match call_kind { + "activity" => { + matches!(event_type, "ActivityScheduled" | "ActivityRetryScheduled" | "ActivityCompleted" + | "ActivityFailed" | "ActivityCancelled" | "ActivityTimedOut") + // `isLocalActivityEvent` + && payload["execution_mode"] != "local" + && payload["local_activity"] != true + } + "timer" => { + matches!(event_type, "TimerScheduled" | "TimerFired" | "TimerCancelled") + // `isPureTimerEvent`: a wait's timeout timer belongs to the wait. + && !matches!(payload["timer_kind"].as_str(), Some("condition_timeout" | "signal_timeout")) + } + _ => false, + } +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -202,7 +271,7 @@ where .filter(|span| (1..=PARALLEL_SPAN_LIMIT).contains(span) && sequence.checked_add(*span).is_some()) .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_cancellation_delivery"))?, }; - if !matches!(call_kind, "timer" | "child" | "parallel" | "condition" | "signal") + if !matches!(call_kind, "timer" | "child" | "parallel" | "condition" | "signal" | "activity") || (call_kind != "parallel" && span != 1) || body.get("operation_sequence").is_some_and(|v| !v.is_null()) || body @@ -237,13 +306,16 @@ where } let run_id = DB::string(&task, "workflow_run_id")?; let run = Self::active_run(&mut tx, &run_id).await?; + // `CooperativeCancellationDelivery::validate` refusals carry the + // bridge's delivery body. + let refusal = |error: RuntimeError| validation_refusal(task_id, &run_id, error); if DB::optional_string(&run, "cancellation_request_command_id")?.as_deref() != Some(request_id) { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_request_mismatch", - )); + ))); } if DB::optional_instant(&run, "cancellation_deadline_at")? .is_none_or(|deadline| deadline <= now()) @@ -258,16 +330,16 @@ where "operation_sequence_span":1,"reason":null}); let requested=Self::query("SELECT sequence,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='CooperativeCancellationRequested' ORDER BY sequence LIMIT 1") .bind(&run_id).bind(request_id).fetch_optional(&mut *tx).await? - .ok_or_else(|| refuse(StatusCode::CONFLICT,"cancellation_request_history_missing"))?; + .ok_or_else(|| refusal(refuse(StatusCode::CONFLICT,"cancellation_request_history_missing")))?; let request_sequence = DB::number(&requested, "sequence")?; let requested = DB::document_row(&requested, "payload")?; if requested["workflow_command_id"] != request_id || requested["cancellation"]["request_id"] != request_id { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_request_history_missing", - )); + ))); } if let Some(delivery)=Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='CooperativeCancellationDelivered' ORDER BY sequence LIMIT 1") .bind(&run_id).fetch_optional(&mut *tx).await? { @@ -275,7 +347,7 @@ where if payload["workflow_command_id"]!=request_id || payload["sequence"]!=sequence || payload["call_kind"]!=call_kind || payload["sequence_span"].as_i64().unwrap_or(1)!=span || payload["cancellation"]!=requested["cancellation"] || DB::optional_number(&run,"cancellation_delivery_sequence")?!=Some(sequence) { - return Err(refuse(StatusCode::CONFLICT,"cancellation_delivery_mismatch")); + return Err(refusal(refuse(StatusCode::CONFLICT,"cancellation_delivery_mismatch"))); } Self::query("UPDATE workflow_tasks SET lease_expires_at=$1 WHERE id=$2") .bind(DB::bind_time(self.cancellation_lease_expiry(&run)?)) @@ -284,26 +356,26 @@ where return Ok(response); } if DB::optional_number(&run, "cancellation_delivery_sequence")?.is_some() { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_delivery_history_missing", - )); + ))); } let next = Self::authored_sequence(&mut tx, &run_id) .await? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; if sequence > next { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_delivery_sequence_mismatch", - )); + ))); } if call_kind == "parallel" { // `parallelEligibility`; a boundary at the next sequence has no // authored member yet. if sequence < next { - for timer in Self::parallel_timer_members(&mut tx, &run_id, sequence, span, request_sequence).await? { + for timer in Self::parallel_timer_members(&mut tx, &run_id, sequence, span, request_sequence).await.map_err(refusal)? { if DB::string(&timer, "status")? == "pending" { Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } @@ -317,7 +389,8 @@ where sequence, request_sequence, ) - .await?; + .await + .map_err(refusal)?; // `ChildCancellation::prepare`: a waiting policy holds the // delivery and releases the claim until the child closes. if boundary.policy != "abandon" @@ -336,9 +409,24 @@ where "operation_sequence_span":null,"reason":"cancellation_waiting_for_child","claim_released":true})); } } + } else if call_kind == "activity" { + // `validateCallBoundary`, then `cancelOpenWaitAtSequence` with + // the default `try_cancel` policy: an open activity at the + // sequence is cancelled under the request, and one that resolved + // after the request takes the delivery as it is. Native records + // no explicit activity cancellation policy, so it has no waiting + // or abandoning delivery. + Self::validate_call_boundary(&mut tx, &run_id, sequence, call_kind, sequence < next, request_sequence) + .await + .map_err(refusal)?; + if sequence < next { + Self::deliver_activity_cancellation(&mut tx, &run_id, request_id, sequence, now()).await?; + } } else if matches!(call_kind, "condition" | "signal") { if sequence < next { - Self::validate_wait_cancellation_boundary(&mut tx, &run_id, sequence, call_kind, request_sequence).await?; + Self::validate_wait_cancellation_boundary(&mut tx, &run_id, sequence, call_kind, request_sequence) + .await + .map_err(refusal)?; } // `cancelOpenWaitAtSequence`: a timed wait's pending timeout timer is // cancelled, as at a timer boundary. @@ -349,28 +437,18 @@ where Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } } else { - let timer = Self::query( - "SELECT * FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence=$2", - ) - .bind(&run_id) - .bind(sequence) - .fetch_optional(&mut *tx) - .await?; - if sequence < next && timer.is_none() { - return Err(refuse( - StatusCode::CONFLICT, - "cancellation_delivery_shape_mismatch", - )); - } - if let Some(timer) = timer { - if DB::string(&timer, "status")? != "pending" { - return Err(refuse( - StatusCode::CONFLICT, - "cancellation_delivery_not_eligible", - )); - } - Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()) - .await?; + // A timer: `validateCallBoundary`, then `cancelOpenWaitAtSequence` + // cancels it while pending. A wait's timeout timer is not a timer + // boundary, and a timer that fired after the request takes the + // delivery as it is. + Self::validate_call_boundary(&mut tx, &run_id, sequence, call_kind, sequence < next, request_sequence) + .await + .map_err(refusal)?; + if let Some(timer) = Self::query("SELECT * FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence=$2") + .bind(&run_id).bind(sequence).fetch_optional(&mut *tx).await? + && DB::string(&timer, "status")? == "pending" + { + Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } } let at = now(); @@ -411,6 +489,63 @@ where /// at its index, and a group whose members all resolved before the /// request is not eligible. Native cancels timer members only; a group /// with another kind of member keeps its earlier refusal. + /// `ActivityCancellation::record` for a cooperative request: the open + /// execution at `sequence`, its current attempt and that attempt's own + /// task are cancelled, and `ActivityCancelled` names the request. PHP + /// 2.11.1 picks the execution's first task here, which for a retried + /// activity is an earlier, closed one (Workflow #923); native cancels the + /// live attempt's task, so its worker can acknowledge the stop. + async fn deliver_activity_cancellation( + tx: &mut Transaction<'_, DB>, + run_id: &str, + request_id: &str, + sequence: i64, + at: DateTime, + ) -> Result<()> { + let Some(activity) = Self::query("SELECT * FROM activity_executions WHERE workflow_run_id=$1 AND sequence=$2 AND status IN ('pending','running')") + .bind(run_id).bind(sequence).fetch_optional(&mut **tx).await? + else { + return Ok(()); + }; + let activity_id = DB::string(&activity, "id")?; + let attempt_id = DB::optional_string(&activity, "current_attempt_id")?; + Self::query("UPDATE activity_executions SET status='cancelled',closed_at=COALESCE(closed_at,$1) WHERE id=$2") + .bind(DB::bind_time(at)).bind(&activity_id).execute(&mut **tx).await?; + let mut attempt_snapshot = Value::Null; + let mut task_id = None; + if let Some(attempt_id) = &attempt_id { + Self::query("UPDATE activity_attempts SET status='cancelled',lease_expires_at=NULL,closed_at=COALESCE(closed_at,$1) WHERE id=$2 AND activity_execution_id=$3 AND workflow_run_id=$4") + .bind(DB::bind_time(at)).bind(attempt_id).bind(&activity_id).bind(run_id).execute(&mut **tx).await?; + let attempt = Self::query("SELECT * FROM activity_attempts WHERE id=$1 AND activity_execution_id=$2 AND workflow_run_id=$3") + .bind(attempt_id).bind(&activity_id).bind(run_id).fetch_one(&mut **tx).await?; + let live = DB::string(&attempt, "workflow_task_id")?; + // The task keeps its owner and attempt count, as PHP's does. + Self::query("UPDATE workflow_tasks SET status='cancelled',lease_expires_at=NULL,last_error=NULL WHERE id=$1 AND workflow_run_id=$2 AND status IN ('ready','leased')") + .bind(&live).bind(run_id).execute(&mut **tx).await?; + attempt_snapshot = json!({"id":attempt_id,"activity_execution_id":activity_id,"task_id":live, + "attempt_number":DB::number(&attempt,"attempt_number")?,"status":"cancelled", + "lease_owner":DB::optional_string(&attempt,"lease_owner")?, + "started_at":DB::optional_instant(&attempt,"started_at")?,"closed_at":DB::optional_instant(&attempt,"closed_at")?}); + task_id = Some(live); + } + let cancelled = Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(&activity_id).fetch_one(&mut **tx).await?; + let mut payload = Self::activity_payload(&cancelled)?; + payload["workflow_command_id"] = json!(request_id); + payload["activity_attempt_id"] = json!(attempt_id); + payload["attempt_number"] = json!(DB::number(&cancelled, "attempt_count")?); + payload["cancelled_at"] = json!(DB::optional_instant(&cancelled, "closed_at")?); + payload["activity_attempt"] = attempt_snapshot; + if let Some(task_id) = &task_id { + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + payload["task"] = Self::task_snapshot(&task)?; + } + // Recorded with the request's id, as the delivery itself is. + Self::append_by_command_id(tx, run_id, "ActivityCancelled", payload, task_id.as_deref(), request_id).await?; + tx.summary_changed(run_id); + Ok(()) + } + async fn parallel_timer_members( tx: &mut Transaction<'_, DB>, run_id: &str, @@ -476,6 +611,55 @@ where Ok(members) } + /// `validateCallBoundary` for an activity or timer call. Every typed step + /// at the sequence must have the call's shape + /// (`WorkflowStepHistory::assertCompatible`), and an authored sequence + /// must have typed history (`assertTypedHistoryRecorded`); either fault is + /// a shape mismatch. Then a call that resolved before the request is not + /// eligible (`resolvedBeforeRequest`). The next sequence may be a call not + /// yet authored. + async fn validate_call_boundary( + tx: &mut Transaction<'_, DB>, + run_id: &str, + sequence: i64, + call_kind: &str, + authored: bool, + request_sequence: i64, + ) -> Result<()> { + let events = Self::query("SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('CancellationScopeOpened','ActivityScheduled','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityCancelled','ActivityTimedOut','ChildWorkflowScheduled','ChildRunStarted','ChildRunCompleted','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','ServiceCallStarted','ServiceCallCompleted','ServiceCallFailed','ServiceCallCancelled','WorkflowContinuedAsNew','ConditionWaitOpened','ConditionWaitSatisfied','ConditionWaitTimedOut','SignalWaitOpened','SignalApplied','MemoUpserted','SearchAttributesUpserted','SideEffectRecorded','VersionMarkerRecorded','TimerScheduled','TimerFired','TimerCancelled','SelectionResolved','SelectionOperationCancelled') ORDER BY sequence LIMIT 10001") + .bind(run_id) + .fetch_all(&mut **tx) + .await?; + if events.len() > 10000 { + return Err(refuse(StatusCode::SERVICE_UNAVAILABLE, "cancellation_inventory_exceeds_budget")); + } + let (mut typed, mut conflicting, mut resolved) = (false, false, false); + for event in &events { + let kind = DB::string(event, "event_type")?; + let payload = DB::document_row(event, "payload")?; + let at = payload["sequence"].as_i64() == Some(sequence); + if at && STEP_EVENTS.contains(&kind.as_str()) { + typed = true; + conflicting |= !matches_call_shape(call_kind, &kind, &payload); + } + // A cancelled selection operation resolves each member it spans. + let covered = at + || (kind == "SelectionOperationCancelled" + && payload["member_base_sequence"].as_i64().zip(payload["member_size"].as_i64()) + .is_some_and(|(base, size)| base >= 1 && base <= sequence && size > sequence - base)); + resolved |= covered + && RESOLUTION_EVENTS.contains(&kind.as_str()) + && DB::number(event, "sequence")? < request_sequence; + } + if conflicting || (authored && !typed) { + return Err(refuse(StatusCode::CONFLICT, "cancellation_delivery_shape_mismatch")); + } + if resolved { + return Err(refuse(StatusCode::CONFLICT, "cancellation_delivery_not_eligible")); + } + Ok(()) + } + /// `validateCallBoundary` for a condition or signal wait: the authored /// step at the sequence must be that wait (a timed wait's timeout timer /// belongs to it), and it must not have resolved before the request. @@ -869,9 +1053,9 @@ where break; } } - let activities = Self::scalar("SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$1 AND status IN ('pending','running')") - .bind(run_id).fetch_one(&mut *tx).await?; - if unsupported_child || activities != 0 { + // Open activities no longer block a request: the workflow delivers it + // to each activity it awaits (`call_kind: activity`), as PHP's does. + if unsupported_child { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, "cooperative_cancellation_propagation_not_available", diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index a75eadc46..bac63dd88 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -585,6 +585,89 @@ impl Runtime { Ok(view) } + delegate!(message_stream_workflow_bound(workflow_id: &str) -> Result); + delegate!(record_malformed_message(workflow_id: &str, stream: &str, message_id: &Value) -> Result<()>); + delegate!(message_stream_diagnostics(workflow_id: &str, stream: Option<&str>) -> Result>); + + /// `MessageStreamController::append`: the stream name, the workflow in + /// this namespace, the request rules and the payload envelope, in PHP's + /// order. A refused request or payload counts as malformed input. + pub(crate) async fn append_message(&self, workflow_id: &str, stream: &str, body: Value, context: Value) -> Result<(StatusCode, Value)> { + super::message_streams::check_path_name(stream)?; + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"accepted":false,"workflow_id":workflow_id,"stream_name":stream,"outcome":"rejected","reason":"instance_not_found"}), + }); + } + let body = super::laravel_rules::normalize(&body); + let blob = match self.message_payload(&body).await { + Ok(blob) => blob, + Err(error) => { + if matches!(&error, super::RuntimeError::Protocol { status, .. } | super::RuntimeError::Refused { status, .. } if *status == StatusCode::UNPROCESSABLE_ENTITY) { + self.record_malformed_message(workflow_id, stream, &body["message_id"]).await?; + } + return Err(error); + } + }; + let message_id = body["message_id"].as_str().unwrap_or_default().to_owned(); + let (codec, structural) = (self.signal_codec.clone(), self.structural); + match &*self.storage { + Storage::Sqlite(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::Postgres(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::MySql(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + } + } + + /// The request rules, then `AvroPayloadEnvelopeResolver::resolve`: no + /// input is an empty argument list, plain values are encoded, and an + /// inline envelope keeps its exact bytes. + async fn message_payload(&self, body: &Value) -> Result { + super::message_streams::check_append(body)?; + if body.get("input").is_none_or(Value::is_null) { + return super::queries::empty_arguments(self.signal_codec.clone()).await; + } + super::input_envelope::transport(body)?; + if let Some(blob) = self.plain_arguments(body).await? { + return Ok(blob); + } + if self.prepare_external(&body["input"]).await?.is_some() { + // PHP keeps the reference and resolves it for workers; native + // does not resolve stream payload references yet. + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_IMPLEMENTED, + response: json!({"accepted":false,"workflow_id":null,"message_id":body["message_id"],"outcome":"rejected", + "reason":"message_stream_external_payload_unavailable"}), + }); + } + self.decodable(&body["input"]["blob"]).await?; + super::envelope(body, "input") + } + + /// `MessageStreamController::index` and `show`. + pub(crate) async fn message_streams(&self, workflow_id: &str, stream: Option<&str>) -> Result { + if let Some(stream) = stream { + super::message_streams::check_path_name(stream)?; + } + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Workflow not found.","reason":"instance_not_found"}), + }); + } + let streams = self.message_stream_diagnostics(workflow_id, stream).await?; + match stream { + None => Ok(json!({"workflow_id":workflow_id,"count":streams.len(),"streams":streams})), + Some(stream) => match streams.into_iter().next() { + Some(found) => Ok(json!({"workflow_id":workflow_id,"stream":found})), + None => Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"workflow_id":workflow_id,"stream_name":stream,"reason":"message_stream_not_found"}), + }), + }, + } + } + /// `ActivityController::index`. pub(crate) async fn standalone_activities(&self, query: Value) -> Result { let request = super::standalone_activities::list_request(&query)?; @@ -1256,17 +1339,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } } @@ -1401,6 +1484,7 @@ impl Runtime { delegate!(heartbeat_activity(task_id: &str, body: Value, protocol: &str) -> Result); delegate!(activity_status(task_id: &str, body: Value, protocol: &str) -> Result); delegate!(renew_activity_lease(task_id: &str, body: Value) -> Result); + delegate!(acknowledge_activity_cancellation(task_id: &str, body: Value, protocol: &str) -> Result); pub(crate) async fn fail_activity(&self, task_id: &str, body: Value) -> Result { let (failure, blob) = super::activity_failures::prepare_failure(&body, self.signal_codec.clone()).await?; diff --git a/rust/src/runtime/external_wait_summary.rs b/rust/src/runtime/external_wait_summary.rs index 0f9a959a1..06abf03fa 100644 --- a/rust/src/runtime/external_wait_summary.rs +++ b/rust/src/runtime/external_wait_summary.rs @@ -19,6 +19,8 @@ pub(super) struct ExternalWaitHistory { struct ExpectedWait { kind: &'static str, + /// The signal name, or the condition key; PHP allows a condition + /// without one, which reads as empty. name: String, sequence: i64, timeout: Option, @@ -27,6 +29,26 @@ struct ExpectedWait { identity: Value, } +/// A wait's name: a signal always has one; a condition's key is optional. +fn wait_name<'a>(kind: &str, value: &'a Value) -> Option<&'a str> { + match value.as_str().filter(|name| !name.is_empty()) { + Some(name) => Some(name), + None if kind == "condition" && value.is_null() => Some(""), + None => None, + } +} + +impl ExpectedWait { + /// `conditionLabel`: the kind, then the name when there is one. + fn label(&self) -> String { + if self.name.is_empty() { + self.kind.to_owned() + } else { + format!("{} {}", self.kind, self.name) + } + } +} + impl ExternalWaitHistory { pub(super) fn apply(&mut self, kind: &str, payload: &Value) -> bool { if matches!(kind, "TimerScheduled" | "TimerFired" | "TimerCancelled") { @@ -70,7 +92,7 @@ impl ExternalWaitHistory { } let (Some(id), Some(name), Some(sequence)) = ( payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64().filter(|v| *v > 0), ) else { return false; @@ -102,7 +124,7 @@ impl ExternalWaitHistory { let (Some(timer_id), Some(wait_id), Some(name), Some(sequence), Some(delay)) = ( payload["timer_id"].as_str().filter(|v| !v.is_empty()), payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64(), payload["delay_seconds"].as_i64(), ) else { @@ -230,14 +252,11 @@ where kind: expected.kind, id: DB::string(row, "wait_id")?, reason: if expected.fired { - format!( - "Waiting to apply {} {} timeout", - expected.kind, expected.name - ) + format!("Waiting to apply {} timeout", expected.label()) } else if expected.timeout.is_some() { - format!("Waiting for {} {} or timeout", expected.kind, expected.name) + format!("Waiting for {} or timeout", expected.label()) } else { - format!("Waiting for {} {}", expected.kind, expected.name) + format!("Waiting for {}", expected.label()) }, started_at: DB::optional_instant(row, "opened_at")?, deadline_at: expected.timer.as_ref().map(|(_, at)| *at), @@ -272,7 +291,7 @@ where } fn external_wait_matches(row: &DB::Row, expected: &ExpectedWait) -> Result { if DB::string(row, "kind")? != expected.kind - || DB::optional_string(row, "target_name")?.as_deref() != Some(expected.name.as_str()) + || DB::optional_string(row, "target_name")?.unwrap_or_default() != expected.name || DB::optional_number(row, "sequence")? != Some(expected.sequence) || DB::optional_string(row, "history_authority")?.as_deref() != Some("typed_history") || DB::optional_string(row, "source_status")?.as_deref() diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 3cc195f41..bc5734823 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -120,6 +120,12 @@ pub fn router(runtime: Runtime) -> Router { post(repair_run), ) .route("/api/workflows/{workflow_id}/debug", get(debug_current)) + .route("/api/workflows/{workflow_id}/message-streams", get(list_message_streams)) + .route("/api/workflows/{workflow_id}/message-streams/{stream_name}", get(show_message_stream)) + .route( + "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + post(append_message), + ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/debug", get(debug_run), @@ -270,6 +276,10 @@ pub fn router(runtime: Runtime) -> Router { "/api/worker/activity-tasks/{task_id}/renew-lease", post(renew_activity_lease), ) + .route( + "/api/worker/activity-tasks/{task_id}/acknowledge-cancellation", + post(acknowledge_activity_cancellation), + ) .route( "/api/worker/activity-tasks/{task_id}/fail", post(fail_activity), @@ -751,8 +761,8 @@ fn worker_route(rest: &str) -> Option { | ["workflow-tasks" | "activity-tasks", "poll"] | ["query-tasks", "poll"] | ["query-tasks", _, "complete" | "fail" | "status"] - | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat"] - | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease"] + | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat" | "deliver-cancellation"] + | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease" | "acknowledge-cancellation"] ) .then_some(Allowed::Any) } @@ -942,6 +952,7 @@ async fn cluster( "auth_composition_contract": auth_composition, "principal_attribution_contract": principal_attribution, "service_catalog_read_contract": service_catalog, + "message_streams_contract": super::message_streams::manifest(), "payload_codec": "avro", "qualified_for_php_takeover": false}), )) } @@ -983,6 +994,39 @@ fn present(body: &Value, field: &str) -> Option { body.get(field).filter(|value| !value.is_null()).cloned() } +/// `POST /api/workflows/{workflow_id}/message-streams/{stream_name}/messages`. +async fn append_message( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, stream)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let normalized = super::laravel_rules::normalize(&body); + let context = request.context(&body, &workflow_id, "message_stream_append", metadata(&[ + ("stream_name", Some(json!(stream))), + ("message_id", present(&normalized, "message_id")), + ("request_id", present(&normalized, "request_id")), + ])); + let (status, response) = runtime.append_message(&workflow_id, &stream, body, context).await?; + Ok((status, Json(response))) +} + +/// `GET /api/workflows/{workflow_id}/message-streams`. +async fn list_message_streams( + Extension(runtime): Extension, + Path(workflow_id): Path, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, None).await?)) +} + +/// `GET /api/workflows/{workflow_id}/message-streams/{stream_name}`. +async fn show_message_stream( + Extension(runtime): Extension, + Path((workflow_id, stream)): Path<(String, String)>, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, Some(&stream)).await?)) +} + fn signal_context(request: &CommandRequest, body: &Value, workflow_id: &str, name: &str) -> Value { request.context(body, workflow_id, "signal", metadata(&[ ("request_id", present(body, "request_id")), @@ -1862,8 +1906,10 @@ async fn deliver_cancellation( runtime .deliver_cancellation(&task_id, body, request_protocol(&headers)?) .await - // `WorkerProtocol::json`: every answer carries the worker envelope. + // `WorkerProtocol::json`: every answer carries the worker envelope, + // the bridge's refused deliveries included. .map(|body| Json(worker_response(body))) + .map_err(worker_refusal) } async fn complete_workflow( Extension(runtime): Extension, @@ -1872,7 +1918,9 @@ async fn complete_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { - // PHP's transport rules come before every guard, ownership included. + // PHP validates the message-stream fields first, then its transport + // rules, before every guard, ownership included. + super::message_streams::check_completion_fields(&body).map_err(worker_refusal)?; let body = super::completion_rules::checked(body).map_err(worker_refusal)?; if body["commands"].as_array().is_some_and(|commands| { commands @@ -1897,6 +1945,16 @@ async fn complete_workflow( "remediation":format!("Complete typed search-attribute updates only through server nodes advertising worker protocol {minimum} or newer.")}), }); } + if !super::message_streams::Completion::of(&body).is_empty() && !protocol_at_least(&headers, 15)? { + let minimum = super::message_streams::MINIMUM_PROTOCOL_VERSION; + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":body["workflow_task_attempt"], + "outcome":"rejected","reason":"message_streams_unavailable", + "requested_version":request_protocol(&headers)?,"minimum_protocol_version":minimum, + "remediation":format!("Send the X-Durable-Workflow-Protocol-Version header with worker protocol {minimum} or newer.")}), + }); + } let attempt = body["workflow_task_attempt"].clone(); runtime .complete_workflow(&task_id, body, Some(request.principal().clone()), headers.get("x-durable-workflow-protocol-version").and_then(|value| value.to_str().ok())) @@ -2032,6 +2090,18 @@ async fn renew_activity_lease( runtime.renew_activity_lease(&task_id, body).await.map(Json) } +async fn acknowledge_activity_cancellation( + Extension(runtime): Extension, + Path(task_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result> { + runtime + .acknowledge_activity_cancellation(&task_id, body, request_protocol(&headers)?) + .await + .map(Json) +} + async fn activity_status( Extension(runtime): Extension, Path(task_id): Path, diff --git a/rust/src/runtime/message_streams.rs b/rust/src/runtime/message_streams.rs new file mode 100644 index 000000000..42e87ab1a --- /dev/null +++ b/rust/src/runtime/message_streams.rs @@ -0,0 +1,719 @@ +//! Message streams, as published Server 2.11.1 serves them +//! (`MessageStreamController`, `MessageStreamService`, +//! `MessageStreamsContract`). +//! +//! A client appends ordered messages to a named stream of a workflow +//! instance. The server assigns each its position and delivers it to the +//! current run as the runtime-reserved `__durable_workflow_message_stream` +//! signal, in the append's transaction. Workers acknowledge what they consumed +//! with `message_stream_cursors` on a workflow task completion, which records +//! `MessageCursorAdvanced`, and report open receives with +//! `message_stream_waits`. When a run continues as new, the next run receives +//! a payload-free cursor checkpoint and every unconsumed message again. +//! +//! An external payload reference as message input is refused for now: +//! published PHP keeps it by reference and resolves it when history crosses +//! to a worker, which native does not do yet. +use super::laravel_rules::{self, Rule}; +use super::store::{PreparedControlArguments, SignalWrite, Store, id, now}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::Semaphore; + +pub(super) const INTERNAL_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_SCHEMA: &str = "durable-workflow.v2.message-stream.message"; +const CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor"; +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.15"; +const MAX_BATCH: i64 = 100; +const NAME_MESSAGE: &str = "must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens."; + +/// `MessageStreamsContract::manifest`. +pub(super) fn manifest() -> Value { + json!({ + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": MINIMUM_PROTOCOL_VERSION, + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": INTERNAL_SIGNAL, + "message_schema": MESSAGE_SCHEMA, + "cursor_schema": CURSOR_SCHEMA, + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics", + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict", + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run", + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true, + }, + "batch": { + "minimum": 1, + "maximum": MAX_BATCH, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages", + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention", + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await", + }, + "conformance": { + "runtimes": ["php", "python", "rust"], + "targets": ["server", "cloud"], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order", + ], + "artifacts": "published", + }, + }) +} + +/// The worker capability: the manifest, supported. +pub(super) fn worker_capability() -> Value { + let mut capability = manifest(); + capability["supported"] = json!(true); + capability +} + +fn identifier(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) +} + +/// `/^[A-Za-z0-9._:-]{1,128}$/`. +pub(super) fn valid_name(name: &str) -> bool { + name.len() <= 128 && identifier(name) +} + +fn invalid(field: &str, message: String) -> RuntimeError { + laravel_rules::failure(vec![(field.to_owned(), vec![message])]).expect("one message") +} + +/// `validateStreamName`: a path stream name outside the pattern. +pub(super) fn check_path_name(name: &str) -> Result<()> { + if valid_name(name) { + Ok(()) + } else { + Err(invalid("stream_name", format!("The stream name {NAME_MESSAGE}"))) + } +} + +/// The append body's rules, after Laravel's normalization. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("message_id", &[Rule::Required, Rule::String, Rule::Max(191), Rule::Pattern(identifier)]), + ("input", &[Rule::Nullable, Rule::Array]), + ("request_id", &[Rule::Nullable, Rule::String, Rule::Max(255)]), +]; + +pub(super) fn check_append(body: &Value) -> Result<()> { + laravel_rules::validate(body, APPEND_RULES) +} + +/// `diagnosticMessageId`: a refused message id as diagnostics keep it. +pub(super) fn diagnostic_message_id(value: &Value) -> String { + match value.as_str() { + None => "[omitted]".into(), + Some(id) if id.len() <= 191 && identifier(id) => id.into(), + Some(id) => { + use sha2::Digest; + format!("sha256:{}", hex(sha2::Sha256::digest(id.as_bytes()))) + } + } +} + +/// The payload hash PHP stores: sha256 of `codec \0 blob`. +pub(super) fn payload_hash(codec: &str, blob: &str) -> String { + use sha2::Digest; + let mut digest = sha2::Sha256::new(); + digest.update(codec.as_bytes()); + digest.update([0]); + digest.update(blob.as_bytes()); + hex(digest.finalize()) +} + +fn hex(digest: impl AsRef<[u8]>) -> String { + digest.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The worker protocol's message-stream fields on a completion, checked as +/// PHP's first `validate` call does, before the completion's own rules. +const COMPLETION_RULES: &laravel_rules::Rules = &[ + ("message_stream_cursors", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_cursors.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_cursors.*.through_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), + ("message_stream_waits", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_waits.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_waits.*.after_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), +]; + +pub(super) fn check_completion_fields(body: &Value) -> Result<()> { + laravel_rules::validate(&laravel_rules::normalize(body), COMPLETION_RULES) +} + +/// A completion's cursor advances and pending waits, in request order. +pub(super) struct Completion { + pub(super) cursors: Vec, + pub(super) waits: Vec, +} + +impl Completion { + pub(super) fn of(body: &Value) -> Self { + let list = |field: &str| match &body[field] { + Value::Array(items) => items.clone(), + Value::Object(items) => items.values().cloned().collect(), + _ => Vec::new(), + }; + Self { cursors: list("message_stream_cursors"), waits: list("message_stream_waits") } + } + + pub(super) fn is_empty(&self) -> bool { + self.cursors.is_empty() && self.waits.is_empty() + } +} + +/// `metadataStreamName`. +fn metadata_name(item: &Value, field: &str) -> Result { + match item["stream_name"].as_str() { + Some(name) if valid_name(name) => Ok(name.to_owned()), + _ => Err(invalid(field, format!("The stream_name {NAME_MESSAGE}"))), + } +} + +fn non_negative(value: &Value) -> Option { + value.as_i64().filter(|value| *value >= 0) +} + +/// The rejection PHP's service answers for an append it cannot accept. +fn rejection(status: StatusCode, reason: &str, workflow_id: &str, stream: &str, message_id: &str, extra: Value) -> RuntimeError { + let mut response = json!({"accepted":false,"duplicate":false,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":null,"outcome":"rejected","reason":reason}); + if let Value::Object(extra) = extra { + response.as_object_mut().expect("object").extend(extra); + } + RuntimeError::Protocol { status, response } +} + +/// The value a stream delivery signal carries: one map in a list. +fn map(entries: Vec<(&str, crate::codec::Value)>) -> crate::codec::Value { + crate::codec::Value::Map(entries.into_iter().map(|(key, value)| (key.to_owned(), value)).collect::>()) +} + +/// A runtime signal's encoded arguments and command payload. +async fn runtime_arguments(codec: Arc, delivery: crate::codec::Value) -> Result { + let values = vec![delivery]; + let arguments = super::queries::encode_arguments(codec.clone(), crate::codec::Value::Array(values.clone())).await?; + let command = super::queries::encode_arguments( + codec.clone(), + map(vec![ + ("name", crate::codec::Value::String(INTERNAL_SIGNAL.into())), + ("arguments", crate::codec::Value::Array(values.clone())), + ("validation_errors", crate::codec::Value::Array(vec![])), + ]), + ) + .await?; + let value = super::queries::encode_arguments(codec, values[0].clone()).await?; + Ok(PreparedControlArguments { arguments, values, value, command }) +} + +/// `CommandContext::controlPlane`: the context of a delivery with no request. +fn control_plane_context() -> Value { + json!({"caller":{"type":"control_plane","label":"Control Plane"},"auth":{"status":"not_applicable","method":"none"}}) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `NamespaceWorkflowScope::workflowBound`. + pub(crate) async fn message_stream_workflow_bound(&self, workflow_id: &str) -> Result { + Ok(Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// The stream row, created at position zero when absent. + async fn locked_stream(&self, tx: &mut Transaction<'_, DB>, workflow_id: &str, stream: &str) -> Result { + const SELECT: &str = "SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3"; + if let Some(row) = Self::query(SELECT) + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(stream) + .fetch_optional(&mut **tx) + .await? + { + return Ok(row); + } + let at = now(); + Self::query("INSERT INTO workflow_inbound_streams(namespace,workflow_instance_id,stream_name,last_position,cursor_position,duplicate_count,malformed_count,created_at,updated_at) VALUES ($1,$2,$3,0,0,0,0,$4,$5)") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + Ok(Self::query(SELECT).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_one(&mut **tx).await?) + } + + /// The instance's current run, if it is still open. + async fn active_current_run(tx: &mut Transaction<'_, DB>, workflow_id: &str) -> Result> { + let Some(run) = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1") + .bind(workflow_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let open = matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting"); + Ok(open.then_some(run)) + } + + /// `MessageStreamService::append`, in one transaction. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn append_message( + &self, + workflow_id: &str, + stream: &str, + message_id: &str, + payload_codec: &str, + payload_blob: &str, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() { + return Err(rejection(StatusCode::NOT_FOUND, "instance_not_found", workflow_id, stream, message_id, json!({}))); + } + let Some(run) = Self::active_current_run(&mut tx, workflow_id).await? else { + return Err(rejection(StatusCode::CONFLICT, "run_not_active", workflow_id, stream, message_id, json!({}))); + }; + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let stream_id = DB::number(&row, "id")?; + let cursor = DB::number(&row, "cursor_position")?; + let hash = payload_hash(payload_codec, payload_blob); + let at = now(); + if let Some(existing) = Self::query("SELECT position,payload_hash FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).fetch_optional(&mut *tx).await? + { + let same = DB::string(&existing, "payload_hash")? == hash; + let outcome = if same { "duplicate" } else { "message_identity_conflict" }; + Self::query("UPDATE workflow_inbound_streams SET duplicate_count=duplicate_count+1,last_input_outcome=$1,last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(outcome).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + tx.commit().await?; + let response = json!({"accepted":same,"duplicate":same,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":DB::number(&existing,"position")?,"cursor_position":cursor, + "outcome":if same { "duplicate" } else { "rejected" },"reason":if same { Value::Null } else { json!("message_identity_conflict") }}); + return Ok((if same { StatusCode::OK } else { StatusCode::CONFLICT }, response)); + } + let position = DB::number(&row, "last_position")? + 1; + Self::query("INSERT INTO workflow_inbound_stream_items(stream_id,namespace,workflow_instance_id,stream_name,message_id,position,payload_codec,payload_blob,payload_hash,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)") + .bind(stream_id).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).bind(message_id).bind(position) + .bind(payload_codec).bind(payload_blob).bind(&hash).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,last_input_outcome='accepted',last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(position).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + let delivery = Self::message_delivery(stream, message_id, position, payload_codec, payload_blob); + let run_id = DB::string(&run, "id")?; + match Self::deliver_runtime_signal(&mut tx, workflow_id, delivery, context, codec, limits).await? { + Ok(command_id) => { + Self::mark_delivered(&mut tx, stream_id, message_id, &run_id, &command_id).await?; + } + Err(reason) => { + Self::query("DELETE FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,last_input_outcome='message_stream_not_supported' WHERE id=$2") + .bind(position - 1).bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + self.wake.notify_waiters(); + return Err(rejection(StatusCode::CONFLICT, "message_stream_not_supported", workflow_id, stream, message_id, + json!({"detail":reason,"remediation":"Use a PHP, Python, or Rust worker that advertises the message_streams capability."}))); + } + } + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::ACCEPTED, json!({"accepted":true,"duplicate":false,"workflow_id":workflow_id,"run_id":run_id, + "stream_name":stream,"message_id":message_id,"position":position,"cursor_position":cursor, + "outcome":"accepted","reason":null}))) + } + + /// `MessageStreamService::recordMalformed`: counts a refused append + /// against the stream of an open run, keeping no payload. + pub(crate) async fn record_malformed_message(&self, workflow_id: &str, stream: &str, message_id: &Value) -> Result<()> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() || Self::active_current_run(&mut tx, workflow_id).await?.is_none() { + return Ok(()); + } + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET malformed_count=malformed_count+1,last_input_outcome='malformed',last_input_message_id=$1,last_input_at=$2,updated_at=$3 WHERE id=$4") + .bind(diagnostic_message_id(message_id)).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + /// `MessageStreamService::diagnostics`: every stream of the instance, or + /// the named one, by name, without payloads. + pub(crate) async fn message_stream_diagnostics(&self, workflow_id: &str, stream: Option<&str>) -> Result> { + let rows = match stream { + Some(stream) => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_all(&self.pool).await?, + None => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).fetch_all(&self.pool).await?, + }; + let time = |row: &DB::Row, field: &str| -> Result { + Ok(super::task_queues::json_time(DB::optional_instant(row, field)?)) + }; + rows.iter() + .map(|row| { + let last = DB::number(row, "last_position")?; + let cursor = DB::number(row, "cursor_position")?; + let wait = match DB::optional_string(row, "waiting_run_id")? { + Some(run_id) => json!({"run_id":run_id,"after_position":DB::optional_number(row,"waiting_after_position")?, + "since":time(row,"waiting_since")?}), + None => Value::Null, + }; + Ok(json!({"stream_name":DB::string(row,"stream_name")?,"last_position":last,"cursor_position":cursor, + "pending_count":(last - cursor).max(0),"pending_wait":wait, + "duplicate_count":DB::number(row,"duplicate_count")?,"malformed_count":DB::number(row,"malformed_count")?, + "last_input_outcome":DB::optional_string(row,"last_input_outcome")?, + "last_input_message_id":DB::optional_string(row,"last_input_message_id")?, + "last_input_at":time(row,"last_input_at")?})) + }) + .collect() + } + + /// `MessageStreamService::validateCompletion`: the worker's cursor + /// advances and waits, checked before the completion changes anything. + pub(super) async fn validate_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + completion: &Completion, + ) -> Result<()> { + let mut proposed = BTreeMap::new(); + for (index, cursor) in completion.cursors.iter().enumerate() { + let field = |name: &str| format!("message_stream_cursors.{index}.{name}"); + let stream = metadata_name(cursor, &field("stream_name"))?; + if proposed.contains_key(&stream) { + return Err(invalid(&field("stream_name"), "A stream may be acknowledged only once per workflow task completion.".into())); + } + let Some(through) = non_negative(&cursor["through_position"]) else { + return Err(invalid(&field("through_position"), "The through_position must be a non-negative integer.".into())); + }; + let row = Self::query("SELECT id,last_position,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await?; + let Some(row) = row.filter(|row| DB::number(row, "last_position").is_ok_and(|last| through <= last)) else { + return Err(invalid(&field("through_position"), "The cursor cannot advance beyond the last accepted stream position.".into())); + }; + let current = DB::number(&row, "cursor_position")?; + if through > current { + let delivered = Self::scalar("SELECT COUNT(*) FROM workflow_inbound_stream_items WHERE stream_id=$1 AND position>$2 AND position<=$3 AND delivered_run_id=$4") + .bind(DB::number(&row, "id")?).bind(current).bind(through).bind(run_id).fetch_one(&mut **tx).await?; + if delivered != through - current { + return Err(invalid(&field("through_position"), "The cursor must advance contiguously through messages delivered to this workflow run.".into())); + } + } + proposed.insert(stream, current.max(through)); + } + let mut waiting = std::collections::BTreeSet::new(); + for (index, wait) in completion.waits.iter().enumerate() { + let field = |name: &str| format!("message_stream_waits.{index}.{name}"); + let stream = metadata_name(wait, &field("stream_name"))?; + if !waiting.insert(stream.clone()) { + return Err(invalid(&field("stream_name"), "A stream may have only one pending wait per workflow task completion.".into())); + } + let Some(after) = non_negative(&wait["after_position"]) else { + return Err(invalid(&field("after_position"), "The after_position must be a non-negative integer.".into())); + }; + let expected = match proposed.get(&stream) { + Some(cursor) => *cursor, + None => Self::query("SELECT cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await? + .map(|row| DB::number(&row, "cursor_position")).transpose()?.unwrap_or(0), + }; + if after != expected { + return Err(invalid(&field("after_position"), "The pending wait must start after the durable stream cursor.".into())); + } + } + Ok(()) + } + + /// `MessageStreamService::recordCompletion`, after the completion has + /// been applied in the same transaction: consumed messages, each moved + /// cursor with `MessageCursorAdvanced`, pending waits, then the hand-over + /// to a run that continued this one. + pub(super) async fn record_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + task_id: &str, + completion: &Completion, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + for cursor in &completion.cursors { + let stream = cursor["stream_name"].as_str().unwrap_or_default(); + let through = cursor["through_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + let previous = DB::number(&row, "cursor_position")?; + if through <= previous { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET consumed_run_id=$1,consumed_task_id=$2,consumed_at=$3,updated_at=$4 WHERE stream_id=$5 AND position>$6 AND position<=$7") + .bind(run_id).bind(task_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .bind(previous).bind(through).execute(&mut **tx).await?; + Self::query("UPDATE workflow_inbound_streams SET cursor_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,updated_at=$2 WHERE id=$3") + .bind(through).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?).execute(&mut **tx).await?; + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":stream,"previous_position":previous, + "new_position":through,"task":Self::task_snapshot(&task)?}), Some(task_id), None).await?; + } + for wait in &completion.waits { + let stream = wait["stream_name"].as_str().unwrap_or_default(); + let after = wait["after_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + if DB::number(&row, "last_position")? > after { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET waiting_run_id=$1,waiting_after_position=$2,waiting_since=$3,updated_at=$4 WHERE id=$5") + .bind(run_id).bind(after).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut **tx).await?; + } + let current = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1") + .bind(instance_id).fetch_one(&mut **tx).await?; + let current = DB::optional_string(¤t, "current_run_id")?; + if let Some(current) = current.filter(|current| current != run_id) { + self.carry_streams_forward(tx, instance_id, ¤t, codec, limits).await?; + } + Ok(()) + } + + /// `carryForward`: a cursor checkpoint for each acknowledged stream the + /// current run has not received, then every unconsumed message the + /// current run has not received, by append order. + async fn carry_streams_forward( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + current: &str, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + let context = control_plane_context(); + let streams = Self::query("SELECT id,stream_name,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND cursor_position>0 AND (cursor_checkpoint_run_id IS NULL OR cursor_checkpoint_run_id<>$3) ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + for stream in streams { + let checkpoint = map(vec![ + ("schema", crate::codec::Value::String(CURSOR_SCHEMA.into())), + ("stream_name", crate::codec::Value::String(DB::string(&stream, "stream_name")?)), + ("through_position", crate::codec::Value::Long(DB::number(&stream, "cursor_position")?)), + ]); + if Self::deliver_runtime_signal(tx, instance_id, checkpoint, &context, codec.clone(), limits).await?.is_err() { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_cursor_carry_failed" }); + } + Self::query("UPDATE workflow_inbound_streams SET cursor_checkpoint_run_id=$1 WHERE id=$2") + .bind(current).bind(DB::number(&stream, "id")?).execute(&mut **tx).await?; + } + loop { + let items = Self::query("SELECT stream_id,stream_name,message_id,position,payload_codec,payload_blob FROM workflow_inbound_stream_items WHERE namespace=$1 AND workflow_instance_id=$2 AND consumed_at IS NULL AND (delivered_run_id IS NULL OR delivered_run_id<>$3) ORDER BY id LIMIT 100") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + if items.is_empty() { + return Ok(()); + } + for item in items { + let (stream, message_id) = (DB::string(&item, "stream_name")?, DB::string(&item, "message_id")?); + let delivery = Self::message_delivery(&stream, &message_id, DB::number(&item, "position")?, + &DB::string(&item, "payload_codec")?, &DB::string(&item, "payload_blob")?); + match Self::deliver_runtime_signal(tx, instance_id, delivery, &context, codec.clone(), limits).await? { + Ok(command_id) => { + Self::mark_delivered(tx, DB::number(&item, "stream_id")?, &message_id, current, &command_id).await?; + } + Err(_) => { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_carry_failed" }); + } + } + } + } + } + + /// The message a stream delivers: its schema, identity, position and the + /// inline payload envelope. + fn message_delivery(stream: &str, message_id: &str, position: i64, payload_codec: &str, payload_blob: &str) -> crate::codec::Value { + use crate::codec::Value as Payload; + map(vec![ + ("schema", Payload::String(MESSAGE_SCHEMA.into())), + ("stream_name", Payload::String(stream.into())), + ("message_id", Payload::String(message_id.into())), + ("position", Payload::Long(position)), + ("payload_envelope", map(vec![ + ("codec", Payload::String(payload_codec.into())), + ("blob", Payload::String(payload_blob.into())), + ])), + ]) + } + + async fn mark_delivered(tx: &mut Transaction<'_, DB>, stream_id: i64, message_id: &str, run_id: &str, command_id: &str) -> Result<()> { + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET delivered_run_id=$1,workflow_command_id=$2,delivered_at=$3,updated_at=$4 WHERE stream_id=$5 AND message_id=$6") + .bind(run_id).bind(command_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id).bind(message_id) + .execute(&mut **tx).await?; + Ok(()) + } + + /// `WorkflowStub::attemptRuntimeSignalWithArguments` for the instance's + /// current run: no declared-signal admission or argument contract, but + /// the pending-signal and payload ceilings still apply. The accepted + /// command's id, or the refusal's reason. + async fn deliver_runtime_signal( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + delivery: crate::codec::Value, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result> { + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1") + .bind(workflow_id).fetch_one(&mut **tx).await?; + let run_id = DB::string(&instance, "current_run_id")?; + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(&run_id).fetch_one(&mut **tx).await?; + if !matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting") { + return Ok(Err("run_not_active")); + } + let prepared = runtime_arguments(codec.clone(), delivery).await?; + let command_id = id(); + let signal_id = id(); + let command_sequence = DB::number(&run, "last_command_sequence")? + 1; + let message_sequence = DB::number(&instance, "last_message_sequence")? + 1; + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: &run_id, workflow_type: &workflow_type, name: INTERNAL_SIGNAL, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context }; + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") + .bind(&run_id).fetch_one(&mut **tx).await?; + let exceeded = super::memo_limits::Exceeded::pending_signals(limits.pending_signals, pending) + .or_else(|| super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments)); + if let Some(exceeded) = exceeded { + let oversized = exceeded.kind == "payload_size"; + let (values, errors, diagnostics) = if oversized { + (vec![], json!({"arguments":[exceeded.message]}), exceeded.fields()) + } else { + (prepared.values.clone(), json!([]), json!({})) + }; + let command = super::store::prepare_rejected_command(codec.clone(), INTERNAL_SIGNAL.into(), values, + errors.clone(), diagnostics, "signal_codec_unavailable").await?; + let arguments = if oversized { super::queries::empty_arguments(codec).await? } else { prepared.arguments.clone() }; + let row_errors = if oversized { errors } else { json!([]) }; + Self::record_rejected_signal(tx, &write, None, "structural_limit_exceeded", &command, &arguments, &row_errors).await?; + return Ok(Err("structural_limit_exceeded")); + } + Self::record_accepted_signal(tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.summary_changed(&run_id); + Ok(Ok(command_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_follow_phps_pattern() { + assert!(valid_name("orders")); + assert!(valid_name("a.b_c:d-e")); + assert!(valid_name(&"x".repeat(128))); + assert!(!valid_name(&"x".repeat(129))); + assert!(!valid_name("")); + assert!(!valid_name("bad name")); + assert!(!valid_name("bad/name")); + } + + #[test] + fn refused_message_ids_are_bounded_for_diagnostics() { + assert_eq!(diagnostic_message_id(&Value::Null), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!(7)), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!("m3")), "m3"); + let long = "a".repeat(192); + assert_eq!(diagnostic_message_id(&json!(long)).len(), "sha256:".len() + 64); + assert!(diagnostic_message_id(&json!("bad id!")).starts_with("sha256:")); + } + + #[test] + fn payload_hash_covers_codec_and_blob() { + assert_ne!(payload_hash("avro", "a"), payload_hash("avro", "b")); + assert_ne!(payload_hash("avro", "ab"), payload_hash("avroa", "b")); + assert_eq!(payload_hash("avro", "x").len(), 64); + } + + #[test] + fn the_worker_capability_is_the_supported_manifest() { + let capability = worker_capability(); + assert_eq!(capability["supported"], true); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); + assert_eq!(capability["transport"]["internal_signal"], INTERNAL_SIGNAL); + assert_eq!(manifest()["batch"]["maximum"], 100); + } + + #[test] + fn completion_fields_follow_laravels_rules() { + assert!(check_completion_fields(&json!({})).is_ok()); + assert!(check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":0}]})).is_ok()); + let Err(RuntimeError::Protocol { response, .. }) = + check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":-1}]})) + else { + panic!("expected a validation failure"); + }; + assert_eq!(response["message"], "The message_stream_cursors.0.through_position field must be at least 0."); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index a38f42362..84b60c56c 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -24,6 +24,7 @@ macro_rules! store_methods { } mod active_summary; +mod activity_cancellation_ack; mod activity_completion_uploads; mod activity_deadlines; mod activity_failures; @@ -118,6 +119,7 @@ mod service_catalog; mod sqlite; mod start_definition; mod start_request; +mod message_streams; mod storage_test; mod store; mod system_metrics; diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index 9e888fde5..b1a8e1047 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -78,9 +78,12 @@ pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the -/// skip reason. An abandoned activity's hold is read from history. +/// skip reason. An abandoned activity's hold is read from history. A run +/// whose instance has inbound message streams is held too: native does not +/// compact or remove stream state as PHP's MessageStreamRetentionCleanup +/// does yet, so it keeps the run rather than orphan the streams. pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)", 2, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 070e4e364..c18bf74d1 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,8 +31,12 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. -fn oversized(envelope: &Value) -> bool { - envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) +/// PHP's stored envelope names its reference `external_storage`; native's +/// names it `external_payload`. +pub(super) fn oversized(envelope: &Value) -> bool { + ["external_storage", "external_payload"] + .iter() + .any(|key| envelope[key]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes())) } /// Carbon's `toIso8601String()`, which PHP's run description uses. @@ -315,6 +319,8 @@ mod tests { assert!(oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024 + 1}}))); assert!(!oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024}}))); assert!(!oversized(&json!({"codec":"avro","blob":"AA=="}))); + assert!(oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024 + 1}}))); + assert!(!oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024}}))); assert_eq!(without_empty(json!({"a":null,"b":[],"c":{},"d":0})), json!({"c":{},"d":0})); let time = DateTime::parse_from_rfc3339("2026-10-21T07:38:51.123Z").unwrap().with_timezone(&Utc); assert_eq!(iso8601(Some(time)), json!("2026-10-21T07:38:51+00:00")); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index edeba7358..476d353b3 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,8 +15,70 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// The events PHP's Workflow package records with the task model in hand, +/// which therefore carry the task's snapshot. +const TASK_SNAPSHOT_EVENTS: &[&str] = &[ + "ActivityCompleted", + "ActivityFailed", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ChildRunStarted", + "ChildWorkflowScheduled", + "ConditionWaitOpened", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "FailureHandled", + "MemoUpserted", + "MessageCursorAdvanced", + "SearchAttributesUpserted", + "SelectionOperationCancelled", + "SideEffectRecorded", + "SignalApplied", + "SignalWaitOpened", + "TimerFired", + "TimerScheduled", + "VersionMarkerRecorded", + "WorkflowCompleted", + "WorkflowContinuedAsNew", + "WorkflowFailed", + "WorkflowTimedOut", +]; +/// Describe's command journal. Each backend passes its exact, case-sensitive +/// test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`) over the column that holds a +/// command's input: a signal's in its signal record (`s.arguments`), a start's +/// in the command row (`c.payload`). Other commands keep no stored reference. +/// Only a stored reference is read; inline payloads are not. +macro_rules! run_commands_sql { + ($signal:literal, $start:literal) => { + concat!( + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.command_type='signal' AND ", + $signal, + " THEN s.arguments WHEN c.command_type='start' AND ", + $start, + " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" + ) + }; +} +pub(super) use run_commands_sql; +/// One signal command's identity and sequences in its run, as both the +/// public signal route and the runtime's message-stream deliveries write it. +pub(super) struct SignalWrite<'a> { + pub(super) workflow_id: &'a str, + pub(super) run_id: &'a str, + pub(super) workflow_type: &'a str, + pub(super) name: &'a str, + pub(super) command_id: &'a str, + pub(super) signal_id: &'a str, + pub(super) command_sequence: i64, + pub(super) message_sequence: i64, + pub(super) context: &'a Value, +} /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). -const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_STREAM_SIGNAL: &str = super::message_streams::INTERNAL_SIGNAL; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; /// `nextEligibleSignal`: a run's received signals in PHP's order: command /// sequence (unsequenced last), receipt time, then id. @@ -577,18 +639,30 @@ where // refused signals which intentionally have no workflow history event. // Select metadata only: large customer payloads are never decoded or // copied merely to display the command's target and admission result. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + // Only a stored external reference is read, to tell whether its preview + // is omitted; inline payloads stay unread. + let command_rows = Self::query(DB::RUN_COMMANDS_SQL) .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { + // `oversizedPayloadPreview` over the stored reference's envelope; + // an omitted preview also omits the target name, as PHP does. + let omitted = DB::optional_string(&command, "stored_payload")? + .as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)); + let target_name = DB::optional_string(&command, "signal_name")?.filter(|_| !omitted); let mut public = json!({"id":DB::string(&command,"id")?,"sequence":DB::optional_number(&command,"command_sequence")?, "type":DB::string(&command,"command_type")?,"target_scope":DB::string(&command,"target_scope")?, "requested_run_id":DB::optional_string(&command,"requested_workflow_run_id")?, "resolved_run_id":DB::optional_string(&command,"resolved_workflow_run_id")?, - "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, + "target_name":target_name,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, - "workflow_type":DB::optional_string(&command,"workflow_type")?,"workflow_class":DB::optional_string(&command,"workflow_class")?, + // `WorkflowCommand::record` keeps the run's class on each command. + "workflow_type":DB::optional_string(&command,"workflow_type")?, + "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), + "payload_preview_omitted":omitted, "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); @@ -834,25 +908,9 @@ where "signal_codec_unavailable", ) .await?; - let rejected_at = now(); - Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(outcome).bind(&workflow_type) - .bind(&prepared.command).bind(reason).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(rejected_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; - Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name) - .bind(outcome).bind(command_sequence).bind(&row_arguments).bind(DB::document(&row_errors)).bind(reason) - .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut *tx).await?; - Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) - .await?; - Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) - .await?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_rejected_signal(&mut tx, &write, outcome, reason, &prepared.command, &row_arguments, &row_errors).await?; tx.commit().await?; let mut response = json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run, "requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence, @@ -867,40 +925,88 @@ where // signal wait reservation or notification is added here. return Err(super::RuntimeError::Protocol { status, response }); } - let wait = Self::open_wait(&mut tx, ¤t_run).await?; - let mut wait_id = format!("signal-command:{command_id}"); - if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == name) { + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_accepted_signal(&mut tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.commit().await?; + self.wake.notify_waiters(); + Ok( + json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), + ) + } + + /// A refused signal's durable admission record: its rejected command, + /// its rejected signal row and the advanced sequences. No history, task + /// or wait reservation is added. + pub(super) async fn record_rejected_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + outcome: Option<&str>, + reason: &str, + command: &str, + arguments: &str, + errors: &Value, + ) -> Result<()> { + let rejected_at = now(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(outcome).bind(write.workflow_type) + .bind(command).bind(reason).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(rejected_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name) + .bind(outcome).bind(write.command_sequence).bind(arguments).bind(DB::document(errors)).bind(reason) + .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await + } + + /// `recordAcceptedSignalCommand`, `recordAcceptedSignal` and + /// `recordSignalReceivedHistory`: the accepted command, the received + /// signal reserving an open wait for its name, the advanced sequences, + /// `SignalReceived`, then the run's next message task if none is open. + pub(super) async fn record_accepted_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + command: &str, + arguments: &str, + queue: &str, + ) -> Result<()> { + let wait = Self::open_wait(tx, write.run_id).await?; + let mut wait_id = format!("signal-command:{}", write.command_id); + if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == write.name) { let reserved = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_wait_id=$2 AND status='received'") - .bind(¤t_run).bind(text(wait, "signal_wait_id")?).fetch_one(&mut *tx).await?; + .bind(write.run_id).bind(text(wait, "signal_wait_id")?).fetch_one(&mut **tx).await?; if reserved == 0 { wait_id = text(wait, "signal_wait_id")?.to_owned(); } } let accepted_at = now(); Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,command_sequence,message_sequence,accepted_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','accepted','signal_received',$5,'avro',$6,$7,$8,$9,$10)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(DB::string(&run, "workflow_type")?) - .bind(&prepared.command).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(accepted_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.workflow_type) + .bind(command).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(accepted_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,signal_wait_id,status,outcome,command_sequence,payload_codec,arguments,received_at) VALUES ($1,$2,$3,$4,$5,$6,$7,'received','signal_received',$8,'avro',$9,$10)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name).bind(&wait_id) - .bind(command_sequence).bind(&prepared.arguments).bind(DB::bind_time(accepted_at)).execute(&mut *tx).await?; + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name).bind(&wait_id) + .bind(write.command_sequence).bind(arguments).bind(DB::bind_time(accepted_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await?; + Self::append(tx, write.run_id, "SignalReceived", json!({"workflow_command_id":write.command_id,"signal_id":write.signal_id, + "workflow_instance_id":write.workflow_id,"workflow_run_id":write.run_id,"signal_name":write.name,"signal_wait_id":wait_id, + "arguments":wire(arguments),"payload_codec":"avro"}), None, Some(write.command_id)).await?; + Self::enqueue_next_message(tx, write.run_id, queue).await + } + + async fn advance_signal_sequences(tx: &mut Transaction<'_, DB>, write: &SignalWrite<'_>) -> Result<()> { Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) + .bind(write.command_sequence) + .bind(write.run_id) + .execute(&mut **tx) .await?; Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) + .bind(write.message_sequence) + .bind(write.workflow_id) + .execute(&mut **tx) .await?; - Self::append(&mut tx, ¤t_run, "SignalReceived", json!({"workflow_command_id":command_id,"signal_id":signal_id,"workflow_instance_id":workflow_id,"workflow_run_id":current_run,"signal_name":name,"signal_wait_id":wait_id,"arguments":wire(&prepared.arguments),"payload_codec":"avro"}), None, Some(&command_id)).await?; - Self::enqueue_next_message(&mut tx, ¤t_run, &DB::string(&run, "queue")?).await?; - tx.commit().await?; - self.wake.notify_waiters(); - Ok( - json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), - ) + Ok(()) } pub(super) async fn authored_sequence( @@ -1098,6 +1204,17 @@ where run_id: &str, queue: &str, ) -> Result<()> { + Self::next_message_task(tx, run_id, queue).await.map(drop) + } + + /// `PendingMessageTask::createForRun`: the run's next update or signal + /// task, when it is open, has no open workflow task and holds a message + /// that can advance it. The created task's id. + pub(super) async fn next_message_task( + tx: &mut Transaction<'_, DB>, + run_id: &str, + queue: &str, + ) -> Result> { let status = Self::query("SELECT status FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) @@ -1106,12 +1223,12 @@ where DB::string(&status, "status")?.as_str(), "pending" | "running" | "waiting" ) { - return Ok(()); + return Ok(None); } let open = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(run_id).fetch_one(&mut **tx).await?; if open != 0 { - return Ok(()); + return Ok(None); } // `nextReadyUpdate`: the earliest accepted update, unless an earlier // signal is still unapplied. That signal then goes first. @@ -1129,7 +1246,7 @@ where .await? .is_none() { - Self::create_update_task( + let task_id = Self::create_update_task( tx, run_id, queue, @@ -1137,7 +1254,7 @@ where &DB::string(&update, "workflow_command_id")?, ) .await?; - return Ok(()); + return Ok(Some(task_id)); } let (wait, condition_open, declared) = Self::signal_delivery(tx, run_id).await?; let signals = Self::query(RECEIVED_SIGNALS).bind(run_id).fetch_all(&mut **tx).await?; @@ -1147,10 +1264,10 @@ where if !Self::signal_can_advance(wait.as_ref(), condition_open, declared.as_deref(), &name, &signal_wait_id) { continue; } - Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; - break; + let task_id = Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; + return Ok(Some(task_id)); } - Ok(()) + Ok(None) } async fn apply_task_signal( @@ -1187,6 +1304,12 @@ where return Err(refuse(StatusCode::CONFLICT, reason)); } let direct = wait.as_ref().is_some_and(|wait| wait.get("signal_name").is_some()); + // `recordAppliedSignalForSignalResume` applies a signal its open + // signal wait matches, or one the durable contract declares, with + // `SignalApplied`. Otherwise `markConditionWaitSignalConsumed`: the + // open condition wait consumes it at the wait's sequence, without + // that event (an undeclared or runtime-reserved signal). + let applies = direct || declared; let condition_timer = match &wait { Some(wait) if !direct => Self::condition_timer(tx, run_id, wait).await?, _ => None, @@ -1236,7 +1359,7 @@ where .bind(run_id) .execute(&mut **tx) .await?; - Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position}), Some(task_id), None).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position,"task":Self::task_snapshot(task)?}), Some(task_id), None).await?; if let Some(wait) = wait.as_ref().filter(|_| direct) { Self::cancel_wait_timeout(tx, run_id, wait, task_id, applied_at).await?; } @@ -1244,15 +1367,17 @@ where if direct { applied["sequence"] = json!(sequence); } - Self::append( - tx, - run_id, - "SignalApplied", - applied, - Some(task_id), - Some(&command_id), - ) - .await?; + if applies { + Self::append( + tx, + run_id, + "SignalApplied", + applied, + Some(task_id), + Some(&command_id), + ) + .await?; + } if !direct && let Some(mut satisfied) = wait && !condition_timer.as_ref().is_some_and(|(_, fired)| *fired) @@ -1718,7 +1843,10 @@ where self.cancellation_lease_expiry(&run)? }; let leased_at = now(); - if kind == "workflow" { + // PHP binds the immutable claim on every workflow and activity + // claim (`PollRequestLeaseBinding::bindClaimedTask`); cooperative + // delivery and the activity's stop acknowledgement read it back. + if matches!(kind, "workflow" | "activity") { task_payload[super::cooperative::CLAIM_KEY] = json!({"registration_id":DB::number(&worker,"id")?, "worker_id":worker_id,"namespace":self.namespace.as_ref(),"protocol_version":protocol_version, "capabilities":DB::optional_document_row(&worker,"capabilities")?.unwrap_or(json!([])), @@ -1739,7 +1867,7 @@ where let mut claim = json!({"task_id": task_id, "workflow_id": DB::string(&run,"workflow_instance_id")?, "run_id": run_id, "workflow_instance_id": DB::string(&run,"workflow_instance_id")?, "workflow_run_id": run_id, "workflow_type": DB::string(&run,"workflow_type")?, "namespace": self.namespace.as_ref(), "payload_codec": "avro", - "queue": queue, "task_queue": queue, "lease_owner": worker_id, "lease_expires_at": expires}); + "queue": queue, "task_queue": queue, "lease_owner": worker_id, "lease_expires_at": super::cooperative::timestamp(expires)}); if let Some(activity) = activity { let activity_id = DB::string(&activity, "id")?; let policy = @@ -1768,7 +1896,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":leased_at, - "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue}); + "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue, + "connection":DB::PHP_CONNECTION}); Self::append_at( &mut tx, &run_id, @@ -2167,7 +2296,9 @@ where external: super::external_resolution::PreparedCommands, worker: Option, protocol: Option<&str>, + limits: super::memo_limits::StructuralLimits, ) -> Result { + let transition_codec_for_streams = transition_codec.clone(); let commands = body["commands"] .as_array() .filter(|c| !c.is_empty() && c.len() <= 100) @@ -2260,17 +2391,9 @@ where { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_request")); } - text(command, "condition_key")?; + // `normalizeOpenConditionWaitCommand`: the key and the + // definition fingerprint are optional strings, of any form. super::wait_timeouts::timeout_for(command, "invalid_condition_timeout")?; - let fingerprint = text(command, "condition_definition_fingerprint")?; - if !fingerprint.strip_prefix("sha256:").is_some_and(|hash| { - hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_condition_fingerprint", - )); - } } "open_signal_wait" => { reject_fields(command, &["type", "signal_name", "timeout_seconds"])?; @@ -2323,6 +2446,17 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + // `MessageStreamService::validateCompletion`, before anything changes. + let streams = super::message_streams::Completion::of(&body); + let stream_instance = Self::query("SELECT workflow_instance_id FROM workflow_runs WHERE id=$1") + .bind(DB::string(&task, "workflow_run_id")?) + .fetch_one(&mut *tx) + .await?; + let stream_instance = DB::string(&stream_instance, "workflow_instance_id")?; + if !streams.is_empty() { + self.validate_stream_completion(&mut tx, &stream_instance, &DB::string(&task, "workflow_run_id")?, &streams) + .await?; + } // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. @@ -2549,8 +2683,7 @@ where Self::query("INSERT INTO workflow_run_timers(id,workflow_run_id,sequence,status,delay_seconds,fire_at) VALUES ($1,$2,$3,'pending',$4,$5)") .bind(&timer_id).bind(&run_id).bind(sequence).bind(delay).bind(DB::bind_time(fire_at)).execute(&mut *tx).await?; let mut timer_payload = json!({"timer_id":timer_id,"sequence":sequence,"delay_seconds":delay,"fire_at":fire_at, - "task":{"id":task_id,"type":"workflow","status":"leased","lease_owner":DB::optional_string(&task,"lease_owner")?, - "lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?,"attempt_count":DB::number(&task,"attempt_count")?}}); + "task":Self::task_snapshot(&task)?}); super::selections::merge(&mut timer_payload, &metadata); Self::append_at( &mut tx, @@ -2616,6 +2749,18 @@ where .unwrap() .remove("condition_wait_occurrence_id"); } + // `normalizeOptionalString`: PHP trims the condition's key and + // definition fingerprint and drops them when empty. + if !direct { + for field in ["condition_key", "condition_definition_fingerprint"] { + match payload[field].as_str().map(str::trim).filter(|value| !value.is_empty()) { + Some(value) => payload[field] = json!(value), + None => { + payload.as_object_mut().unwrap().remove(field); + } + } + } + } let pending_wait_id = if direct { let pending = Self::query("SELECT signal_wait_id FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_name=$2 AND status='received' ORDER BY command_sequence,id LIMIT 1") .bind(&run_id).bind(text(command, "signal_name")?).fetch_optional(&mut *tx).await?; @@ -2715,10 +2860,14 @@ where .bind(task_id) .execute(&mut *tx) .await?; - Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; Self::resume_undelivered_cancellation(&mut tx, &run_id, &DB::string(&run, "queue")?) .await?; - Self::enqueue_next_message(&mut tx, &run_id, &DB::string(&run, "queue")?).await?; + // PHP reports the run's next message task with the tasks the + // commands created. + if let Some(next) = Self::next_message_task(&mut tx, &run_id, &DB::string(&run, "queue")?).await? { + created.push(next); + } + Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; // PHP's executor leaves every run a completed task did not close // waiting, even when the task recorded only markers or side effects. Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1 AND status IN ('pending','running')") @@ -2733,6 +2882,12 @@ where if let Some(worker) = &worker { Self::attribute_worker_terminal(&mut tx, &run_id, task_id, worker).await?; } + // `MessageStreamService::recordCompletion`: only a continued run + // changes the instance's current run, which then receives the streams. + if !streams.is_empty() || commands.iter().any(|command| command["type"] == "continue_as_new") { + self.record_stream_completion(&mut tx, &stream_instance, &run_id, task_id, &streams, transition_codec_for_streams, limits) + .await?; + } tx.summary_changed(&run_id); tx.commit().await?; self.wake.notify_waiters(); @@ -2903,6 +3058,9 @@ where receipt["attempt_status"] = json!(DB::string(&attempt, "status")?); receipt["task_status"] = json!(DB::string(&task, "status")?); } + // Every worker answer carries PHP's protocol fields. + receipt["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + receipt["server_capabilities"] = capabilities(); return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response: receipt, @@ -2948,7 +3106,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":DB::number(&task,"attempt_count")?}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":DB::number(&task,"attempt_count")?,"queue":DB::string(&task,"queue")?,"connection":DB::PHP_CONNECTION}); Self::append( &mut tx, &run_id, @@ -3074,6 +3233,15 @@ where json!({"timer_id":timer_id}) }; super::selections::merge(&mut payload, &metadata); + // `RunTimerTask`: the timer task is leased to itself while its + // timer fires, so `TimerFired` carries the leased task. + Self::query("UPDATE workflow_tasks SET status='leased',leased_at=$1,lease_owner=$2,lease_expires_at=$3,attempt_count=attempt_count+1 WHERE id=$4") + .bind(DB::bind_time(fired_at)) + .bind(&task_id) + .bind(DB::bind_time(fired_at + chrono::Duration::seconds(self.workflow_task_lease_seconds))) + .bind(&task_id) + .execute(&mut *tx) + .await?; Self::append( &mut tx, &run_id, @@ -3097,7 +3265,7 @@ where .bind(&timer_id) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_tasks SET status='completed' WHERE id=$1") + Self::query("UPDATE workflow_tasks SET status='completed',lease_expires_at=NULL WHERE id=$1") .bind(&task_id) .execute(&mut *tx) .await?; @@ -3426,9 +3594,22 @@ where .execute(&mut **tx) .await?; let recorded_at = recorded_at.unwrap_or_else(now); - // `WorkflowHistoryEvent::record`: an event recorded with a command - // carries that command's snapshot. let mut payload = payload; + // `WorkflowHistoryEvent::record`: an event PHP records with its task + // model carries that task's snapshot. + if let Some(task_id) = task_id + && TASK_SNAPSHOT_EVENTS.contains(&event_type) + && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1 AND workflow_run_id=$2") + .bind(task_id) + .bind(run_id) + .fetch_optional(&mut **tx) + .await? + { + // Only the run's own task: another run's is never copied. + payload["task"] = Self::task_snapshot(&task)?; + } + // An event recorded with a command carries that command's snapshot. if let Some(command_id) = command_id.filter(|_| command_snapshot) && payload.as_object().is_some_and(|fields| !fields.contains_key("command")) && let Some(snapshot) = Self::command_snapshot(tx, command_id).await? @@ -4055,9 +4236,9 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, - "activity_lease_renewal": true}); + "activity_lease_renewal": true, "activity_cancellation_acknowledgement": true}); value["runtime_external_payload_transport"] = super::payload_discovery::transport_manifest(); value["supported_workflow_task_commands"] .as_array_mut() @@ -4086,3 +4267,7 @@ pub(crate) fn capabilities() -> Value { } value } + +#[cfg(test)] +#[path = "store_record_tests.rs"] +mod record_tests; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs new file mode 100644 index 000000000..9529c9473 --- /dev/null +++ b/rust/src/runtime/store_record_tests.rs @@ -0,0 +1,152 @@ +//! History recording and described-command boundaries that a public route +//! cannot reach with mismatched identifiers. +use super::Store; +use crate::runtime::backend::Backend; +use crate::runtime::external_resolution::STORED_PREFIX; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[test] +fn described_commands_read_only_the_exact_native_stored_prefix() { + // Each backend compares the first 30 characters, byte for byte, with the + // whole prefix: a signal's in its signal record, a start's in its command. + assert_eq!(STORED_PREFIX.len(), 30); + let quoted = format!("'{STORED_PREFIX}'"); + for (backend, sql, signal, start) in [ + ("sqlite", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30) COLLATE BINARY=", "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30)=", "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)=", "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + ] { + let expected = format!("CASE WHEN c.command_type='signal' AND {signal}{quoted} THEN s.arguments \ + WHEN c.command_type='start' AND {start}{quoted} THEN c.payload END"); + assert!(sql.contains(&expected), "{backend}: {sql}"); + assert!(!sql.contains("LIKE"), "{backend}: {sql}"); + } +} + +async fn recorded_task(store: &Store, run_id: &str) -> Value { + let row = Store::::query( + "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SideEffectRecorded'", + ) + .bind(run_id) + .fetch_one(&store.pool) + .await + .unwrap(); + ::document_row(&row, "payload").unwrap()["task"].clone() +} + +#[tokio::test] +async fn a_task_snapshot_comes_only_from_the_events_own_run() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("record.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + store + .register( + json!({"worker_id":"record-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]}), + &crate::runtime::QueryTaskConfig::default(), + None, + ) + .await + .unwrap(); + let mut runs = Vec::new(); + for workflow_id in ["record-own", "record-other"] { + let started = store + .start( + json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":{"codec":"avro","blob":"AA=="}}), + BTreeMap::new(), + None, + None, + ) + .await + .unwrap(); + runs.push(started["run_id"].as_str().unwrap().to_owned()); + } + let task = Store::::query("SELECT id FROM workflow_tasks WHERE workflow_run_id=$1") + .bind(&runs[0]) + .fetch_one(&store.pool) + .await + .unwrap(); + let task_id = ::string(&task, "id").unwrap(); + + let mut tx = store.begin().await.unwrap(); + for run_id in &runs { + Store::::record(&mut tx, run_id, "SideEffectRecorded", json!({"sequence":1}), + Some(&task_id), None, None, false) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + assert_eq!(recorded_task(&store, &runs[0]).await["id"], json!(task_id), "the run's own task"); + // Recorded on the other run with the first run's task, the event carries + // no snapshot at all. + assert!(recorded_task(&store, &runs[1]).await.is_null()); +} + +// The query itself, run on rows whose inputs begin like a stored reference: +// only native's exact prefix in the command type's own input column is read. +#[tokio::test] +async fn described_commands_select_no_inline_payload() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("commands.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + let exact = format!("{STORED_PREFIX}{{}}"); + let large = format!("dw-{}", "A".repeat(4096)); + let near = format!("dw-native-external-payload-v1{}", "A".repeat(4096)); + let upper = exact.to_uppercase(); + // (id, type, command payload, signal arguments, read) + let rows = [ + ("c1", "signal", "AA==", Some(exact.as_str()), Some(exact.as_str())), + ("c2", "signal", "AA==", Some(upper.as_str()), None), + ("c3", "signal", "AA==", Some(near.as_str()), None), + ("c4", "signal", "AA==", Some(large.as_str()), None), + // A signal's command row is not its input. + ("c5", "signal", exact.as_str(), Some("AA=="), None), + ("c6", "start", exact.as_str(), None, Some(exact.as_str())), + ("c7", "start", large.as_str(), None, None), + // Other commands keep no stored reference. + ("c8", "update", exact.as_str(), None, None), + ]; + for (sequence, (id, kind, payload, arguments, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run',$2,'accepted',$3,$4)") + .bind(*id) + .bind(*kind) + .bind(*payload) + .bind(sequence as i64 + 1) + .execute(&store.pool) + .await + .unwrap(); + if let Some(arguments) = arguments { + Store::::query("INSERT INTO workflow_signal_records(id,workflow_command_id,signal_name,status,arguments) VALUES ($1,$2,'s','received',$3)") + .bind(format!("s-{id}")) + .bind(*id) + .bind(*arguments) + .execute(&store.pool) + .await + .unwrap(); + } + } + let selected = Store::::query(::RUN_COMMANDS_SQL) + .bind("run") + .bind("instance") + .fetch_all(&store.pool) + .await + .unwrap(); + assert_eq!(selected.len(), rows.len()); + for (row, (id, _, _, _, read)) in selected.iter().zip(&rows) { + assert_eq!(::string(row, "id").unwrap(), *id); + let stored = ::optional_string(row, "stored_payload").unwrap(); + assert_eq!(stored.as_deref(), *read, "{id}"); + } +} diff --git a/rust/src/runtime/wait_projection.rs b/rust/src/runtime/wait_projection.rs index 6467aa436..6dfdfc2ba 100644 --- a/rust/src/runtime/wait_projection.rs +++ b/rust/src/runtime/wait_projection.rs @@ -267,7 +267,8 @@ where ) -> Result<()> { if event_type == "ConditionWaitOpened" { let wait_id = super::text(payload, "condition_wait_id")?; - let key = super::text(payload, "condition_key")?; + // PHP records a condition wait without a key when none was given. + let key = payload["condition_key"].as_str(); let sequence = wait_sequence(payload)?; let projection_id = projection_id(run_id, wait_id); let summary = payload["timeout_seconds"].as_i64().map_or_else( diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 5685a92be..e3079316a 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -349,8 +349,9 @@ where "last_claim_failed_at", "repair_available_at", ] { + // PHP's snapshot times are UTC with microseconds. if let Some(value) = DB::optional_instant(task, field)? { - snapshot[field] = json!(value); + snapshot[field] = json!(super::query_authority::wire(value)); } } for field in ["lease_owner", "compatibility", "last_dispatch_error", "last_claim_error"] { diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 96f2db7e4..99c7cab6b 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod rejected_updates; mod continue_messages; #[path = "execution/update_wait.rs"] mod update_wait; +#[path = "execution/keyless_condition_waits.rs"] +mod keyless_condition_waits; #[path = "execution/payload_limits.rs"] mod payload_limits; #[path = "execution/start_definition.rs"] @@ -113,6 +115,8 @@ mod history_pages; mod declared_signals; #[path = "execution/worker_history_presentation.rs"] mod worker_history_presentation; +#[path = "execution/message_streams.rs"] +mod message_streams; #[path = "execution/poll_waits.rs"] mod poll_waits; #[path = "execution/child_cascade.rs"] @@ -287,6 +291,12 @@ mod external_payloads; #[path = "execution/trace_carriers.rs"] mod trace_carriers; +#[path = "execution/activity_cancellation_ack.rs"] +mod activity_cancellation_ack; + +#[path = "execution/route_coverage.rs"] +mod route_coverage; + enum TestDatabase { Sqlite(tempfile::TempDir), Postgres { @@ -4533,10 +4543,12 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o conflicting["failure"]["message"] = json!("different"); let refused = request(&app_b, "POST", &fail_path, conflicting).await; assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + let mut receipt = refused.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( - refused.1, + receipt, json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null,"protocol_version":"1.20"}) ); let old_complete_path = format!( "/api/worker/activity-tasks/{}/complete", @@ -4582,11 +4594,14 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o // stale receipt PHP returns to that attempt's owner. let refused = request(&app_b, "POST", &old_complete_path, old_complete.clone()).await; assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + let mut receipt = refused.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( - refused.1, + receipt, json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"worker","activity_status":"running","attempt_status":"failed","task_status":"completed"}) + "lease_owner":"worker","activity_status":"running","attempt_status":"failed","task_status":"completed", + "protocol_version":"1.20"}) ); assert_eq!( request(&app_b, "POST", &old_complete_path, old_complete) diff --git a/rust/tests/execution/activity_cancellation_ack.rs b/rust/tests/execution/activity_cancellation_ack.rs new file mode 100644 index 000000000..6ce97c951 --- /dev/null +++ b/rust/tests/execution/activity_cancellation_ack.rs @@ -0,0 +1,196 @@ +//! An activity worker's stop acknowledgement after cooperative delivery. +//! Native cancels a retried activity's live attempt and task, so its worker +//! can acknowledge the stop; published Server 2.11.1 delivers to the +//! execution's first task instead (Workflow #923), which the shared fixture +//! records as a documented reference boundary. +use super::*; + +async fn acknowledge(app: &Router, task: &Value, attempt: &Value, owner: &Value, request_id: &Value) -> (StatusCode, Value) { + request_protocol(app, "POST", + &format!("/api/worker/activity-tasks/{}/acknowledge-cancellation", task.as_str().unwrap()), + json!({"activity_attempt_id":attempt,"lease_owner":owner,"request_id":request_id}), "1.20").await +} + +/// Whether a task row has this status. +async fn task_has_status(database: &TestDatabase, task: &Value, status: &str) -> bool { + let sql = format!("SELECT COUNT(*) FROM workflow_tasks WHERE id='{}' AND status='{status}'", task.as_str().unwrap()); + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await == 1 +} + +#[tokio::test] +async fn a_retried_activity_acknowledges_on_its_live_task() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let registered = request_protocol(&app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["stop-activity"], + "capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, "retried-ack").await; + let task = cooperative_poll(&app, "workflow").await; + let scheduled = finish_task(&app, &task, json!([{"type":"schedule_activity","activity_type":"stop-activity", + "arguments":envelope(Payload::Array(vec![])),"retry_policy":{"max_attempts":2,"backoff_seconds":[0]}, + "start_to_close_timeout":300}])).await; + assert_eq!(scheduled.0, StatusCode::OK, "{}", scheduled.1); + let first = cooperative_poll(&app, "activity").await; + let failed = request_protocol(&app, "POST", + &format!("/api/worker/activity-tasks/{}/fail", first["task_id"].as_str().unwrap()), + json!({"activity_attempt_id":first["activity_attempt_id"],"lease_owner":first["lease_owner"], + "failure":{"message":"retry me","type":"parity.retryable","non_retryable":false}}), "1.20").await; + assert_eq!(failed.0, StatusCode::OK, "{}", failed.1); + let second = cooperative_poll(&app, "activity").await; + assert_ne!(second["task_id"], first["task_id"], "the retry has its own task"); + let requested = request(&app, "POST", "/api/workflows/retried-ack/request-cancellation", + json!({"reason":"stop the retry","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + let request_id = requested.1["cancellation_request"]["request_id"].clone(); + let resumed = cooperative_poll(&app, "workflow").await; + let delivered = request_protocol(&app, "POST", + &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", resumed["task_id"].as_str().unwrap()), + json!({"lease_owner":resumed["lease_owner"],"workflow_task_attempt":resumed["workflow_task_attempt"], + "request_id":request_id,"sequence":1,"call_kind":"activity"}), "1.20").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone()), (StatusCode::OK, json!(true)), "{}", delivered.1); + + // The live task is cancelled and the earlier one keeps its outcome. + assert!(task_has_status(&database, &second["task_id"], "cancelled").await, "live task cancelled"); + assert!(task_has_status(&database, &first["task_id"], "completed").await, "earlier task keeps its outcome"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let cancelled = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ActivityCancelled").unwrap(); + assert_eq!(cancelled["payload"]["task"]["id"], second["task_id"], "{cancelled}"); + assert_eq!(cancelled["payload"]["activity_attempt"]["id"], second["activity_attempt_id"], "{cancelled}"); + + // The earlier attempt is fenced; the live attempt acknowledges once. + let stale = acknowledge(&app, &first["task_id"], &first["activity_attempt_id"], &first["lease_owner"], &request_id).await; + assert_eq!((stale.0, stale.1["reason"].clone()), + (StatusCode::CONFLICT, json!("activity_cancellation_acknowledgement_fence_mismatch")), "{}", stale.1); + let acknowledged = acknowledge(&app, &second["task_id"], &second["activity_attempt_id"], &second["lease_owner"], &request_id).await; + assert_eq!((acknowledged.0, acknowledged.1["acknowledged"].clone(), acknowledged.1["duplicate"].clone()), + (StatusCode::OK, json!(true), json!(false)), "{}", acknowledged.1); + let repeated = acknowledge(&app, &second["task_id"], &second["activity_attempt_id"], &second["lease_owner"], &request_id).await; + assert_eq!((repeated.0, repeated.1["duplicate"].clone(), repeated.1["history_event_id"].clone()), + (StatusCode::OK, json!(true), acknowledged.1["history_event_id"].clone()), "{}", repeated.1); + runtime.close().await; + database.remove().await; +} + +/// The bridge's refused delivery (`DefaultWorkflowTaskBridge::deliverCancellation` +/// over `CooperativeCancellationDelivery::validate`), as published PHP answers +/// it under 409, without the capability map. +pub(super) fn refused_delivery(task: &Value, run_id: &Value, reason: &str) -> Value { + json!({"delivered":false,"task_id":task["task_id"],"workflow_run_id":run_id,"request_id":null,"sequence":null, + "call_kind":null,"sequence_span":null,"operation_sequence":null,"operation_sequence_span":null, + "reason":reason,"protocol_version":"1.20"}) +} + +pub(super) async fn deliver_at(app: &Router, task: &Value, request_id: &Value, sequence: i64, kind: &str) -> (StatusCode, Value) { + let (status, mut body) = request_protocol(app, "POST", + &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "request_id":request_id,"sequence":sequence,"call_kind":kind}), "1.20").await; + let capabilities = body.as_object_mut().and_then(|body| body.remove("server_capabilities")); + assert!(capabilities.is_some_and(|map| map.is_object()), "every delivery answer carries the worker envelope: {body}"); + (status, body) +} + +/// A run whose activity at sequence 1 completed before the request and whose +/// timer at sequence 2 is still pending, with the request's workflow task +/// claimed. Returns the run, the claimed task and the request id. +async fn resolved_activity_then_timer(app: &Router, workflow: &str) -> (Value, Value, Value) { + let registered = request_protocol(app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["stop-activity"], + "capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(app, workflow).await; + let task = cooperative_poll(app, "workflow").await; + let scheduled = finish_task(app, &task, json!([{"type":"schedule_activity","activity_type":"stop-activity", + "arguments":envelope(Payload::Array(vec![])),"retry_policy":{"max_attempts":1,"backoff_seconds":[]}, + "start_to_close_timeout":300}])).await; + assert_eq!(scheduled.0, StatusCode::OK, "{}", scheduled.1); + let activity = cooperative_poll(app, "activity").await; + let completed = request_protocol(app, "POST", + &format!("/api/worker/activity-tasks/{}/complete", activity["task_id"].as_str().unwrap()), + json!({"activity_attempt_id":activity["activity_attempt_id"],"lease_owner":activity["lease_owner"], + "result":envelope(Payload::String("done".into()))}), "1.20").await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + let resumed = cooperative_poll(app, "workflow").await; + let timer = finish_task(app, &resumed, json!([{"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(timer.0, StatusCode::OK, "{}", timer.1); + let requested = request(app, "POST", &format!("/api/workflows/{workflow}/request-cancellation"), + json!({"reason":"stop after the activity","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + let claimed = cooperative_poll(app, "workflow").await; + assert_eq!(claimed["cancellation_request"]["request_id"], requested.1["cancellation_request"]["request_id"], "{claimed}"); + (started, claimed, requested.1["cancellation_request"]["request_id"].clone()) +} + +// Published Workflow 2.11.1 validates an authored activity boundary against +// its typed history before cancelling anything +// (CooperativeCancellationDelivery::validateCallBoundary): another call's +// sequence is a shape mismatch, and an activity that resolved before the +// request is not eligible. Neither refusal records a delivery, and the +// request can still be delivered at its real boundary. +#[tokio::test] +async fn an_activity_delivery_is_refused_at_a_resolved_activity_or_another_calls_sequence() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (started, task, request_id) = resolved_activity_then_timer(&app, "resolved-activity").await; + let run_id = &started["run_id"]; + let before = run_history(&app, &started["workflow_id"], run_id).await; + let undelivered = "SELECT COUNT(*) FROM workflow_runs WHERE cancellation_request_command_id IS NOT NULL AND cancellation_delivery_sequence IS NULL AND cancellation_delivered_at IS NULL"; + assert_eq!(task_retry::rows(&database, undelivered).await, 1); + + let resolved = deliver_at(&app, &task, &request_id, 1, "activity").await; + assert_eq!(resolved, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_not_eligible"))); + let timer_sequence = deliver_at(&app, &task, &request_id, 2, "activity").await; + assert_eq!(timer_sequence, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_shape_mismatch"))); + let activity_sequence = deliver_at(&app, &task, &request_id, 1, "timer").await; + assert_eq!(activity_sequence, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_shape_mismatch"))); + let beyond = deliver_at(&app, &task, &request_id, 4, "activity").await; + assert_eq!(beyond, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_sequence_mismatch"))); + + // Nothing was delivered or cancelled. + assert_eq!(run_history(&app, &started["workflow_id"], run_id).await, before); + assert_eq!(task_retry::rows(&database, undelivered).await, 1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_executions WHERE status='completed'").await, 1); + let duplicate = request(&app, "POST", "/api/workflows/resolved-activity/request-cancellation", json!({})).await; + assert_eq!((duplicate.0, duplicate.1["duplicate"].clone()), (StatusCode::OK, json!(true)), "{}", duplicate.1); + assert_eq!((duplicate.1["cancellation_request"]["delivery_sequence"].clone(), + duplicate.1["cancellation_request"]["delivered_at"].clone()), (Value::Null, Value::Null), "{}", duplicate.1); + + // The pending timer still takes the delivery. + let delivered = deliver_at(&app, &task, &request_id, 2, "timer").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone()), + (StatusCode::OK, json!(true), json!(2)), "{}", delivered.1); + let history = run_history(&app, &started["workflow_id"], run_id).await; + let events = history.as_array().unwrap(); + assert_eq!(&events[..before.as_array().unwrap().len()], before.as_array().unwrap().as_slice()); + let appended: Vec<_> = events[before.as_array().unwrap().len()..].iter().map(|event| event["event_type"].clone()).collect(); + assert_eq!(appended, [json!("TimerCancelled"), json!("CooperativeCancellationDelivered")], "{history}"); + runtime.close().await; + database.remove().await; +} + +// The next sequence may be a call the workflow has not authored yet, so it +// takes an activity delivery without typed history, as published PHP does. +#[tokio::test] +async fn an_activity_delivery_at_the_next_sequence_needs_no_authored_activity() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (started, task, request_id) = resolved_activity_then_timer(&app, "next-activity").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let delivered = deliver_at(&app, &task, &request_id, 3, "activity").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone(), delivered.1["call_kind"].clone()), + (StatusCode::OK, json!(true), json!(3), json!("activity")), "{}", delivered.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let appended: Vec<_> = history.as_array().unwrap()[before.as_array().unwrap().len()..].iter() + .map(|event| event["event_type"].clone()).collect(); + assert_eq!(appended, [json!("CooperativeCancellationDelivered")], "{history}"); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index c67b5f724..ff93c43c1 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -234,22 +234,27 @@ async fn every_authored_deadline_fences_outcomes_and_resolves_once() { json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], "result":envelope(Payload::Long(-1))})).await; assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + let mut receipt = late.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "{field}: worker protocol fields"); assert_eq!( - late.1, + receipt, json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, "lease_owner":activity["lease_owner"],"activity_status":"failed","attempt_status":"failed", - "task_status":"cancelled"}), + "task_status":"cancelled","protocol_version":"1.20"}), "{field}" ); let late = request(&app, "POST", &path("fail"), json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], "failure":{"type":"RuntimeException","message":"late","code":0}})).await; assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + let mut receipt = late.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "{field}: worker protocol fields"); assert_eq!( - late.1, + receipt, json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}), + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null, + "protocol_version":"1.20"}), "{field}" ); } diff --git a/rust/tests/execution/cooperative_waits.rs b/rust/tests/execution/cooperative_waits.rs index 54ce76dab..832bcf919 100644 --- a/rust/tests/execution/cooperative_waits.rs +++ b/rust/tests/execution/cooperative_waits.rs @@ -68,3 +68,87 @@ async fn a_run_parked_in_a_signal_wait_takes_cancellation_there_and_cleans_up() assert_eq!(cancelled["principal"], Value::Null, "{cancelled}"); assert_eq!(cancelled["payload"]["cancellation_cleanup"]["outcome"], "completed", "{cancelled}"); } + +async fn cooperative_worker(app: &Router) { + let registered = request_protocol(app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); +} + +async fn request_root(app: &Router, workflow: &str) -> Value { + let requested = request(app, "POST", &format!("/api/workflows/{workflow}/request-cancellation"), + json!({"reason":"timer boundary","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + requested.1["cancellation_request"]["request_id"].clone() +} + +// A timed condition wait's timeout timer belongs to the wait +// (WorkflowStepHistory::isPureTimerEvent), so published PHP refuses a timer +// delivery there as a shape mismatch without touching the timer; the +// condition delivery then cancels it. +#[tokio::test] +async fn a_timer_delivery_at_a_timed_waits_timeout_is_a_shape_mismatch() { + use super::activity_cancellation_ack::{deliver_at, refused_delivery}; + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + cooperative_worker(&app).await; + let run = start(&app, "timed-wait-timer").await; + let task = cooperative_poll(&app, "workflow").await; + let opened = finish_task(&app, &task, json!([{"type":"open_condition_wait","timeout_seconds":600}])).await; + assert_eq!(opened.0, StatusCode::OK, "{}", opened.1); + let request_id = request_root(&app, "timed-wait-timer").await; + let root = cooperative_poll(&app, "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let wrong = deliver_at(&app, &root, &request_id, 1, "timer").await; + assert_eq!(wrong, (StatusCode::CONFLICT, refused_delivery(&root, &run["run_id"], "cancellation_delivery_shape_mismatch"))); + assert_eq!(run_history(&app, &run["workflow_id"], &run["run_id"]).await, before); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + let delivered = deliver_at(&app, &root, &request_id, 1, "condition").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone()), (StatusCode::OK, json!(true)), "{}", delivered.1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 0); + runtime.close().await; + database.remove().await; +} + +// Only a timer that resolved before the request is not eligible +// (resolvedBeforeRequest). One that fired after it still takes the delivery, +// as it is: published PHP's cancelOpenWaitAtSequence leaves a fired timer +// alone and records the delivery. +#[tokio::test] +async fn a_timer_that_fired_after_the_request_still_takes_the_delivery() { + use super::activity_cancellation_ack::deliver_at; + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + cooperative_worker(&app).await; + let run = start(&app, "fired-after-request").await; + let task = cooperative_poll(&app, "workflow").await; + let timer = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await; + assert_eq!(timer.0, StatusCode::OK, "{}", timer.1); + let request_id = request_root(&app, "fired-after-request").await; + let root = cooperative_poll(&app, "workflow").await; + assert_eq!(root["cancellation_request"]["request_id"], request_id, "{root}"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20); + let fired = loop { + let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + if history.as_array().unwrap().iter().any(|event| event["event_type"] == "TimerFired") { + break history; + } + assert!(tokio::time::Instant::now() < deadline, "timer never fired: {history}"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + }; + let events = fired.as_array().unwrap(); + let position = |kind: &str| events.iter().position(|event| event["event_type"] == kind).unwrap(); + assert!(position("CooperativeCancellationRequested") < position("TimerFired"), "{fired}"); + let delivered = deliver_at(&app, &root, &request_id, 1, "timer").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone()), + (StatusCode::OK, json!(true), json!(1)), "{}", delivered.1); + let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let kinds: Vec<_> = history.as_array().unwrap().iter().map(|event| event["event_type"].clone()).collect(); + assert!(!kinds.contains(&json!("TimerCancelled")), "{history}"); + assert_eq!(kinds.last(), Some(&json!("CooperativeCancellationDelivered")), "{history}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/external_payloads.rs b/rust/tests/execution/external_payloads.rs index c3771c42e..5acb9e25a 100644 --- a/rust/tests/execution/external_payloads.rs +++ b/rust/tests/execution/external_payloads.rs @@ -133,7 +133,7 @@ async fn unclaimed(database: &TestDatabase) -> i64 { } /// One integer the database answers. -async fn count(database: &TestDatabase, query: &'static str) -> i64 { +async fn count(database: &TestDatabase, query: impl sqlx::SqlSafeStr) -> i64 { match database { TestDatabase::Sqlite(dir) => { let pool = sqlx::SqlitePool::connect(&format!( diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index 82237259c..e93b63183 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -151,3 +151,86 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { runtime.close().await; database.remove().await; } + +/// One described command of `kind`, which must show only metadata. +async fn described(app: &Router, workflow_id: &str, kind: &str) -> Value { + let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == kind).unwrap().clone(); + for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { + assert!(command.get(field).is_none(), "{field}: {command}"); + } + assert!(!command.to_string().contains("ep_"), "no reference identity: {command}"); + command +} + +// `WorkflowController::oversizedPayloadPreview` over each command's stored +// envelope: only a stored external reference larger than the payload limit +// (2 MiB by default) omits the preview, and then the target name too. Native +// keeps a signal's stored reference in its signal record and a start's in the +// command row. +#[tokio::test] +async fn described_commands_omit_only_an_oversized_stored_reference() { + let temp = tempfile::tempdir().unwrap(); + let (database, runtime, app) = setup(ExternalPayloadConfig::default(), &temp.path().join("objects")).await; + waiting_run(&app, "described-reference", json!({"worker_id":"describe-worker","task_queue":"described-reference", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + let reference = upload(&app, &argument("signal")).await; + let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", + json!({"input":{"codec":"avro","external_payload":reference}})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + let signal = command["id"].as_str().unwrap().to_owned(); + // The real stored reference is in the signal record, not the command row. + let identity = reference["reference_id"].as_str().unwrap(); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{signal}' AND arguments LIKE 'dw-native-external-payload-v1:%{identity}%'"))).await, 1); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_commands WHERE id='{signal}' AND payload LIKE 'dw-native%'"))).await, 0); + + // The same stored reference, as native writes it, at sizes around the + // limit, and counterfactuals that are not native stored references. + let limit = 2 * 1024 * 1024_i64; + let stored = |size: i64| { + let mut reference = reference.clone(); + reference["size_bytes"] = json!(size); + format!("dw-native-external-payload-v1:{reference}") + }; + let cases = [ + (stored(limit), false), + (stored(limit + 1), true), + // Not native's exact prefix, so never read. + (stored(limit + 1).replace("dw-native-external-payload-v1:", "DW-NATIVE-EXTERNAL-PAYLOAD-V1:"), false), + (stored(limit + 1).replace("dw-native-external-payload-v1:", "dw-native-external-payload-v2:"), false), + // Native's prefix with no valid reference after it. + (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), + // A large inline payload. + (format!("dw-{}", "A".repeat(limit as usize + 1)), false), + ]; + for (arguments, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{arguments}' WHERE workflow_command_id='{signal}'"))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &arguments[..arguments.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), *omitted, "{command}"); + } + // An oversized reference in the signal's command row is not its input. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{}' WHERE workflow_command_id='{signal}'", stored(limit)))).await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{}' WHERE id='{signal}'", stored(limit + 1)))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + + // A start keeps its stored input reference in the command row. + let start = described(&app, "described-reference", "start").await["id"].as_str().unwrap().to_owned(); + for (payload, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{payload}' WHERE id='{start}'"))).await; + let command = described(&app, "described-reference", "start").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "start {}: {command}", &payload[..payload.len().min(48)]); + } + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/keyless_condition_waits.rs b/rust/tests/execution/keyless_condition_waits.rs new file mode 100644 index 000000000..2d66bfa6a --- /dev/null +++ b/rust/tests/execution/keyless_condition_waits.rs @@ -0,0 +1,63 @@ +use super::*; + +// Workflow 2.9's `normalizeOpenConditionWaitCommand`: a condition wait's key +// and definition fingerprint are optional strings of any form. PHP trims them +// and leaves them out when empty. + +fn opened(history: &Value) -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "ConditionWaitOpened") + .map(|event| event["payload"].clone()).unwrap() +} + +#[tokio::test] +async fn condition_waits_need_neither_a_key_nor_a_fingerprint() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "keyless-condition").await; + register(&app, "keyless", json!(["echo"]), json!([])).await; + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_kind='condition' AND wait_reason='Waiting for condition'").await, 1); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_waits WHERE kind='condition' AND status='open' AND target_name IS NULL").await, 1); + + // A signal wakes the wait; the next one keeps a trimmed key and any + // fingerprint, and drops a blank one. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ready ", + "condition_definition_fingerprint":"any form"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let payload = opened(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + assert_eq!((payload["condition_key"].clone(), payload["condition_definition_fingerprint"].clone()), + (json!("ready"), json!("any form")), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition ready'").await, 1); + + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/again", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ", + "condition_definition_fingerprint":"","timeout_seconds":60}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + let scheduled = history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "TimerScheduled").unwrap(); + assert!(scheduled["payload"].get("condition_key").is_none(), "{scheduled}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition or timeout'").await, 1); + + // A key that is not a string is still refused. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/third", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":7}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); +} diff --git a/rust/tests/execution/message_streams.rs b/rust/tests/execution/message_streams.rs new file mode 100644 index 000000000..254cfb6f8 --- /dev/null +++ b/rust/tests/execution/message_streams.rs @@ -0,0 +1,178 @@ +use super::*; + +// Server 2.11.1 message streams: ordered, deduplicated appends delivered to +// the current run as the runtime-reserved signal; worker acknowledgements and +// waits on workflow task completions; the hand-over to a continued run. + +const STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; + +fn stream_worker() -> Value { + json!({"worker_id":"streams","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["message_streams"]}) +} + +/// An SDK receive: a condition wait keyed by the stream cursor. +fn receive(cursor: i64) -> Value { + json!([{"type":"open_condition_wait","condition_key":format!("message-stream:orders:{cursor}"), + "condition_definition_fingerprint":format!("sha256:{}", "a".repeat(64))}]) +} + +async fn append(app: &Router, workflow: &str, stream: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/workflows/{workflow}/message-streams/{stream}/messages"), body).await +} + +async fn complete(app: &Router, task: &Value, commands: Value, extra: Value, protocol: &str) -> (StatusCode, Value) { + let mut body = completion(task, commands); + body.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + request_protocol(app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()), body, protocol).await +} + +async fn streams(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}/message-streams"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +/// The stream deliveries a history holds: (schema tail, stream, message id or cursor). +fn deliveries(history: &Value) -> Vec<(String, String, String)> { + history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "SignalReceived" && event["payload"]["signal_name"] == STREAM_SIGNAL) + .map(|event| { + let blob = event["payload"]["arguments"]["blob"].as_str().unwrap(); + let Payload::Array(values) = durable_workflow_server::codec::ValueCodec::new().unwrap().decode(blob).unwrap() else { + panic!("one argument list"); + }; + let Payload::Map(delivery) = &values[0] else { panic!("one delivery map") }; + let text = |key: &str| match delivery.get(key) { + Some(Payload::String(value)) => value.clone(), + Some(Payload::Long(value)) => value.to_string(), + other => panic!("{key}: {other:?}"), + }; + let schema = text("schema"); + let marker = if schema.ends_with(".cursor") { text("through_position") } else { text("message_id") }; + (schema.rsplit('.').next().unwrap().to_owned(), text("stream_name"), marker) + }) + .collect() +} + +fn types(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +#[tokio::test] +async fn appended_messages_reach_the_run_and_follow_it_when_it_continues() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + let started = start(&app, "streams-flow").await; + let task = poll(&app, "streams", "workflow").await; + let waits = json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}); + let (status, body) = complete(&app, &task, receive(0), waits, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let listed = streams(&app, "streams-flow").await; + assert_eq!(listed["streams"][0]["pending_wait"]["after_position"], 0, "{listed}"); + assert_eq!(listed["streams"][0]["pending_wait"]["run_id"], started["run_id"]); + + let (status, first) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &first["position"], &first["run_id"]), (StatusCode::ACCEPTED, &json!(1), &started["run_id"]), "{first}"); + let (status, again) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &again["duplicate"], &again["position"]), (StatusCode::OK, &json!(true), &json!(1)), "{again}"); + assert!(again.get("run_id").is_none(), "a duplicate names no run: {again}"); + let (status, conflict) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":9}]})).await; + assert_eq!((status, &conflict["reason"]), (StatusCode::CONFLICT, &json!("message_identity_conflict")), "{conflict}"); + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m2","input":[{"n":2}]})).await.1["position"], 2); + let listed = streams(&app, "streams-flow").await; + let orders = &listed["streams"][0]; + assert_eq!((&orders["last_position"], &orders["pending_count"], &orders["duplicate_count"]), (&json!(2), &json!(2), &json!(2)), "{listed}"); + assert!(orders["pending_wait"].is_null(), "an append clears the wait: {listed}"); + + // The condition wait wakes for the first message; the worker reads both. + let task = poll(&app, "streams", "workflow").await; + assert_eq!(deliveries(&task["history_events"]), [("message".into(), "orders".into(), "m1".into()), ("message".into(), "orders".into(), "m2".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}); + let (status, body) = complete(&app, &task, receive(1), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["created_task_ids"].as_array().unwrap().len(), 1, "the next message task is reported: {body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(!types(&history).contains(&"SignalApplied".to_owned()), "the condition wait consumes the message: {history}"); + let cursors: Vec<_> = history.as_array().unwrap().iter().filter(|event| event["event_type"] == "MessageCursorAdvanced").collect(); + assert_eq!(cursors.len(), 2, "{history}"); + assert_eq!(cursors[0]["payload"]["stream_key"], "instance:streams-flow"); + assert_eq!(cursors[0]["payload"]["task"]["status"], "leased"); + assert_eq!((&cursors[1]["payload"]["stream_key"], &cursors[1]["payload"]["new_position"]), (&json!("orders"), &json!(1))); + assert_eq!(cursors[1]["payload"]["task"]["status"], "completed"); + assert_eq!(streams(&app, "streams-flow").await["streams"][0]["cursor_position"], 1); + + // Continue as new acknowledging the second message: the next run gets the + // cursor and the third message, which was still unconsumed. + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m3","input":[{"n":3}]})).await.1["position"], 3); + let task = poll(&app, "streams", "workflow").await; + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":2}]}); + let (status, body) = complete(&app, &task, json!([{"type":"continue_as_new","arguments":envelope(Payload::Array(vec![]))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "streams", "workflow").await; + assert_ne!(task["run_id"], started["run_id"], "{task}"); + assert_eq!(deliveries(&task["history_events"]), [("cursor".into(), "orders".into(), "2".into()), ("message".into(), "orders".into(), "m3".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":3}]}); + let (status, body) = complete(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::String("done".into()))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let orders = &streams(&app, "streams-flow").await["streams"][0]; + assert_eq!((&orders["cursor_position"], &orders["pending_count"]), (&json!(3), &json!(0)), "{orders}"); + let (status, closed) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &closed["reason"]), (StatusCode::CONFLICT, &json!("run_not_active")), "{closed}"); +} + +#[tokio::test] +async fn refused_appends_and_acknowledgements_change_nothing_but_the_counts() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + start(&app, "streams-refused").await; + let task = poll(&app, "streams", "workflow").await; + for (extra, field) in [ + (json!({"message_stream_waits":[{"stream_name":"orders","after_position":1}]}), "message_stream_waits.0.after_position"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}), "message_stream_cursors.0.through_position"), + (json!({"message_stream_waits":[{"stream_name":"bad name","after_position":0}]}), "message_stream_waits.0.stream_name"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":-1}]}), "message_stream_cursors.0.through_position"), + ] { + let (status, body) = complete(&app, &task, receive(0), extra, "1.20").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert!(body["errors"].get(field).is_some(), "{field}: {body}"); + } + let (status, body) = complete(&app, &task, receive(0), json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}), "1.14").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("message_streams_unavailable")), "{body}"); + assert_eq!(streams(&app, "streams-refused").await["count"], 0, "a refused completion records no stream"); + + for (body, field) in [(json!({"input":[1]}), "message_id"), (json!({"message_id":"bad id!"}), "message_id"), + (json!({"message_id":"x".repeat(192)}), "message_id"), (json!({"message_id":"m1","input":"text"}), "input")] { + let (status, refused) = append(&app, "streams-refused", "orders", body).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert!(refused["errors"].get(field).is_some(), "{refused}"); + } + let (status, refused) = append(&app, "streams-refused", "bad%20name", json!({"message_id":"m1"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + let (status, refused) = append(&app, "streams-missing", "orders", json!({"message_id":"m1"})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::NOT_FOUND, &json!("instance_not_found")), "{refused}"); + let orders = &streams(&app, "streams-refused").await["streams"][0]; + assert_eq!((&orders["malformed_count"], &orders["last_position"]), (&json!(4), &json!(0)), "{orders}"); + // The last refused append kept a valid id; the refused stream name and + // the unknown workflow count nowhere. + assert_eq!((&orders["last_input_outcome"], &orders["last_input_message_id"]), (&json!("malformed"), &json!("m1")), "{orders}"); + let (status, refused) = request(&app, "POST", &format!("/api/workflows/streams-refused/signal/{STREAM_SIGNAL}"), json!([1])).await; + assert_eq!((status, &refused["reason"]), (StatusCode::UNPROCESSABLE_ENTITY, &json!("runtime_reserved_signal")), "{refused}"); + let (status, missing) = request(&app, "GET", "/api/workflows/streams-refused/message-streams/other", Value::Null).await; + assert_eq!((status, &missing["reason"]), (StatusCode::NOT_FOUND, &json!("message_stream_not_found")), "{missing}"); +} + +#[tokio::test] +async fn cluster_info_and_workers_see_the_contract() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, info) = request(&app, "GET", "/api/cluster/info", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{info}"); + assert_eq!(info["message_streams_contract"]["schema"], "durable-workflow.v2.message-streams.contract"); + assert_eq!(info["message_streams_contract"]["transport"]["internal_signal"], STREAM_SIGNAL); + let capability = &info["worker_protocol"]["server_capabilities"]["message_streams"]; + assert_eq!(capability["supported"], true, "{capability}"); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); +} diff --git a/rust/tests/execution/route_coverage.rs b/rust/tests/execution/route_coverage.rs new file mode 100644 index 000000000..e49e1ff40 --- /dev/null +++ b/rust/tests/execution/route_coverage.rs @@ -0,0 +1,76 @@ +use super::*; + +// The published Server 2.11.1 API routes native does not serve yet. A port +// that serves a route removes it from this list in the same change, so the +// list in docs/rust-server-port.md cannot drift from the code. +const UNSERVED: &str = include_str!("../fixtures/unserved-php-routes.v2.11.1.txt"); + +/// Sends every method of every published PHP API route through the native +/// router, with each path parameter filled in, and lists those that answer +/// 501 rust_capability_not_implemented. +#[tokio::test] +async fn unserved_published_routes_match_the_recorded_list() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let table: Value = serde_json::from_str(include_str!("../../src/runtime/php-routes.v2.11.1.json")).unwrap(); + let mut unserved = Vec::new(); + let mut checked = 0; + for route in table["routes"].as_array().unwrap() { + let (methods, uri) = (route[0].as_str().unwrap(), route[1].as_str().unwrap()); + if !uri.starts_with("api/") { + continue; + } + for method in methods.split('|').filter(|method| *method != "HEAD") { + checked += 1; + let path = format!("/{}", fill_parameters(uri)); + let worker = path.starts_with("/api/worker/"); + let body = if matches!(method, "GET" | "DELETE") { Body::empty() } else { Body::from("{}") }; + let request = Request::builder() + .method(method) + .uri(&path) + .header("authorization", "Bearer test-token") + .header("accept", "application/json") + .header("content-type", "application/json") + .header( + if worker { "x-durable-workflow-protocol-version" } else { "x-durable-workflow-control-plane-version" }, + if worker { "1.20" } else { "2" }, + ) + .body(body) + .unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 8 * 1024 * 1024).await.unwrap(); + if status == StatusCode::NOT_IMPLEMENTED + && String::from_utf8_lossy(&bytes).contains("rust_capability_not_implemented") + { + unserved.push(format!("{method} /{uri}")); + } + } + } + assert!(checked > 170, "the published route table lists {checked} API route methods"); + let recorded: Vec<&str> = UNSERVED.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect(); + let now_served: Vec<&&str> = recorded.iter().filter(|route| !unserved.iter().any(|found| found == **route)).collect(); + let now_unserved: Vec<&String> = unserved.iter().filter(|found| !recorded.contains(&found.as_str())).collect(); + assert!( + now_served.is_empty() && now_unserved.is_empty(), + "update rust/tests/fixtures/unserved-php-routes.v2.11.1.txt and the port log. Now served: {now_served:?}. Now unserved: {now_unserved:?}" + ); + assert_eq!(unserved, recorded, "keep the recorded list in route-table order"); + runtime.close().await; + database.remove().await; +} + +/// Replaces each `{parameter}` in a route with a placeholder value. +fn fill_parameters(uri: &str) -> String { + let mut path = String::new(); + let mut rest = uri; + while let Some(open) = rest.find('{') { + let close = open + rest[open..].find('}').expect("closed route parameter"); + path.push_str(&rest[..open]); + path.push_str("probe-x"); + rest = &rest[close + 1..]; + } + path.push_str(rest); + path +} diff --git a/rust/tests/execution/worker_history_presentation.rs b/rust/tests/execution/worker_history_presentation.rs index c1f0de28d..b96250e2b 100644 --- a/rust/tests/execution/worker_history_presentation.rs +++ b/rust/tests/execution/worker_history_presentation.rs @@ -63,6 +63,65 @@ async fn a_worker_reads_the_sequence_that_consumed_a_signal() { assert!(completed["payload"]["task"].get("last_claim_error").is_none(), "unset fields stay out: {completed}"); } +// `WorkflowHistoryEvent::record` with the task model: each event PHP records +// while handling a task carries that task's snapshot, and a fired timer +// carries its timer task, leased to itself. +#[tokio::test] +async fn events_recorded_with_a_task_carry_its_snapshot() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "snapshot-events").await; + register(&app, "snapshotter", json!(["echo", "child"]), json!([])).await; + let events = || run_history(&app, &started["workflow_id"], &started["run_id"]); + let snapshot_of = |history: &Value, kind: &str| -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == kind) + .map(|event| event["payload"]["task"].clone()).unwrap_or(Value::Null) + }; + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await.0, StatusCode::OK); + let scheduled = snapshot_of(&events().await, "TimerScheduled"); + assert_eq!((scheduled["id"].clone(), scheduled["type"].clone()), (task["task_id"].clone(), json!("workflow")), "{scheduled}"); + for field in ["queue", "connection", "available_at", "leased_at", "repair_count"] { + assert!(scheduled.get(field).is_some(), "{field}: {scheduled}"); + } + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(15); + let fired = loop { + let fired = snapshot_of(&events().await, "TimerFired"); + if !fired.is_null() { + break fired; + } + assert!(tokio::time::Instant::now() < deadline, "the timer never fired"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + }; + assert_eq!((fired["type"].clone(), fired["status"].clone(), fired["attempt_count"].clone()), + (json!("timer"), json!("leased"), json!(1)), "{fired}"); + assert_eq!(fired["lease_owner"], fired["id"], "{fired}"); + assert!(fired["lease_expires_at"].is_string() && fired["leased_at"].is_string(), "{fired}"); + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([ + {"type":"start_child_workflow","workflow_type":"child","arguments":envelope(Payload::Array(vec![]))}, + {"type":"open_signal_wait","signal_name":"go"}])).await.0, StatusCode::OK); + let history = events().await; + for kind in ["ChildWorkflowScheduled", "ChildRunStarted", "SignalWaitOpened"] { + assert_eq!(snapshot_of(&history, kind)["id"], task["task_id"], "{kind}: {history}"); + } + let (status, body) = request(&app, "POST", "/api/workflows/snapshot-events/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let mut task = poll(&app, "snapshotter", "workflow").await; + // The child's first task may come first on the shared queue. + if task["workflow_type"] == "child" { + assert_eq!(finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await.0, StatusCode::OK); + task = poll(&app, "snapshotter", "workflow").await; + } + assert_eq!(finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":"k", + "condition_definition_fingerprint":format!("sha256:{}", "c".repeat(64))}])).await.0, StatusCode::OK); + let history = events().await; + assert_eq!(snapshot_of(&history, "SignalApplied")["id"], task["task_id"], "{history}"); + assert_eq!(snapshot_of(&history, "ConditionWaitOpened")["id"], task["task_id"], "{history}"); +} + // Workflow 2.10.2 `DefaultWorkflowTaskBridge::applySignal` takes a workflow // sequence only from the signal's own wait. A signal the run's contract // declares is applied without one even while a condition wait is open, so diff --git a/rust/tests/execution/worker_listing.rs b/rust/tests/execution/worker_listing.rs index 034b47201..36aae24c2 100644 --- a/rust/tests/execution/worker_listing.rs +++ b/rust/tests/execution/worker_listing.rs @@ -332,11 +332,14 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { ); let (status, stale) = request(&app, "POST", &path, complete(&original)).await; assert_eq!(status, StatusCode::CONFLICT, "{stale}"); + let mut stale = stale; + assert!(stale.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( stale, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"replaced","activity_status":"running","attempt_status":"expired","task_status":"leased"}) + "lease_owner":"replaced","activity_status":"running","attempt_status":"expired","task_status":"leased", + "protocol_version":"1.20"}) ); let (status, done) = request(&app, "POST", &path, complete(&replacement)).await; assert_eq!(status, StatusCode::OK, "{done}"); @@ -345,11 +348,14 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { // result is still stale, as in PHP, not a conflicting duplicate. let (status, late) = request(&app, "POST", &path, complete(&original)).await; assert_eq!(status, StatusCode::CONFLICT, "{late}"); + let mut late = late; + assert!(late.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( late, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"replaced","activity_status":"completed","attempt_status":"expired","task_status":"completed"}) + "lease_owner":"replaced","activity_status":"completed","attempt_status":"expired","task_status":"completed", + "protocol_version":"1.20"}) ); let fail_path = format!( "/api/worker/activity-tasks/{}/fail", @@ -359,10 +365,12 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { json!({"activity_attempt_id":original["activity_attempt_id"],"lease_owner":"replaced", "failure":{"message":"late","type":"RuntimeException"}})).await; assert_eq!(status, StatusCode::CONFLICT, "{late_failure}"); + let mut late_failure = late_failure; + assert!(late_failure.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( late_failure, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null,"protocol_version":"1.20"}) ); // The replacement's own repeated result keeps its idempotent receipt. let (status, again) = request(&app, "POST", &path, complete(&replacement)).await; diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt new file mode 100644 index 000000000..c0a0082ea --- /dev/null +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -0,0 +1,48 @@ +# Published Server 2.11.1 API routes that native answers with 501 +# rust_capability_not_implemented, in route-table order. Checked by +# rust/tests/execution/route_coverage.rs. +POST /api/bridge-adapters/webhook/{adapter} +GET /api/service-endpoints +POST /api/service-endpoints +GET /api/service-endpoints/{endpointName} +PUT /api/service-endpoints/{endpointName} +DELETE /api/service-endpoints/{endpointName} +GET /api/service-endpoints/{endpointName}/services +POST /api/service-endpoints/{endpointName}/services +GET /api/service-endpoints/{endpointName}/services/{serviceName} +PUT /api/service-endpoints/{endpointName}/services/{serviceName} +DELETE /api/service-endpoints/{endpointName}/services/{serviceName} +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +PUT /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +DELETE /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/execute +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId} +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}/cancel +GET /api/worker-sessions +GET /api/worker-sessions/{sessionId} +POST /api/worker/sessions +DELETE /api/worker/sessions/{sessionId} +POST /api/worker/sessions/{sessionId}/heartbeat +POST /api/worker/update-validation-tasks/poll +POST /api/worker/update-validation-tasks/{taskId}/approve +POST /api/worker/update-validation-tasks/{taskId}/reject +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/deliver +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/open +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/prepare +POST /api/worker/workflow-tasks/{taskId}/local-activities/checkpoint +POST /api/worker/workflow-tasks/{taskId}/local-activities/checkpoint-group +POST /api/worker/workflow-tasks/{taskId}/local-activities/prepare +POST /api/worker/workflow-tasks/{taskId}/local-activities/recover +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome +POST /api/workflows/import/embedded-v2 +POST /api/workflows/import/waterline-v1 +GET /api/workflows/{workflowId}/nexus-operations +GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations +POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close +POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs new file mode 100644 index 000000000..c40e89805 --- /dev/null +++ b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs @@ -0,0 +1,281 @@ +import assert from 'node:assert/strict'; + +const MICROS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; +// Identities are opaque ULIDs; their case differs between servers, as the +// other contracts allow. +const ULID = /^[0-9A-Za-z]{26}$/; +const REQUEST = ULID; +const ACKS = ['before_request', 'before_delivery', 'foreign_owner', 'unknown_attempt', 'unknown_task', 'task_mismatch', + 'other_namespace', 'protocol_1_19', 'protocol_missing', 'protocol_1_21', 'missing_fields', 'non_string_fields', + 'long_request_id', 'wrong_request_id', 'acknowledged', 'duplicate', 'legacy_ack', 'immediate_ack', + 'stale_first_attempt_own_task', 'stale_first_attempt_new_task', 'stale_second_attempt_old_task', 'stale_second_attempt', + 'terminated_ack']; +const MAIN_EVENTS = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', + 'CooperativeCancellationRequested', 'ActivityCancelled', 'CooperativeCancellationDelivered']; +const BOUNDARY_EVENTS = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', + 'TimerScheduled', 'CooperativeCancellationRequested']; + +// Milliseconds since the epoch of a UTC timestamp with any fraction length. +function at(value, label) { + assert.ok(typeof value === 'string', `${label}: a timestamp, got ${JSON.stringify(value)}`); + const millis = Date.parse(value.replace(/\.(\d{3})\d*Z$/, '.$1Z')); + assert.ok(!Number.isNaN(millis), `${label}: ${value}`); + return millis; +} + +// Reviewed activity cancellation acknowledgement +// (ActivityTaskController::acknowledgeCancellation over Workflow's +// ActivityCancellationAcknowledgement::recordStopped), recorded against +// published Server 2.11.1. Identities and times come from the observation; +// their relations and every other field are exact. +// +// One documented reference boundary: published 2.11.1 delivers a cooperative +// cancellation of a retried activity to the execution's first task, not the +// live attempt's task (Workflow #923), so that attempt's acknowledgement is +// refused. The retried-attempt case accepts either the correct outcome or +// exactly that defect's signature, and nothing else; native's own execution +// tests require the correct outcome. +export function checkActivityCancellationAcknowledgement(fixture, observation, workflowId) { + const definition = fixture.activity_cancellation_acknowledgement; + const state = observation.activity_cancellation_acknowledgement; + const projection = {scope: 'activity_cancellation_acknowledgement', cleanup_timeout_seconds: definition.cleanup_timeout_seconds}; + assert.ok(state && typeof state === 'object', 'actual cancellation acknowledgement observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded cancellation acknowledgement explicitly inapplicable'); + return projection; + } + const equal = (actual, expected, label) => assert.deepStrictEqual(actual, expected, label); + const main = state.main; + equal([state.applicable, state.namespace, main.workflow_id], [true, `${workflowId}-cack`.toLowerCase(), `${workflowId}-main`], + 'acknowledgement namespace and run'); + equal(state.discovery, {activity_cancellation_acknowledgement: true, cooperative_cancellation: true}, + 'cluster info advertises cooperative cancellation and its acknowledgement'); + for (const key of ACKS) { + equal([state[key]?.response?.protocol_version, state[key]?.acknowledgement_capability], ['1.20', true], + `${key} answers in worker protocol 1.20 and advertises acknowledgement`); + } + + // A cancellation request on the main run, read back from its workflow task. + equal(state.request.status, 202, 'cancellation requested'); + const {cancellation_request: request, ...requested} = state.request.response; + equal(requested, {workflow_id: main.workflow_id, run_id: main.run_id, accepted: true, duplicate: false, run_status: 'waiting', + reason: null}, 'cancellation request receipt'); + assert.match(request.request_id, REQUEST, 'request id'); + assert.match(request.requested_at, MICROS, 'request time in UTC microseconds'); + equal(at(request.cleanup_deadline_at, 'cleanup deadline') - at(request.requested_at, 'request time'), + definition.cleanup_timeout_seconds * 1000, 'the cleanup budget runs from the request'); + equal([request.delivery_sequence, request.delivered_at, request.history_refresh_page_token], [null, null, 'MA=='], + 'request not yet delivered'); + equal(state.poll_cancellation_request, request, 'the workflow task carries the request'); + const requestId = request.request_id; + + const ack = (claim, requestValue, fields) => ({task_id: claim.task_id, activity_attempt_id: claim.activity_attempt_id, + lease_owner: claim.lease_owner, request_id: requestValue, acknowledged: false, duplicate: false, reason: null, + history_event_id: null, heartbeat_recorded: false, ...fields, protocol_version: '1.20'}); + const refused = (key, claim, requestValue, reason) => equal(state[key], + {status: 409, response: ack(claim, requestValue, {reason}), acknowledgement_capability: true}, key); + refused('before_request', main, 'not-yet-requested', 'cancellation_request_mismatch'); + refused('before_delivery', main, requestId, 'activity_cancellation_not_recorded'); + + // Workflow's attempt status for a claim, before any field is overridden. + const status = (claim, fields = {}) => ({task_id: claim.task_id, activity_attempt_id: claim.activity_attempt_id, + lease_owner: claim.lease_owner, can_continue: true, cancel_requested: false, reason: null, heartbeat_recorded: false, + last_heartbeat_at: null, run_status: 'waiting', run_closed_reason: null, run_closed_at: null, activity_status: 'running', + attempt_status: 'running', task_status: 'leased', worker_session: null, ...fields, protocol_version: '1.20'}); + const withoutTimes = (key, recorded) => { + equal(recorded.status, 200, `${key} answers`); + const {lease_expires_at: lease, deadlines, ...body} = recorded.response; + equal(Object.keys(deadlines), ['start_to_close'], `${key} deadlines`); + assert.match(deadlines.start_to_close, MICROS, `${key} start-to-close deadline`); + return {lease, deadline: deadlines.start_to_close, body}; + }; + const pending = withoutTimes('status_before_delivery', state.status_before_delivery); + assert.match(pending.lease, MICROS, 'a pending request leaves the lease'); + equal(pending.body, status(main), 'a pending request does not stop the attempt'); + + // Delivery to the activity cancels its attempt; the worker learns it from + // its heartbeat and finds the request on its status. + const deliveryBody = (claim, requestValue) => ({delivered: true, task_id: state.delivery.response.task_id, + workflow_run_id: claim.run_id, request_id: requestValue, sequence: 1, call_kind: 'activity', sequence_span: 1, + operation_sequence: null, operation_sequence_span: 1, reason: null, protocol_version: '1.20'}); + equal(state.delivery, {status: 200, response: deliveryBody(main, requestId), acknowledgement_capability: true}, 'delivered'); + assert.match(state.delivery.response.task_id, ULID, 'delivered from the cancellation workflow task'); + equal(state.heartbeat_after_delivery, {status: 200, acknowledgement_capability: true, response: {task_id: main.task_id, + activity_attempt_id: main.activity_attempt_id, lease_owner: main.lease_owner, cancel_requested: true, can_continue: false, + reason: 'attempt_closed', run_closed_reason: null, run_closed_at: null, heartbeat_recorded: false, lease_expires_at: null, + last_heartbeat_at: null, worker_session: null, protocol_version: '1.20'}}, 'heartbeat reports the stop'); + const stopped = (claim) => status(claim, {can_continue: false, cancel_requested: true, reason: 'attempt_closed', + activity_status: 'cancelled', attempt_status: 'cancelled', task_status: 'cancelled'}); + const receipt = (cancellationEvent, fields = {}) => ({request_id: requestId, root_request_id: requestId, + cleanup_deadline_at: request.cleanup_deadline_at, cancellation_history_event_id: cancellationEvent, callback_state: 'unknown', + history_event_id: null, acknowledged_at: null, received_after_deadline: null, ...fields}); + const unacknowledged = withoutTimes('status_before_ack', state.status_before_ack); + equal([unacknowledged.lease, unacknowledged.deadline], [null, pending.deadline], 'delivery ends the lease, not the deadline'); + const {cancellation_acknowledgement: before, ...beforeBody} = unacknowledged.body; + const cancellationEvent = before?.cancellation_history_event_id; + assert.match(cancellationEvent ?? '', ULID, 'the receipt names the ActivityCancelled event'); + equal(beforeBody, stopped(main), 'the delivered attempt is stopped'); + equal(before, receipt(cancellationEvent), 'the receipt before acknowledgement'); + equal(state.history_before_ack.map(event => event.event_type), MAIN_EVENTS, 'request and delivery history'); + + // Ownership in PHP's order and shape, protocol, then Laravel's validation. + // None records anything. + const fault = (key, code, task, attempt, error, reason, extra = {}) => equal(state[key], {status: code, + acknowledgement_capability: true, response: {task_id: task, activity_attempt_id: attempt, error, reason, ...extra, + protocol_version: '1.20'}}, key); + fault('foreign_owner', 409, main.task_id, main.activity_attempt_id, 'Activity attempt lease is owned by another worker.', + 'lease_owner_mismatch', {lease_owner: main.lease_owner}); + fault('unknown_attempt', 404, main.task_id, 'no-such-attempt', 'Activity attempt not found.', 'attempt_not_found'); + fault('unknown_task', 404, 'no-such-task', main.activity_attempt_id, 'Activity task not found.', 'task_not_found'); + fault('task_mismatch', 409, main.workflow_task_id, main.activity_attempt_id, 'Activity attempt is not leased for this task.', + 'task_mismatch'); + fault('other_namespace', 404, main.task_id, main.activity_attempt_id, 'Activity task not found.', 'task_not_found'); + equal(state.protocol_1_19, {status: 422, acknowledgement_capability: true, response: {acknowledged: false, + reason: 'activity_cancellation_acknowledgement_not_supported', minimum_protocol_version: '1.20', + unavailable: ['worker_protocol'], protocol_version: '1.20'}}, 'a 1.19 request'); + for (const key of ['protocol_missing', 'protocol_1_21', 'missing_fields', 'non_string_fields', 'long_request_id']) { + const {status: code, ...body} = definition.refusals[key]; + equal(state[key], {status: code, acknowledgement_capability: true, response: {...body, protocol_version: '1.20'}}, key); + } + refused('wrong_request_id', main, 'other-request', 'cancellation_request_mismatch'); + equal(state.history_after_refusals, state.history_before_ack, 'refusals record nothing'); + + // The acknowledgement: one event, then the receipt reads stopped and a + // repeat returns the same event. + const acknowledgedEvent = state.acknowledged?.response?.history_event_id; + assert.match(acknowledgedEvent ?? '', ULID, 'acknowledgement event id'); + equal(state.acknowledged, {status: 200, acknowledgement_capability: true, + response: ack(main, requestId, {acknowledged: true, history_event_id: acknowledgedEvent})}, 'acknowledged'); + const acknowledgedStatus = withoutTimes('status_after_ack', state.status_after_ack); + const {cancellation_acknowledgement: after, ...afterBody} = acknowledgedStatus.body; + equal([acknowledgedStatus.lease, acknowledgedStatus.deadline, afterBody], [null, pending.deadline, stopped(main)], + 'acknowledgement changes no attempt state'); + assert.match(after?.acknowledged_at ?? '', MICROS, 'acknowledgement time in UTC microseconds'); + equal(after, receipt(cancellationEvent, {callback_state: 'stopped', history_event_id: acknowledgedEvent, + acknowledged_at: after.acknowledged_at, received_after_deadline: false}), 'the receipt after acknowledgement'); + assert.ok(at(after.acknowledged_at, 'acknowledged') < at(request.cleanup_deadline_at, 'deadline'), 'acknowledged within the budget'); + equal(state.duplicate, {status: 200, acknowledgement_capability: true, + response: ack(main, requestId, {acknowledged: true, duplicate: true, history_event_id: acknowledgedEvent})}, 'duplicate'); + equal(state.completed_after_ack, {status: 409, acknowledgement_capability: true, response: {task_id: main.task_id, + activity_attempt_id: main.activity_attempt_id, outcome: 'completed', recorded: false, reason: 'stale_attempt', + next_task_id: null, lease_owner: main.lease_owner, activity_status: 'cancelled', attempt_status: 'cancelled', + task_status: 'cancelled', protocol_version: '1.20'}}, 'a later outcome is refused'); + const history = state.history_after_ack; + equal(history.slice(0, -1), state.history_before_ack, 'acknowledgement only appends'); + const event = history.at(-1); + const delivered = history.at(-2); + const {task, ...payload} = event.payload; + equal([event.event_type, event.sequence, event.principal], ['ActivityCancellationAcknowledged', delivered.sequence + 1, null], + 'one acknowledgement event'); + equal(payload, {sequence: 1, activity_execution_id: main.activity_execution_id, activity_attempt_id: main.activity_attempt_id, + lease_owner: main.lease_owner, cancellation_history_event_id: cancellationEvent, request_id: requestId, + root_request_id: requestId, cleanup_deadline_at: request.cleanup_deadline_at, callback_state: 'stopped', + evidence_source: 'activity_worker', acknowledged_at: after.acknowledged_at, received_after_deadline: false}, + 'acknowledgement payload'); + const {available_at: available, leased_at: leased, connection, ...snapshot} = task; + assert.match(available, MICROS, 'snapshot availability'); + assert.match(leased, MICROS, 'snapshot lease time'); + const cancelledTask = history.find(item => item.event_type === 'ActivityCancelled')?.payload?.task; + equal([cancelledTask?.id, cancelledTask?.status], [main.task_id, 'cancelled'], 'delivery snapshots the cancelled activity task'); + assert.ok(typeof connection === 'string' && connection !== '', 'snapshot connection'); + equal(connection, cancelledTask.connection, 'both activity task snapshots name one storage connection'); + equal(snapshot, {id: main.task_id, type: 'activity', status: 'cancelled', lease_owner: main.lease_owner, attempt_count: 1, + repair_count: 0, queue: main.queue}, 'acknowledgement snapshots the cancelled activity task, without a lease'); + + // A claim polled on protocol 1.19 cannot acknowledge. + const legacy = state.legacy_claim; + equal([state.legacy_request, state.legacy_delivery.status, state.legacy_delivery.response.delivered], [202, 200, true], + 'legacy claim cancelled cooperatively'); + equal(state.legacy_ack, {status: 409, acknowledgement_capability: true, response: {acknowledged: false, + reason: 'activity_cancellation_acknowledgement_not_supported', required_capability: 'cooperative_cancellation', + minimum_protocol_version: '1.20', worker_id: legacy.lease_owner, unavailable: ['worker_claim_capability'], + remediation: 'Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.', + protocol_version: '1.20'}}, 'a 1.19 claim cannot acknowledge'); + + // An immediate cancel records no cooperative request and no receipt. + const immediate = state.immediate_claim; + equal(state.immediate_cancel.status, 200, 'immediate cancel accepted'); + const commandId = state.immediate_cancel.response.command_id; + assert.match(commandId, ULID, 'immediate cancel command'); + refused('immediate_ack', immediate, commandId, 'cancellation_request_mismatch'); + const cancelled = withoutTimes('immediate_status', state.immediate_status); + assert.match(cancelled.body.run_closed_at ?? '', MICROS, 'immediate cancel closes the run'); + equal([cancelled.lease, cancelled.body], [null, status(immediate, {can_continue: false, cancel_requested: true, + reason: 'run_cancelled', run_status: 'cancelled', run_closed_reason: 'cancelled', run_closed_at: cancelled.body.run_closed_at, + activity_status: 'cancelled', attempt_status: 'cancelled', task_status: 'cancelled'})], 'immediate cancel status, no receipt'); + + // A retried attempt: the first attempt is fenced, and each attempt belongs + // to its own task. + const [first, second] = [state.stale_first, state.stale_second]; + equal([state.stale_fail.status, state.stale_fail.response.recorded, state.stale_request, state.stale_delivery.status, + state.stale_delivery.response.delivered], [200, true, 202, 200, true], 'retried attempt cancelled cooperatively'); + assert.notEqual(first.task_id, second.task_id, 'the retry has its own task'); + const staleRequest = state.stale_delivery.response.request_id; + refused('stale_first_attempt_own_task', first, staleRequest, 'activity_cancellation_acknowledgement_fence_mismatch'); + fault('stale_first_attempt_new_task', 409, second.task_id, first.activity_attempt_id, + 'Activity attempt is not leased for this task.', 'task_mismatch'); + fault('stale_second_attempt_old_task', 409, first.task_id, second.activity_attempt_id, + 'Activity attempt is not leased for this task.', 'task_mismatch'); + const staleCancelled = state.stale_history.find(item => item.event_type === 'ActivityCancelled'); + equal(staleCancelled?.payload?.activity_attempt?.id, second.activity_attempt_id, 'the live attempt is cancelled'); + const correct = staleCancelled.payload.task?.id === second.task_id; + if (correct) { + equal(state.stale_second_attempt.status, 200, 'the live attempt acknowledges'); + equal(state.stale_second_attempt.response.acknowledged, true, 'the live attempt acknowledges'); + } else { + // Workflow #923 in published 2.11.1: the delivery cancels the first, + // already completed task, so the live task stays leased and the + // acknowledgement is refused as not fenced. + equal(staleCancelled.payload.task?.id, first.task_id, 'reference defect: delivery snapshots the first task'); + equal([state.stale_second_status.response.attempt_status, state.stale_second_status.response.task_status], + ['cancelled', 'leased'], 'reference defect: the live task stays leased'); + refused('stale_second_attempt', second, staleRequest, 'activity_cancellation_not_fenced'); + } + + // A run terminated after delivery still records the late stop, after its + // terminal event. + const terminated = state.terminated_claim; + equal([state.terminated_request, state.terminated_delivery.status, state.terminate], [202, 200, 200], + 'terminated after delivery'); + const terminatedEvent = state.terminated_ack?.response?.history_event_id; + assert.match(terminatedEvent ?? '', ULID, 'terminated acknowledgement event id'); + equal(state.terminated_ack, {status: 200, acknowledgement_capability: true, response: ack(terminated, + state.terminated_delivery.response.request_id, {acknowledged: true, history_event_id: terminatedEvent})}, + 'a terminated run still acknowledges'); + equal(state.terminated_history, [...MAIN_EVENTS, 'TerminateRequested', 'WorkflowTerminated', 'ActivityCancellationAcknowledged'], + 'the late acknowledgement follows the terminal event'); + + // An activity delivery is checked against the run's typed history before + // anything is cancelled (CooperativeCancellationDelivery::validateCallBoundary). + // The activity at sequence 1 completed before the request and sequence 2 is + // a timer. Both refusals carry the bridge's undelivered body and record + // nothing, and the pending timer then takes the delivery. + const boundary = state.boundary_claim; + equal(boundary.workflow_id, `${workflowId}-boundary`, 'boundary run'); + equal([state.boundary_complete, state.boundary_timer, state.boundary_request], [200, 200, 202], + 'activity completed, timer started, then cancellation requested'); + assert.match(state.boundary_task_id ?? '', ULID, 'boundary cancellation workflow task'); + const undelivered = reason => ({status: 409, acknowledgement_capability: true, response: {delivered: false, + task_id: state.boundary_task_id, workflow_run_id: boundary.run_id, request_id: null, sequence: null, call_kind: null, + sequence_span: null, operation_sequence: null, operation_sequence_span: null, reason, protocol_version: '1.20'}}); + equal(state.boundary_resolved_activity, undelivered('cancellation_delivery_not_eligible'), + 'an activity that resolved before the request is not eligible'); + equal(state.boundary_timer_as_activity, undelivered('cancellation_delivery_shape_mismatch'), + 'an activity delivery at a timer is a shape mismatch'); + equal(state.boundary_history_before.map(event => event.event_type), BOUNDARY_EVENTS, 'boundary history before delivery'); + equal(state.boundary_history_after_refusals, state.boundary_history_before, 'boundary refusals record nothing'); + const pendingRequest = state.boundary_pending.response?.cancellation_request; + equal([state.boundary_pending.status, state.boundary_pending.response?.duplicate, pendingRequest?.delivery_sequence, + pendingRequest?.delivered_at], [200, true, null, null], 'the request is still undelivered after the refusals'); + assert.match(pendingRequest.request_id ?? '', REQUEST, 'boundary request id'); + equal(state.boundary_delivery, {status: 200, acknowledgement_capability: true, response: {delivered: true, + task_id: state.boundary_task_id, workflow_run_id: boundary.run_id, request_id: pendingRequest.request_id, sequence: 2, + call_kind: 'timer', sequence_span: 1, operation_sequence: null, operation_sequence_span: 1, reason: null, + protocol_version: '1.20'}}, 'the pending timer takes the delivery'); + const boundaryHistory = state.boundary_history_after_delivery; + equal(boundaryHistory.slice(0, -2), state.boundary_history_before, 'delivery only appends'); + equal(boundaryHistory.slice(-2).map(event => event.event_type), ['TimerCancelled', 'CooperativeCancellationDelivered'], + 'the timer is cancelled under the delivery'); + return projection; +} diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-probe.php b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php new file mode 100644 index 000000000..5cd84d983 --- /dev/null +++ b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php @@ -0,0 +1,222 @@ +createNamespace($namespace, 'activity cancellation acknowledgement λ', 30); + $admin->createNamespace($other, 'activity cancellation acknowledgement other λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new CancellationAckHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = [], ?string $protocol = '1.20', ?string $space = null) use ($http, $token, $namespace): array { + $options = ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $space ?? $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => $protocol, + ])]; + if ($body !== null) { + $options['json'] = $body === [] ? (object) [] : $body; + } + $response = $http->request($method, $path, $options); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + $capability = is_array($decoded) && array_key_exists('server_capabilities', $decoded) + ? ($decoded['server_capabilities']['activity_cancellation_acknowledgement'] ?? null) : 'absent'; + unset($decoded['server_capabilities'], $decoded['control_plane']); + + return ['status' => $response->getStatusCode(), 'response' => $decoded, 'acknowledgement_capability' => $capability]; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Activity cancellation acknowledgement '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + // Avro payloads precomputed by the published serializer: [] and 'done'. + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $done = ['codec' => 'avro', 'blob' => $definition['result_blob']]; + // Starts a run on its own queue with its own cooperative worker and claims + // its single activity, on the given poll protocol. + $claim = static function (string $suffix, array $retry = ['max_attempts' => 1, 'backoff_seconds' => []], string $activityProtocol = '1.20') use ($raw, $ok, $empty, $definition, $workflowId): array { + $id = $workflowId.'-'.$suffix; + $queue = $id.'-queue'; + $worker = $id.'-worker'; + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => ['cooperative_cancellation'], + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])]), 201, 'registration'); + $poll = ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 0]; + $started = $ok($raw('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => $definition['workflow_type'], + 'task_queue' => $queue, 'input' => $empty]), 201, 'start'); + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $poll), 200, 'workflow claim')['task']; + $ok($raw('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $empty, 'retry_policy' => $retry, 'start_to_close_timeout' => 300]]]), 200, 'schedule activity'); + $activity = $ok($raw('POST', '/api/worker/activity-tasks/poll', $poll, $activityProtocol), 200, 'activity claim')['task']; + + return ['workflow_id' => $id, 'run_id' => $started['run_id'], 'queue' => $queue, 'worker' => $worker, + 'workflow_task_id' => $task['task_id'], 'task_id' => $activity['task_id'], + 'activity_attempt_id' => $activity['activity_attempt_id'], + 'activity_execution_id' => $activity['activity_execution_id'] ?? null, 'lease_owner' => $activity['lease_owner']]; + }; + $own = static fn (array $claimed): array => ['activity_attempt_id' => $claimed['activity_attempt_id'], 'lease_owner' => $claimed['lease_owner']]; + $ack = static fn (array $claimed, ?array $body, ?string $protocol = '1.20', ?string $space = null, ?string $task = null): array => $raw('POST', + '/api/worker/activity-tasks/'.($task ?? $claimed['task_id']).'/acknowledge-cancellation', $body, $protocol, $space); + $status = static fn (array $claimed): array => $raw('POST', '/api/worker/activity-tasks/'.$claimed['task_id'].'/status', $own($claimed)); + $history = static fn (array $claimed): array => $ok($raw('GET', + '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/history', null), 200, 'history')['events'] ?? []; + // Requests cooperative cancellation of the selected run. + $request = static fn (array $claimed): array => $raw('POST', + '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/request-cancellation', + ['reason' => 'acknowledgement fixture λ', 'cleanup_timeout_seconds' => $definition['cleanup_timeout_seconds']]); + // Polls the run's cancellation workflow task and delivers the request to + // the activity, the first command of the run. + $deliver = static function (array $claimed) use ($raw, $ok): array { + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', + ['worker_id' => $claimed['worker'], 'task_queue' => $claimed['queue'], 'timeout_seconds' => 0]), 200, 'cancellation claim')['task']; + $requestId = $task['cancellation_request']['request_id'] ?? null; + + return ['request_id' => $requestId, 'poll_cancellation_request' => $task['cancellation_request'] ?? null, + 'delivery' => $raw('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/deliver-cancellation', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => 'activity'])]; + }; + + $state = ['applicable' => true, 'namespace' => $namespace]; + $cluster = $ok($raw('GET', '/api/cluster/info', null), 200, 'cluster info'); + $capabilities = $cluster['worker_protocol']['server_capabilities'] ?? []; + $state['discovery'] = ['activity_cancellation_acknowledgement' => $capabilities['activity_cancellation_acknowledgement'] ?? 'absent', + 'cooperative_cancellation' => $capabilities['cooperative_cancellation'] ?? 'absent']; + + $main = $state['main'] = $claim('main'); + $state['before_request'] = $ack($main, $own($main) + ['request_id' => 'not-yet-requested']); + $state['request'] = $request($main); + $requestId = $state['request']['response']['cancellation_request']['request_id'] ?? null; + $owned = $own($main) + ['request_id' => $requestId]; + $state['before_delivery'] = $ack($main, $owned); + $state['status_before_delivery'] = $status($main); + $delivered = $deliver($main); + $state['poll_cancellation_request'] = $delivered['poll_cancellation_request']; + $state['delivery'] = $delivered['delivery']; + $state['heartbeat_after_delivery'] = $raw('POST', '/api/worker/activity-tasks/'.$main['task_id'].'/heartbeat', $own($main)); + $state['status_before_ack'] = $status($main); + $state['history_before_ack'] = $history($main); + $state['foreign_owner'] = $ack($main, ['lease_owner' => 'someone-else'] + $owned); + $state['unknown_attempt'] = $ack($main, ['activity_attempt_id' => 'no-such-attempt'] + $owned); + $state['unknown_task'] = $ack($main, $owned, '1.20', null, 'no-such-task'); + $state['task_mismatch'] = $ack($main, $owned, '1.20', null, $main['workflow_task_id']); + $state['other_namespace'] = $ack($main, $owned, '1.20', $other); + $state['protocol_1_19'] = $ack($main, $owned, '1.19'); + $state['protocol_missing'] = $ack($main, $owned, null); + $state['protocol_1_21'] = $ack($main, $owned, '1.21'); + $state['missing_fields'] = $ack($main, []); + $state['non_string_fields'] = $ack($main, ['activity_attempt_id' => 5, 'lease_owner' => ['x'], 'request_id' => true]); + $state['long_request_id'] = $ack($main, $own($main) + ['request_id' => str_repeat('r', 256)]); + $state['wrong_request_id'] = $ack($main, $own($main) + ['request_id' => 'other-request']); + $state['history_after_refusals'] = $history($main); + $state['acknowledged'] = $ack($main, $owned + ['message' => 'stopped']); + $state['status_after_ack'] = $status($main); + $state['duplicate'] = $ack($main, $owned); + $state['completed_after_ack'] = $raw('POST', '/api/worker/activity-tasks/'.$main['task_id'].'/complete', $own($main) + ['result' => $done]); + $state['history_after_ack'] = $history($main); + + // The activity was claimed on protocol 1.19, so its claim cannot + // acknowledge even on a 1.20 request. + $legacy = $state['legacy_claim'] = $claim('legacy', activityProtocol: '1.19'); + $state['legacy_request'] = $request($legacy)['status']; + $legacyDelivered = $deliver($legacy); + $state['legacy_delivery'] = $legacyDelivered['delivery']; + $state['legacy_ack'] = $ack($legacy, $own($legacy) + ['request_id' => $legacyDelivered['request_id']]); + + // An immediate cancel records no cooperative request. + $immediate = $state['immediate_claim'] = $claim('immediate'); + $state['immediate_cancel'] = $raw('POST', '/api/workflows/'.$immediate['workflow_id'].'/cancel', ['reason' => 'acknowledgement fixture']); + $state['immediate_ack'] = $ack($immediate, $own($immediate) + + ['request_id' => $state['immediate_cancel']['response']['command_id'] ?? 'no-command-id']); + $state['immediate_status'] = $status($immediate); + + // A retried attempt: attempt 1 fails retryably and attempt 2 is claimed on + // a new task before the request is delivered. + $first = $state['stale_first'] = $claim('stale', ['max_attempts' => 2, 'backoff_seconds' => [0]]); + $state['stale_fail'] = $raw('POST', '/api/worker/activity-tasks/'.$first['task_id'].'/fail', $own($first) + + ['failure' => ['message' => 'retry me', 'type' => 'parity.retryable', 'non_retryable' => false]]); + $retried = $ok($raw('POST', '/api/worker/activity-tasks/poll', + ['worker_id' => $first['worker'], 'task_queue' => $first['queue'], 'timeout_seconds' => 0]), 200, 'retry claim')['task']; + $second = $state['stale_second'] = ['task_id' => $retried['task_id'], 'activity_attempt_id' => $retried['activity_attempt_id'], + 'lease_owner' => $retried['lease_owner']] + $first; + $state['stale_request'] = $request($first)['status']; + $staleDelivered = $deliver($first); + $state['stale_delivery'] = $staleDelivered['delivery']; + $staleRequest = ['request_id' => $staleDelivered['request_id']]; + $state['stale_first_attempt_own_task'] = $ack($first, $own($first) + $staleRequest); + $state['stale_first_attempt_new_task'] = $ack($first, $own($first) + $staleRequest, '1.20', null, $second['task_id']); + $state['stale_second_attempt_old_task'] = $ack($second, $own($second) + $staleRequest, '1.20', null, $first['task_id']); + $state['stale_second_attempt'] = $ack($second, $own($second) + $staleRequest); + $state['stale_second_status'] = $status($second); + $state['stale_history'] = $history($first); + + // Terminated after delivery, then acknowledged. + $terminated = $state['terminated_claim'] = $claim('terminated'); + $state['terminated_request'] = $request($terminated)['status']; + $terminatedDelivered = $deliver($terminated); + $state['terminated_delivery'] = $terminatedDelivered['delivery']; + $state['terminate'] = $raw('POST', '/api/workflows/'.$terminated['workflow_id'].'/terminate', ['reason' => 'acknowledgement fixture'])['status']; + $state['terminated_ack'] = $ack($terminated, $own($terminated) + ['request_id' => $terminatedDelivered['request_id']]); + $state['terminated_history'] = array_column($history($terminated), 'event_type'); + + // An activity delivery is checked against typed history first: the + // activity completed before the request, then a timer was started at + // sequence 2. Delivering to the activity, or as an activity at the + // timer, is refused and records nothing; the timer then takes it. + $boundary = $state['boundary_claim'] = $claim('boundary'); + $boundaryPoll = ['worker_id' => $boundary['worker'], 'task_queue' => $boundary['queue'], 'timeout_seconds' => 0]; + $state['boundary_complete'] = $raw('POST', '/api/worker/activity-tasks/'.$boundary['task_id'].'/complete', + $own($boundary) + ['result' => $done])['status']; + $resumed = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $boundaryPoll), 200, 'boundary resume')['task']; + $state['boundary_timer'] = $raw('POST', '/api/worker/workflow-tasks/'.$resumed['task_id'].'/complete', [ + 'lease_owner' => $resumed['lease_owner'], 'workflow_task_attempt' => $resumed['workflow_task_attempt'], + 'commands' => [['type' => 'start_timer', 'delay_seconds' => 3600]]])['status']; + $state['boundary_request'] = $request($boundary)['status']; + $cancelling = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $boundaryPoll), 200, 'boundary cancellation claim')['task']; + $state['boundary_task_id'] = $cancelling['task_id']; + $deliverAt = static fn (int $sequence, string $kind): array => $raw('POST', + '/api/worker/workflow-tasks/'.$cancelling['task_id'].'/deliver-cancellation', [ + 'lease_owner' => $cancelling['lease_owner'], 'workflow_task_attempt' => $cancelling['workflow_task_attempt'], + 'request_id' => $cancelling['cancellation_request']['request_id'] ?? null, 'sequence' => $sequence, + 'call_kind' => $kind]); + $state['boundary_history_before'] = $history($boundary); + $state['boundary_resolved_activity'] = $deliverAt(1, 'activity'); + $state['boundary_timer_as_activity'] = $deliverAt(2, 'activity'); + $state['boundary_history_after_refusals'] = $history($boundary); + $state['boundary_pending'] = $request($boundary); + $state['boundary_delivery'] = $deliverAt(2, 'timer'); + $state['boundary_history_after_delivery'] = $history($boundary); + + return $state; +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 69b947698..a08e8d3f4 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -13,6 +13,7 @@ import {checkLateWorkflowCompletion} from './late-workflow-completion-contract.m import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; +import {checkActivityCancellationAcknowledgement} from './activity-cancellation-ack-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; import {checkOperatorDashboardTypes} from './operator-dashboard-types-contract.mjs'; @@ -21,6 +22,7 @@ import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; import {checkClusterInfo} from './cluster-info-contract.mjs'; import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; +import {checkMessageStreams} from './message-streams-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -151,6 +153,8 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const searchAttributes = fixture.search_attributes ? checkSearchAttributes(fixture, observation, workflowId) : null; const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; + const activityCancellationAcknowledgement = fixture.activity_cancellation_acknowledgement + ? checkActivityCancellationAcknowledgement(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; const operatorDashboardTypes = fixture.operator_dashboard_types ? checkOperatorDashboardTypes(fixture, observation, workflowId) : null; @@ -159,6 +163,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const clusterInfo = fixture.cluster_info ? checkClusterInfo(fixture, observation, workflowId) : null; const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; + const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -706,6 +711,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(searchAttributes ? {search_attributes: searchAttributes} : {}), ...(storageTest ? {storage_test: storageTest} : {}), ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), + ...(activityCancellationAcknowledgement ? {activity_cancellation_acknowledgement: activityCancellationAcknowledgement} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), ...(operatorDashboardTypes ? {operator_dashboard_types: operatorDashboardTypes} : {}), @@ -714,6 +720,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(clusterInfo ? {cluster_info: clusterInfo} : {}), ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), + ...(messageStreams ? {message_streams: messageStreams} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/message-streams-contract.mjs b/scripts/conformance/server-parity/message-streams-contract.mjs new file mode 100644 index 000000000..829508c2a --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-contract.mjs @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; + +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/; +const MESSAGE = 'durable-workflow.v2.message-stream.message'; +const CURSOR = 'durable-workflow.v2.message-stream.cursor'; + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + assert.equal(read.status, 422, `${label} status`); + assert.deepStrictEqual(read.response, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, `${label} body`); +} + +// One stream's diagnostics: exact counts, the pending wait by run and a +// microsecond UTC time, and the last input. +function checkStream(entry, expected, runId, label) { + const {pending_wait: wait, last_input_at: inputAt, ...rest} = entry; + const {wait: expectedWait = null, input = null, ...counts} = expected; + assert.deepStrictEqual(rest, {...counts, pending_count: counts.last_position - counts.cursor_position, + duplicate_count: counts.duplicate_count ?? 0, malformed_count: counts.malformed_count ?? 0, + last_input_outcome: input?.outcome ?? null, last_input_message_id: input?.message_id ?? null}, `${label} counts`); + if (expectedWait === null) { + assert.equal(wait, null, `${label} has no pending wait`); + } else { + assert.equal(wait?.run_id, runId, `${label} pending wait run`); + assert.equal(wait?.after_position, expectedWait.after_position, `${label} pending wait position`); + assert.match(wait?.since ?? '', TIME, `${label} pending wait time`); + } + if (input === null) { + assert.equal(inputAt, null, `${label} has no input yet`); + } else { + assert.match(inputAt ?? '', TIME, `${label} last input time`); + } +} + +function checkList(read, workflowId, streams, runId, label) { + assert.equal(read.status, 200, `${label} status`); + assert.equal(read.response.workflow_id, workflowId, `${label} workflow`); + assert.equal(read.response.count, Object.keys(streams).length, `${label} count`); + assert.deepStrictEqual(read.response.streams.map(entry => entry.stream_name), Object.keys(streams), `${label} streams by name`); + for (const entry of read.response.streams) { + checkStream(entry, {stream_name: entry.stream_name, ...streams[entry.stream_name]}, runId, `${label} ${entry.stream_name}`); + } +} + +function accepted(read, workflowId, runId, stream, messageId, position, cursor, label) { + assert.equal(read.status, 202, `${label} status`); + assert.deepStrictEqual(read.response, {accepted: true, duplicate: false, workflow_id: workflowId, run_id: runId, + stream_name: stream, message_id: messageId, position, cursor_position: cursor, outcome: 'accepted', reason: null}, `${label} body`); +} + +function completed(read, runId, status, created, label) { + assert.equal(read.status, 200, `${label} status`); + const {task_id: taskId, created_task_ids: createdIds, ...rest} = read.response; + assert.equal(typeof taskId, 'string', `${label} task`); + assert.deepStrictEqual(rest, {workflow_task_attempt: 1, outcome: 'completed', recorded: true, run_id: runId, + run_status: status, reason: null}, `${label} outcome`); + assert.equal(createdIds.length, created, `${label} created tasks, the next message task included`); +} + +// A message delivery or cursor checkpoint as the worker reads it. +function delivery(stream, messageId, position, payload) { + return [{schema: MESSAGE, stream_name: stream, message_id: messageId, position, payload_envelope: ['blob', 'codec'], payload}]; +} + +// Reviewed message streams (published Server 2.11.1). +export function checkMessageStreams(fixture, observation, workflowId) { + const definition = fixture.message_streams; + const state = observation.message_streams; + // The same for every target: embedded applications have no HTTP message streams. + const projection = {scope: 'message_streams', first_run_events: definition.first_run_events.length, + continued_run_events: definition.continued_run_events.length}; + assert.ok(state && typeof state === 'object', 'actual message stream observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_message_streams'}, + 'embedded message streams explicitly inapplicable'); + return projection; + } + const id = `${workflowId}-ms`; + const {stream, other_stream: other, refusals: text, messages} = definition; + const run = state.run_id; + assert.equal(state.workflow_id, id, 'stream workflow'); + assert.deepStrictEqual(state.cluster_contract, definition.manifest, 'cluster info publishes the contract'); + assert.deepStrictEqual(state.worker_capability, {...definition.manifest, supported: true}, 'workers see the supported contract'); + + assert.deepStrictEqual(state.list_before, {status: 200, response: {workflow_id: id, count: 0, streams: []}}, 'no streams yet'); + assert.deepStrictEqual(state.show_before, {status: 404, response: {workflow_id: id, stream_name: stream, reason: 'message_stream_not_found'}}, + 'an unknown stream'); + refused(state.show_bad_name, 'stream_name', text.stream_name, 'a stream name outside the pattern'); + assert.deepStrictEqual(state.list_unknown, {status: 404, response: {message: 'Workflow not found.', reason: 'instance_not_found'}}, + 'an unknown workflow'); + assert.deepStrictEqual(state.poll1.history, [], 'nothing delivered before the first append'); + + const first = state.complete1_refusals; + refused(first.wait_ahead, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait ahead of the cursor'); + refused(first.cursor_ahead, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(first.bad_name, 'message_stream_waits.0.stream_name', text.metadata_stream_name, 'a wait stream name outside the pattern'); + refused(first.repeated_wait, 'message_stream_waits.1.stream_name', text.duplicate_wait, 'two waits on one stream'); + assert.equal(first.old_protocol.status, 409, 'stream metadata before protocol 1.15'); + const {task_id: oldTask, ...oldProtocol} = first.old_protocol.response; + assert.equal(typeof oldTask, 'string', 'the refused task'); + assert.deepStrictEqual(oldProtocol, {workflow_task_attempt: 1, outcome: 'rejected', reason: 'message_streams_unavailable', + requested_version: '1.14', minimum_protocol_version: '1.15', remediation: text.old_protocol_remediation}, 'the protocol refusal'); + completed(state.complete1, run, 'waiting', 0, 'the first completion with a wait'); + checkList(state.list_waiting, id, {[stream]: {last_position: 0, cursor_position: 0, wait: {after_position: 0}}}, run, 'the waiting stream'); + + const appends = state.appends; + accepted(appends.m1, id, run, stream, 'm1', 1, 0, 'the first message'); + assert.deepStrictEqual(appends.m1_duplicate, {status: 200, response: {accepted: true, duplicate: true, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'duplicate', reason: null}}, 'the same message again'); + assert.deepStrictEqual(appends.m1_conflict, {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'rejected', reason: 'message_identity_conflict'}}, 'the id with another payload'); + assert.deepStrictEqual(appends.m2_sdk.response, {accepted: true, duplicate: false, workflow_id: id, run_id: run, stream_name: stream, + message_id: 'm2', position: 2, cursor_position: 0, outcome: 'accepted', reason: null}, 'the published SDK append'); + refused(appends.no_id, 'message_id', text.message_id_required, 'no message id'); + refused(appends.bad_id, 'message_id', text.message_id_format, 'a message id outside the pattern'); + refused(appends.long_id, 'message_id', text.message_id_max, 'a message id too long'); + refused(appends.bad_input, 'input', text.input_array, 'input that is not a list'); + refused(appends.bad_stream, 'stream_name', text.stream_name, 'an append to a stream name outside the pattern'); + assert.deepStrictEqual(appends.unknown_workflow, {status: 404, response: {accepted: false, workflow_id: `${id}-missing`, + stream_name: stream, outcome: 'rejected', reason: 'instance_not_found'}}, 'an append to an unknown workflow'); + accepted(appends.other_stream, id, run, other, 'a1', 1, 0, 'a message without input on another stream'); + assert.equal(appends.reserved_signal.status, 422, 'the runtime signal is not a public signal'); + assert.equal(appends.reserved_signal.response.reason, 'runtime_reserved_signal', 'the reserved signal refusal'); + // Four refused appends to the stream are malformed; the last kept the + // valid id m3. The refused stream name counts nowhere. + const appended = { + [other]: {last_position: 1, cursor_position: 0, input: {outcome: 'accepted', message_id: 'a1'}}, + [stream]: {last_position: 2, cursor_position: 0, duplicate_count: 2, malformed_count: 4, input: {outcome: 'malformed', message_id: 'm3'}}, + }; + checkList(state.list_after_appends, id, appended, run, 'after the appends'); + assert.equal(state.show_after_appends.status, 200, 'one stream'); + assert.deepStrictEqual(state.show_after_appends.response, {workflow_id: id, + stream: state.list_after_appends.response.streams.find(entry => entry.stream_name === stream)}, 'one stream as listed'); + + assert.equal(state.poll2.run_id, run, 'the woken run'); + assert.deepStrictEqual(state.poll2.history, [ + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm1', 1, messages.m1)}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm2', 2, messages.m2)}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + ], 'each message is one runtime signal, in append order'); + const second = state.complete2_refusals; + refused(second.beyond, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(second.negative, 'message_stream_cursors.0.through_position', text.negative_cursor, 'a negative cursor'); + refused(second.wait_behind, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait behind the new cursor'); + // Two messages are still unconsumed, so the next message task follows. + completed(state.complete2, run, 'waiting', 1, 'the first acknowledgement'); + assert.deepStrictEqual(state.history_after2.filter(event => event.event_type === 'MessageCursorAdvanced'), [ + {event_type: 'MessageCursorAdvanced', stream_key: 'instance', previous_position: 0, new_position: 1, task_status: 'leased'}, + {event_type: 'MessageCursorAdvanced', stream_key: stream, previous_position: 0, new_position: 1, task_status: 'completed'}, + ], 'the instance cursor while the task runs, then the stream cursor once it completed'); + checkList(state.list_after2, id, {...appended, [stream]: {...appended[stream], cursor_position: 1}}, run, 'after the acknowledgement'); + + accepted(appends.m3, id, run, stream, 'm3', 3, 1, 'the third message'); + assert.equal(state.poll3.run_id, run, 'the same run again'); + assert.deepStrictEqual(state.poll3.history.filter(event => event.event_type === 'SignalReceived').at(-1), + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, 'the third message delivered'); + completed(state.complete3, run, 'completed', 1, 'continue as new'); + checkList(state.list_after_continue, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 2, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'after continue as new'); + + const continued = state.continued_run_id; + assert.equal(state.poll4.run_id, continued, 'the continued run'); + assert.notEqual(continued, run, 'a new run'); + assert.deepStrictEqual(state.poll4.history, [ + {event_type: 'SignalReceived', deliveries: [{schema: CURSOR, stream_name: stream, through_position: 2}]}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, + ], 'the continued run receives the cursor, then every unconsumed message by append order'); + completed(state.complete4, continued, 'completed', 0, 'the continued run completes'); + checkList(state.list_final, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 3, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'the final streams'); + for (const [name, messageId] of [['closed', 'm4'], ['closed_duplicate', 'm1']]) { + assert.deepStrictEqual(appends[name], {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: messageId, position: null, outcome: 'rejected', reason: 'run_not_active'}}, `${name}: the instance has no open run`); + } + + assert.deepStrictEqual(state.first_run_events, definition.first_run_events, + 'the first run: deliveries consumed by the condition waits without SignalApplied, the rest rejected at the close'); + assert.deepStrictEqual(state.continued_run_events, definition.continued_run_events, 'the continued run'); + assert.deepStrictEqual(state.first_run_streams.filter(event => event.event_type === 'MessageCursorAdvanced').map(event => event.stream_key), + ['instance', stream, 'instance', stream], 'first run cursors'); + assert.ok(!JSON.stringify(state.list_final).includes('"n":'), 'diagnostics never carry payloads'); + for (const event of [...state.first_run_streams, ...state.continued_run_streams]) { + if (event.event_type === 'SignalReceived') { + assert.equal(event.deliveries.length, 1, 'one delivery per runtime signal'); + } + } + + return projection; +} diff --git a/scripts/conformance/server-parity/message-streams-probe.php b/scripts/conformance/server-parity/message-streams-probe.php new file mode 100644 index 000000000..bd1869b16 --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-probe.php @@ -0,0 +1,163 @@ +payloadCodec(); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new MessageStreamsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, ?string $worker = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + $worker !== null ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ?? '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $append = static fn (string $streamName, array $body): array => $raw('POST', + "workflows/{$id}/message-streams/".rawurlencode($streamName).'/messages', $body); + $complete = static fn (array $task, array $body, string $protocol = '1.20'): array => $raw('POST', + "worker/workflow-tasks/{$task['task_id']}/complete", + ['lease_owner' => $task['lease_owner'] ?? $worker, 'workflow_task_attempt' => $task['workflow_task_attempt']] + $body, $protocol); + $wait = static fn (int $cursor): array => ['type' => 'open_condition_wait', + 'condition_key' => "message-stream:{$stream}:{$cursor}", + 'condition_definition_fingerprint' => 'sha256:'.hash('sha256', $definition['condition_fingerprint_source'])]; + // A stream's deliveries and cursor moves in history, decoded. + $streamEvents = static function (array $events) use ($codec, $id): array { + $rows = []; + foreach ($events as $event) { + $payload = $event['payload'] ?? []; + if (($event['event_type'] ?? null) === 'SignalReceived' + && ($payload['signal_name'] ?? null) === '__durable_workflow_message_stream') { + $deliveries = $codec->decodeEnvelope($payload['arguments']); + foreach ($deliveries as &$delivery) { + if (is_array($delivery['payload_envelope'] ?? null)) { + $envelope = $delivery['payload_envelope']; + $delivery['payload_envelope'] = array_keys($envelope); + sort($delivery['payload_envelope']); + $delivery['payload'] = $codec->decodeEnvelope($envelope); + } + } + unset($delivery); + $rows[] = ['event_type' => 'SignalReceived', 'deliveries' => $deliveries]; + } elseif (($event['event_type'] ?? null) === 'MessageCursorAdvanced') { + $key = $payload['stream_key'] ?? null; + $rows[] = ['event_type' => 'MessageCursorAdvanced', + 'stream_key' => $key === "instance:{$id}" ? 'instance' : $key, + 'previous_position' => $payload['previous_position'] ?? null, + 'new_position' => $payload['new_position'] ?? null, + 'task_status' => $payload['task']['status'] ?? null]; + } + } + + return $rows; + }; + $history = static function (string $runId) use ($raw, $id): array { + $read = $raw('GET', "workflows/{$id}/runs/{$runId}/history"); + + return $read['response']['events'] ?? []; + }; + $poll = static function () use ($client, $worker, $queue, $streamEvents): array { + $task = $client->pollWorkflowTask($worker, $queue, 0); + if (! is_array($task)) { + throw new RuntimeException('No workflow task was ready for the message stream run.'); + } + + return [$task, ['run_id' => $task['run_id'] ?? null, 'history' => $streamEvents($task['history_events'] ?? [])]]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $info = $raw('GET', 'cluster/info')['response']; + $state['cluster_contract'] = $info['message_streams_contract'] ?? null; + $state['worker_capability'] = $info['worker_protocol']['server_capabilities']['message_streams'] ?? null; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $queue, [$fixture['input']])->selectedRunId; + $state['list_before'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_before'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + $state['show_bad_name'] = $raw('GET', "workflows/{$id}/message-streams/".rawurlencode($definition['bad_stream'])); + $state['list_unknown'] = $raw('GET', "workflows/{$id}-missing/message-streams"); + $client->registerWorker($worker, $queue, [$definition['workflow_type']], []); + [$task, $state['poll1']] = $poll(); + $state['complete1_refusals'] = [ + 'wait_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 1]]]), + 'cursor_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]), + 'bad_name' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => 'bad name', 'after_position' => 0]]]), + 'repeated_wait' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [ + ['stream_name' => $stream, 'after_position' => 0], ['stream_name' => $stream, 'after_position' => 0]]]), + 'old_protocol' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]], '1.14'), + ]; + $state['complete1'] = $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]); + $state['list_waiting'] = $raw('GET', "workflows/{$id}/message-streams"); + $messages = $definition['messages']; + $state['appends'] = [ + 'm1' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_duplicate' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_conflict' => $append($stream, ['message_id' => 'm1', 'input' => $messages['conflict']]), + // The published SDK client's own append. + 'm2_sdk' => ['status' => null, 'response' => $client->appendMessageStream($id, $stream, 'm2', $messages['m2'])], + 'no_id' => $append($stream, ['input' => $messages['m3']]), + 'bad_id' => $append($stream, ['message_id' => 'bad id!', 'input' => $messages['m3']]), + 'long_id' => $append($stream, ['message_id' => str_repeat('a', $definition['long_message_id_length']), 'input' => $messages['m3']]), + 'bad_input' => $append($stream, ['message_id' => 'm3', 'input' => 'text']), + 'bad_stream' => $append($definition['bad_stream'], ['message_id' => 'm3', 'input' => $messages['m3']]), + 'unknown_workflow' => $raw('POST', "workflows/{$id}-missing/message-streams/{$stream}/messages", ['message_id' => 'm1', 'input' => [1]]), + 'other_stream' => $append($definition['other_stream'], ['message_id' => 'a1']), + 'reserved_signal' => $raw('POST', "workflows/{$id}/signal/__durable_workflow_message_stream", ['input' => [1]]), + ]; + $state['list_after_appends'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_after_appends'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + [$task, $state['poll2']] = $poll(); + $state['complete2_refusals'] = [ + 'beyond' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]), + 'negative' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => -1]]]), + 'wait_behind' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]], + 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]), + ]; + $state['complete2'] = $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]); + $state['history_after2'] = $streamEvents($history($state['run_id'])); + $state['list_after2'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['m3'] = $append($stream, ['message_id' => 'm3', 'input' => $messages['m3']]); + [$task, $state['poll3']] = $poll(); + $state['complete3'] = $complete($task, ['commands' => [['type' => 'continue_as_new', 'arguments' => $codec->envelope([$fixture['input']])]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 2]]]); + $state['list_after_continue'] = $raw('GET', "workflows/{$id}/message-streams"); + [$task, $state['poll4']] = $poll(); + $state['complete4'] = $complete($task, ['commands' => [['type' => 'complete_workflow', 'result' => $codec->envelope('done')]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]); + $state['continued_run_id'] = $state['poll4']['run_id']; + $state['list_final'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['closed'] = $append($stream, ['message_id' => 'm4', 'input' => [1]]); + $state['appends']['closed_duplicate'] = $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]); + $eventTypes = static fn (array $events): array => array_map(static fn (array $event): string => $event['event_type'], $events); + $firstRun = $history($state['run_id']); + $continuedRun = $history($state['continued_run_id']); + $state['first_run_events'] = $eventTypes($firstRun); + $state['first_run_streams'] = $streamEvents($firstRun); + $state['continued_run_events'] = $eventTypes($continuedRun); + $state['continued_run_streams'] = $streamEvents($continuedRun); + + return $state; +} diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index 69df4275e..b3b0d1d69 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -1,8 +1,8 @@ import assert from 'node:assert/strict'; import {isDeepStrictEqual} from 'node:util'; -// Reviewed operator metrics and health of one namespace, as published PHP -// 2.5.14 reports them through a run's stages. Counts must match exactly, with +// Reviewed operator metrics and health of one namespace, as published Server +// 2.11.1 reports them through a run's stages. Counts must match exactly, with // two normalizations: // - Laravel compares "now" in whole seconds, so a task made available within // the current second counts as delayed. Ready-due and delayed tasks are @@ -70,18 +70,99 @@ const EXPECTED = { backlog: {tasks_added_last_minute: 4}, history: {events: 8, max_event_count: 8}, command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, closed_backfill_needed_runs: 1}}, }; -// PHP's maintenance loop redispatches an overdue ready task (TaskRepair), so -// past the redispatch window published PHP may already show the task repaired -// instead of overdue. Native runs no repair loop and keeps reporting it overdue. -const REPAIRED = {...EXPECTED.waiting_after_redispatch, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, - backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1}}; -const REPAIRED_WARNINGS = ['command_contract_snapshots']; -// PHP's TaskWatchdog repairs the task and projects the run before -// TaskDispatcher redispatches it and projects the run again. Until then the -// task is still overdue and the run reads as needing repair. -const REPAIRING = {...EXPECTED.waiting_after_redispatch, runs: {...waitingRun, repair_needed: 1}, - backlog: {...EXPECTED.waiting_after_redispatch.backlog, repair_needed_runs: 1}}; -const REPAIRING_WARNINGS = ['command_contract_snapshots', 'task_transport', 'routing_health', 'durable_resume_paths']; +// Past the redispatch window, published PHP's maintenance loop moves the +// overdue ready task through four states: +// 0 overdue; +// 1 repairing: TaskWatchdog::recoverExistingTask sets the repair count and +// projects the run as needing repair, in one transaction; +// 2 dispatched: TaskDispatcher::markDispatched saves the task as dispatched, +// and its run is not yet projected again; +// 3 redispatched: the run is projected again. +// The task is overdue in states 0 and 1, carries a repair count from state 1, +// and its run reads as needing repair in states 1 and 2. +// Workflow aca7928484's OperatorMetrics::snapshot reads each value with its +// own query, so one document can see several states, never going backwards +// in query order. Its reads that depend on the state, in that order: +// runMetrics: oldest_repair_needed_at, then repair_needed; +// taskMetrics: oldest_dispatch_overdue_since (oldest_unhealthy_at is built +// from the same timestamps), then dispatch_overdue, then +// max_repair_count, then unhealthy, counted again; +// backlogMetrics: unhealthy_tasks, then repair_needed_runs. +// The health request reads its own snapshot afterwards, in the same order: +// task_transport warns on tasks.unhealthy, routing_health on +// tasks.dispatch_overdue, and durable_resume_paths on +// backlog.repair_needed_runs. Native runs no repair loop: it keeps the task +// overdue, or shows it redispatched once repaired. +const PHP_REPAIR_STATES = [0, 1, 2, 3]; +const PHP_READS = 8; +const overdueAt = state => state <= 1; +const runNeedsRepairAt = state => state === 1 || state === 2; +const runsAt = state => (runNeedsRepairAt(state) ? {...waitingRun, repair_needed: 1} : waitingRun); +const backlogAt = (unhealthy, repairNeeded) => ({runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1, + ...(overdueAt(unhealthy) ? {unhealthy_tasks: 1} : {}), ...(runNeedsRepairAt(repairNeeded) ? {repair_needed_runs: 1} : {})}); +const warningsAt = (task, run) => ['command_contract_snapshots', + ...(overdueAt(task) ? ['task_transport', 'routing_health'] : []), ...(runNeedsRepairAt(run) ? ['durable_resume_paths'] : [])]; +const REDISPATCHED = {...EXPECTED.waiting_after_redispatch, runs: runsAt(3), + tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, backlog: backlogAt(3, 3)}; +// The timestamps a PHP read may take apart from their counts. +const PHP_STAMPED = {runs: ['oldest_repair_needed_at'], tasks: ['oldest_dispatch_overdue_since', 'oldest_unhealthy_at']}; + +// Every way `count` reads can see the states without going backwards. +function forwardReads(count, from = 0) { + if (count === 0) return [[]]; + return PHP_REPAIR_STATES.filter(state => state >= from) + .flatMap(state => forwardReads(count - 1, state).map(rest => [state, ...rest])); +} +const PHP_DOCUMENT_READS = forwardReads(PHP_READS); + +// What a PHP document whose reads saw these states reports. +function phpDocument([runStamp, runCount, taskStamp, overdue, repairs, unhealthy, backlogUnhealthy, backlogRepair]) { + return { + counts: { + runs: runsAt(runCount), + tasks: {open: 1, ready: 1, ready_due_or_delayed: 1, ...(overdueAt(overdue) ? {dispatch_overdue: 1} : {}), + ...(repairs >= 1 ? {max_repair_count: 1} : {}), ...(overdueAt(unhealthy) ? {unhealthy: 1} : {})}, + backlog: backlogAt(backlogUnhealthy, backlogRepair), + }, + timestamps: { + runs: {oldest_repair_needed_at: runNeedsRepairAt(runStamp)}, + tasks: {oldest_dispatch_overdue_since: overdueAt(taskStamp), oldest_unhealthy_at: overdueAt(taskStamp)}, + }, + }; +} + +// The warnings of a PHP health request whose reads saw these states. +const phpWarnings = reads => ['command_contract_snapshots', ...(overdueAt(reads[5]) ? ['task_transport'] : []), + ...(overdueAt(reads[3]) ? ['routing_health'] : []), ...(runNeedsRepairAt(reads[7]) ? ['durable_resume_paths'] : [])]; + +const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); + +// Published PHP past the redispatch window: the expected counts, timestamps +// and the warnings its health may report. `timestamps` holds the presence of +// the PHP_STAMPED fields. The document's reads take the states that match +// and end earliest; the health request's reads start from there. With no +// match, the reads that match the counts alone are expected, so a timestamp +// difference is reported; with none either, the overdue state is expected. +export function phpRedispatchExpectation(counts, timestamps) { + const base = EXPECTED.waiting_after_redispatch; + const seen = {runs: nonzero(counts.runs), tasks: nonzero(counts.tasks), backlog: nonzero(counts.backlog)}; + const earliest = matches => matches.reduce((best, reads) => (best === null || reads[PHP_READS - 1] < best[PHP_READS - 1] ? reads : best), null); + const countMatches = PHP_DOCUMENT_READS.filter(reads => isDeepStrictEqual(phpDocument(reads).counts, seen)); + const best = earliest(countMatches.filter(reads => isDeepStrictEqual(phpDocument(reads).timestamps, timestamps))) + ?? earliest(countMatches); + if (best === null) { + const overdue = phpDocument(Array(PHP_READS).fill(0)); + return {counts: {...base, ...overdue.counts}, timestamps: overdue.timestamps, warnings: [WARNINGS.waiting_after_redispatch]}; + } + const document = phpDocument(best); + const warnings = []; + for (const reads of forwardReads(PHP_READS, best[PHP_READS - 1])) { + const list = phpWarnings(reads); + if (!warnings.some(listed => isDeepStrictEqual(listed, list))) warnings.push(list); + } + return {counts: {...base, ...document.counts}, timestamps: document.timestamps, warnings}; +} + // Checks other than ok at each stage (the uncompared check aside). const WARNINGS = { empty: [], @@ -93,16 +174,6 @@ const WARNINGS = { completed: [], }; -// Past the redispatch window published PHP is in one of these states, in -// order: overdue, repairing, or redispatched. Metrics and health are separate -// requests, so its health may be at the state its metrics show or a later one. -// Native runs no repair loop and is held to the state its metrics show. -const REPAIR_STATES = [ - {counts: EXPECTED.waiting_after_redispatch, warnings: WARNINGS.waiting_after_redispatch}, - {counts: REPAIRING, warnings: REPAIRING_WARNINGS}, - {counts: REPAIRED, warnings: REPAIRED_WARNINGS}, -]; - function normalized(sections) { const counts = {}; for (const [name, section] of Object.entries(sections)) { @@ -135,8 +206,12 @@ function checkStage(name, stage, namespace, family) { assert.deepStrictEqual([stage.metrics_status, stage.metrics_namespace, defect ? namespace : stage.health_namespace, stage.expected_namespace], [200, namespace, namespace, namespace], `${label} namespace`); assert.deepStrictEqual(stage.metrics_keys, ['namespace', 'operator_metrics'], `${label} document`); + const phpRedispatch = name === 'waiting_after_redispatch' && stage.runtime === 'php'; for (const [section, pairs] of Object.entries(TIMESTAMPS)) { for (const [field, count] of Object.entries(pairs)) { + // PHP's repair loop may move between a timestamp's query and its + // count's; the redispatch model checks those timestamps. + if (phpRedispatch && PHP_STAMPED[section]?.includes(field)) continue; assert.equal(stage.sections[section].timestamps[field], stage.sections[section].counts[count] > 0, `${label} ${section}.${field} present exactly when ${count} is not zero`); } @@ -152,19 +227,20 @@ function checkStage(name, stage, namespace, family) { } const counts = normalized(raw); const redispatch = name === 'waiting_after_redispatch'; - const php = redispatch && stage.runtime === 'php'; - const repaired = redispatch && counts.tasks.max_repair_count > 0; - if (repaired) delete counts.tasks.max_repair_count; - const states = redispatch ? REPAIR_STATES : [{counts: EXPECTED[name], warnings: WARNINGS[name]}]; - // A repair count means PHP repaired the task: mid-repair (PHP only) or redispatched. - const possible = !repaired ? [0] : php ? [1, 2] : [2]; - const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); - const matches = index => Object.entries(counts) - .every(([section, values]) => isDeepStrictEqual(nonzero(values), states[index].counts[section] ?? {})); - const state = possible.find(matches) ?? possible[possible.length - 1]; + let expected = {counts: EXPECTED[name], warnings: [WARNINGS[name]]}; + let stamps = null; + if (phpRedispatch) { + stamps = Object.fromEntries(Object.entries(PHP_STAMPED).map(([section, fields]) => + [section, Object.fromEntries(fields.map(field => [field, raw[section].timestamps[field]]))])); + expected = phpRedispatchExpectation(counts, stamps); + } else if (redispatch && counts.tasks.max_repair_count > 0) { + delete counts.tasks.max_repair_count; + expected = {counts: REDISPATCHED, warnings: [warningsAt(3, 3)]}; + } for (const [section, values] of Object.entries(counts)) { - assert.deepStrictEqual(nonzero(values), states[state].counts[section] ?? {}, `${label} ${section} counts`); + assert.deepStrictEqual(nonzero(values), expected.counts[section] ?? {}, `${label} ${section} counts`); } + if (stamps !== null) assert.deepStrictEqual(stamps, expected.timestamps, `${label} repair timestamps`); assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, `${label} active workers`); if (defect) return counts; @@ -172,8 +248,7 @@ function checkStage(name, stage, namespace, family) { const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); const warnings = statuses.filter(check => check.status !== 'ok').map(check => check.name); - const allowed = (php ? states.slice(state) : [states[state]]).map(candidate => candidate.warnings); - const settled = allowed.find(list => isDeepStrictEqual(list, warnings)) ?? states[state].warnings; + const settled = expected.warnings.find(list => isDeepStrictEqual(list, warnings)) ?? expected.warnings[0]; assert.deepStrictEqual(warnings, settled, `${label} warnings`); const warning = settled.length > 0 || stage.checks.some(check => check.status === 'warning'); assert.deepStrictEqual([stage.health_status, stage.health.status, stage.health.healthy], [200, warning ? 'warning' : 'ok', true], diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 4df73a65f..cd3d66aae 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -431,6 +431,8 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $searchAttributes = isset($fixture['search_attributes']) ? finishHttpSearchAttributes($client, $fixture, $workflowId, $url) : null; $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; + $activityCancellationAcknowledgement = isset($fixture['activity_cancellation_acknowledgement']) + ? finishHttpActivityCancellationAcknowledgement($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; $operatorDashboardTypes = isset($fixture['operator_dashboard_types']) ? finishHttpOperatorDashboardTypes($client, $fixture, $workflowId, $url) : null; @@ -439,6 +441,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $clusterInfo = isset($fixture['cluster_info']) ? finishHttpClusterInfo($client, $fixture, $workflowId, $url) : null; $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; + $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -539,6 +542,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'search_attributes' => $searchAttributes, 'storage_test' => $storageTest, 'activity_lease_renewal' => $activityLeaseRenewal, + 'activity_cancellation_acknowledgement' => $activityCancellationAcknowledgement, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, 'operator_dashboard_types' => $operatorDashboardTypes, @@ -547,6 +551,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'cluster_info' => $clusterInfo, 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, + 'message_streams' => $messageStreams, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -851,6 +856,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/search-attribute-probe.php'; require __DIR__.'/storage-test-probe.php'; require __DIR__.'/activity-lease-renewal-probe.php'; + require __DIR__.'/activity-cancellation-ack-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; require __DIR__.'/operator-dashboard-types-probe.php'; @@ -859,6 +865,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/cluster-info-probe.php'; require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; + require __DIR__.'/message-streams-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1020,6 +1027,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['activity_lease_renewal'])) { $observation['activity_lease_renewal'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['activity_cancellation_acknowledgement'])) { + $observation['activity_cancellation_acknowledgement'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } @@ -1044,6 +1054,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } + if ($mode === 'embedded' && isset($fixture['message_streams'])) { + $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 5e012bd34..738c21d1d 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -25,14 +25,29 @@ shared=(-v "$PWD:/source:ro" -v "$PWD/published-sdk-adapters:/sdk-adapters:ro" -e DW_PARITY_RUST_PATCH_WORKER=/sdk-adapters/rust-worker -e PYTHONDONTWRITEBYTECODE=1 -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) -for target in php rust; do - if test "$target" = php; then url="$php_url"; else url="$native_url"; fi +# Server #721. On SQLite the published PHP API server and its maintenance loop +# share one database file. SQLite refuses a deferred transaction's upgrade to a +# writer at once, without a busy wait, so PHP answers a contended poll with its +# typed backend_lock_pressure refusal. The published Python run_until helper +# then ends its Worker (sdk-python #218). This cohort tests patch replacement, +# not writer contention, so the PHP record runs with the maintenance loop +# stopped, and the loop restarts right after it. The fixtures use no timers or +# repair. The observer still never retries a poll refusal. +. scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +record() { docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ - scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ + scripts/conformance/server-parity.mjs record --mode http --url "$1" --target "$2" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ - --output "/evidence/published-sdk-patch/$target/record.json" -done + --output "/evidence/published-sdk-patch/$2/record.json" +} +if test "$backend" = sqlite; then + run_without_maintenance parity-evidence/published-sdk-patch/maintenance-schedule.txt -- \ + record "$php_url" php +else + record "$php_url" php +fi +record "$native_url" rust case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password) ;; diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh new file mode 100644 index 000000000..33528001f --- /dev/null +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -0,0 +1,62 @@ +# Sourced by published-sdk-patch-profile.sh (Server #721). +# +# run_without_maintenance SCHEDULE -- COMMAND... +# +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + test "$(maintenance_state)" = running || return 1 + sleep "$maintenance_poll_seconds" + done + test "$(maintenance_state)" = running +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then + return 0 + fi + done + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 + return 1 +} + +run_without_maintenance() { + local schedule="$1" + shift + test "$1" = -- && shift + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + test "$maintenance_container" = "$RESOURCE_SCOPE-php-maintenance" + echo "before: $(maintenance_state)" > "$schedule" + grep -qx 'before: running' "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/scripts/conformance/server-parity/query-authority/python.py b/scripts/conformance/server-parity/query-authority/python.py index 023aa84cb..e3f0d8b1c 100644 --- a/scripts/conformance/server-parity/query-authority/python.py +++ b/scripts/conformance/server-parity/query-authority/python.py @@ -117,6 +117,17 @@ async def main(): if MODE == 'timely': result = await client.query_workflow(namespace, 'value', args=[]) assert result['result'] == EXPECTED and type(result['result']['long']) is int + # The independent caller can observe the commit before the + # Worker receives its acknowledgement. Wait inside the + # original budget before stopping it, as worker.php does. + PHASE = 'acknowledgement' + while not any(x['event'] == 'acknowledgement' for x in events()): + if runner.done(): + await runner + raise RuntimeError('Worker stopped before its acknowledgement') + if time.monotonic() - started >= 30: + raise AssertionError('No acknowledgement inside the original budget') + await asyncio.sleep(.02) else: try: await client.query_workflow(namespace, 'value', args=[]) @@ -125,12 +136,19 @@ async def main(): assert error.status == 504 and error.reason == 'query_worker_execution_timeout' # Delivery is recorded before the SDK receives its response. # Wait for its actual refusal before stopping the worker. + # A late handler's callback counter moves before the SDK + # logs the lost authority, so every mode also waits for + # that evidence, as worker.php waits for its refusal. PHASE = 'expiry-observation' - while (MODE == 'late-poll' and not any( - expired_losses(event['claim']['query_task_id']) - for event in events() if event['event'] == 'claim') + while (not any(expired_losses(event['claim']['query_task_id']) + for event in events() if event['event'] == 'claim') or MODE == 'late-handler-async' and COUNTERS['cancelled'] != 1 or MODE == 'late-handler-sync' and COUNTERS['returned'] != 1): + if runner.done(): + await runner + raise RuntimeError('Worker stopped before its expiry evidence') + if time.monotonic() - started >= 30: + raise AssertionError('No expiry evidence inside the original budget') await asyncio.sleep(.02) finally: PHASE = 'worker-stop' diff --git a/scripts/conformance/server-parity/query-authority/rust/src/main.rs b/scripts/conformance/server-parity/query-authority/rust/src/main.rs index 45fae759d..d413a6dbc 100644 --- a/scripts/conformance/server-parity/query-authority/rust/src/main.rs +++ b/scripts/conformance/server-parity/query-authority/rust/src/main.rs @@ -169,6 +169,13 @@ async fn main() -> Result<()> { let query = client.query_workflow(&namespace, "value", json!([])).await; if mode == "timely" { assert_eq!(query?, expected()); + // The independent caller can observe the commit before the Worker + // receives its acknowledgement. Wait inside the original budget + // before stopping it, as worker.php does. + wait(until, || { + !named(&events(Path::new(journal)), "acknowledgement").is_empty() + }) + .await?; } else { match query { Err(Error::QueryFailed(failure)) => { diff --git a/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json b/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json new file mode 100644 index 000000000..3acc7f6d7 --- /dev/null +++ b/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json @@ -0,0 +1,129 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "activity-cancellation-acknowledgement", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "cancellation acknowledgement λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "cancellation acknowledgement λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "activity_cancellation_acknowledgement": { + "workflow_type": "parity.v1.cancel_ack", + "activity_type": "parity.v1.cancel_ack_activity", + "cleanup_timeout_seconds": 60, + "empty_blob": "wwHioz3/VYAiNwwA", + "result_blob": "wwHioz3/VYAiNwoIZG9uZQ==", + "refusals": { + "protocol_missing": { + "status": 400, + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests." + }, + "protocol_1_21": { + "status": 400, + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches." + }, + "missing_fields": { + "status": 422, + "message": "The activity attempt id field is required. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "activity_attempt_id": [ + "The activity attempt id field is required." + ], + "lease_owner": [ + "The lease owner field is required." + ], + "request_id": [ + "The request id field is required." + ] + }, + "validation_errors": { + "activity_attempt_id": [ + "The activity attempt id field is required." + ], + "lease_owner": [ + "The lease owner field is required." + ], + "request_id": [ + "The request id field is required." + ] + } + }, + "non_string_fields": { + "status": 422, + "message": "The activity attempt id field must be a string. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "activity_attempt_id": [ + "The activity attempt id field must be a string." + ], + "lease_owner": [ + "The lease owner field must be a string." + ], + "request_id": [ + "The request id field must be a string." + ] + }, + "validation_errors": { + "activity_attempt_id": [ + "The activity attempt id field must be a string." + ], + "lease_owner": [ + "The lease owner field must be a string." + ], + "request_id": [ + "The request id field must be a string." + ] + } + }, + "long_request_id": { + "status": 422, + "message": "The request id field must not be greater than 255 characters.", + "reason": "validation_failed", + "errors": { + "request_id": [ + "The request id field must not be greater than 255 characters." + ] + }, + "validation_errors": { + "request_id": [ + "The request id field must not be greater than 255 characters." + ] + } + } + } + }, + "contract": "POST /api/worker/activity-tasks/{task}/acknowledge-cancellation (published Server 2.11, advertised as server_capabilities.activity_cancellation_acknowledgement, also in cluster info) records an activity worker's stop after a cooperative cancellation reached its attempt. The path is request-cancellation, then deliver-cancellation with call_kind activity from the workflow task, which cancels the attempt and records ActivityCancelled; the activity heartbeat then reports attempt_closed and the status route carries the cancellation_acknowledgement receipt with the request id. The acknowledgement answers 200 and records one ActivityCancellationAcknowledged event with the stop's evidence and the cancelled task's snapshot; the receipt then reads stopped and a repeat answers duplicate with the same event, while later outcomes stay refused. Before the request, with another request id or after an immediate cancel it answers cancellation_request_mismatch; before delivery activity_cancellation_not_recorded; for an older attempt of the execution activity_cancellation_acknowledgement_fence_mismatch. Ownership is checked in PHP's order with its bodies, then the claim: an attempt polled on protocol 1.19 cannot acknowledge. A request below protocol 1.20 is refused before validation; missing, non-string or over-long fields fail Laravel's validation. A run terminated after delivery still records the late stop, after its terminal event. Published 2.11.1 delivers a retried activity's cancellation to the execution's first task (Workflow #923), so that attempt's acknowledgement is refused as not fenced; the contract accepts exactly that reference defect or the correct acknowledgement. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParity/message-streams.json b/tests/Fixtures/ServerParity/message-streams.json new file mode 100644 index 000000000..49b0ea7ef --- /dev/null +++ b/tests/Fixtures/ServerParity/message-streams.json @@ -0,0 +1,182 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "message-streams", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "message_streams": { + "workflow_type": "parity.v1.message_streams", + "stream": "orders", + "other_stream": "audit", + "bad_stream": "bad name!", + "condition_fingerprint_source": "message-stream:orders", + "manifest": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "messages": { + "m1": [ + { + "n": 1 + } + ], + "m2": [ + { + "n": 2 + } + ], + "m3": [ + { + "n": 3 + } + ], + "conflict": [ + { + "n": 9 + } + ] + }, + "long_message_id_length": 192, + "refusals": { + "stream_name": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "metadata_stream_name": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "message_id_required": "The message id field is required.", + "message_id_format": "The message id field format is invalid.", + "message_id_max": "The message id field must not be greater than 191 characters.", + "input_array": "The input field must be an array.", + "wait_after_cursor": "The pending wait must start after the durable stream cursor.", + "cursor_beyond": "The cursor cannot advance beyond the last accepted stream position.", + "duplicate_wait": "A stream may have only one pending wait per workflow task completion.", + "negative_cursor": "The message_stream_cursors.0.through_position field must be at least 0.", + "old_protocol_remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ] + }, + "contract": "Message streams (published Server 2.11.1, MessageStreamController and MessageStreamService over the Workflow package's runtime-reserved signal). Cluster info publishes the message streams contract and the worker capability the same contract with supported true. A client appends ordered messages to a named stream of a workflow instance: the first message is position 1, a repeated message id with the same payload is a duplicate at its original position, and with another payload a 409 message_identity_conflict. Each accepted message reaches the current run as a __durable_workflow_message_stream SignalReceived whose single argument holds the message schema, stream name, message id, position and the inline payload envelope. Refused requests are counted as malformed without keeping their payload, by a diagnostic message id. Diagnostics list every stream of the instance by name with positions, pending counts, the pending wait, duplicate and malformed counts and the last input, without payloads. A worker on protocol 1.15 or newer reports open receives with message_stream_waits and consumed messages with message_stream_cursors on a workflow task completion; the server checks both before the completion changes anything, records MessageCursorAdvanced for each moved cursor and clears the wait. An open condition wait is woken by a stream message, which it consumes without SignalApplied. When the run continues as new, the next run receives a payload-free cursor checkpoint for each acknowledged stream and every unconsumed message again, in append order; signals the old run had not consumed are rejected. Appends to a closed run are refused with run_not_active, duplicates included. Embedded applications have no HTTP message streams." +} diff --git a/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json new file mode 100644 index 000000000..9a060b57a --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json @@ -0,0 +1 @@ +{"applicable": true, "namespace": "pr1-activity-cancellation-acknowledgement-cack", "discovery": {"activity_cancellation_acknowledgement": true, "cooperative_cancellation": true}, "main": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "worker": "pr1-activity-cancellation-acknowledgement-main-worker", "workflow_task_id": "01m5qw91semrqyrf9d8btymy60", "task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker"}, "before_request": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "not-yet-requested", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "request": {"status": 202, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "accepted": true, "duplicate": false, "run_status": "waiting", "cancellation_request": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "reason": null}, "acknowledgement_capability": "absent"}, "before_delivery": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_recorded", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_delivery": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": true, "cancel_requested": false, "reason": null, "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T21:20:13.496866Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "running", "attempt_status": "running", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "poll_cancellation_request": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw92edr1d6qfp9wpvk18p8", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "heartbeat_after_delivery": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancel_requested": true, "can_continue": false, "reason": "attempt_closed", "run_closed_reason": null, "run_closed_at": null, "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_ack": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_before_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}], "foreign_owner": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity attempt lease is owned by another worker.", "reason": "lease_owner_mismatch", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_attempt": {"status": 404, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "no-such-attempt", "error": "Activity attempt not found.", "reason": "attempt_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_task": {"status": 404, "response": {"task_id": "no-such-task", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "task_mismatch": {"status": 409, "response": {"task_id": "01m5qw91semrqyrf9d8btymy60", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "other_namespace": {"status": 404, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_19": {"status": 422, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "minimum_protocol_version": "1.20", "unavailable": ["worker_protocol"], "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_missing": {"status": 400, "response": {"error": "Missing worker protocol version header.", "reason": "missing_protocol_version", "supported_version": "1.20", "requested_version": null, "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_21": {"status": 400, "response": {"error": "Unsupported worker protocol version.", "reason": "unsupported_protocol_version", "supported_version": "1.20", "requested_version": "1.21", "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "missing_fields": {"status": 422, "response": {"message": "The activity attempt id field is required. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "non_string_fields": {"status": 422, "response": {"message": "The activity attempt id field must be a string. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "long_request_id": {"status": 422, "response": {"message": "The request id field must not be greater than 255 characters.", "reason": "validation_failed", "errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "validation_errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "wrong_request_id": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "other-request", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}], "acknowledged": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5qw9483fhm6p895f7baexdc", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_after_ack": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "callback_state": "stopped", "history_event_id": "01m5qw9483fhm6p895f7baexdc", "acknowledged_at": "2026-10-24T21:15:15.585663Z", "received_after_deadline": false}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "duplicate": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": true, "duplicate": true, "reason": null, "history_event_id": "01m5qw9483fhm6p895f7baexdc", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "completed_after_ack": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "outcome": "completed", "recorded": false, "reason": "stale_attempt", "next_task_id": null, "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}, {"sequence": 8, "event_type": "ActivityCancellationAcknowledged", "timestamp": "2026-10-24T21:15:15.586501Z", "principal": null, "payload": {"sequence": 1, "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "callback_state": "stopped", "evidence_source": "activity_worker", "acknowledged_at": "2026-10-24T21:15:15.585663Z", "received_after_deadline": false, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}], "legacy_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-legacy", "run_id": "01m5qw95ncnf12s9cyr66q7pck", "queue": "pr1-activity-cancellation-acknowledgement-legacy-queue", "worker": "pr1-activity-cancellation-acknowledgement-legacy-worker", "workflow_task_id": "01m5qw95nx7rnbn6ngvve83q53", "task_id": "01m5qw95yrdg1xx4g57v6fytj5", "activity_attempt_id": "01M5QW963PD68K0ZPJB6BSP7WY", "activity_execution_id": "01m5qw95yh7rav0ej04snvxqfm", "lease_owner": "pr1-activity-cancellation-acknowledgement-legacy-worker"}, "legacy_request": 202, "legacy_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw968w13nt8j2a0w1pewnq", "workflow_run_id": "01m5qw95ncnf12s9cyr66q7pck", "request_id": "01M5QW968DTFETWENZ785VDCW1", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "legacy_ack": {"status": 409, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "required_capability": "cooperative_cancellation", "minimum_protocol_version": "1.20", "worker_id": "pr1-activity-cancellation-acknowledgement-legacy-worker", "unavailable": ["worker_claim_capability"], "remediation": "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5qw96yehxek7x8mjp3fyqy6", "queue": "pr1-activity-cancellation-acknowledgement-immediate-queue", "worker": "pr1-activity-cancellation-acknowledgement-immediate-worker", "workflow_task_id": "01m5qw96yz3mwt8beb0cngmp8n", "task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "activity_execution_id": "01m5qw977rz8cspyd91hqxexv8", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker"}, "immediate_cancel": {"status": 200, "response": {"outcome": "cancelled", "workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5qw96yehxek7x8mjp3fyqy6", "requested_run_id": null, "resolved_run_id": "01m5qw96yehxek7x8mjp3fyqy6", "command_id": "01m5qw97p9xgyxzwfwmzbczkd1", "command_sequence": 2, "target_scope": "instance", "workflow_type": "parity.v1.cancel_ack", "command_status": "accepted", "command_source": "control_plane", "reason": "acknowledgement fixture", "rejection_reason": null, "validation_errors": []}, "acknowledgement_capability": "absent"}, "immediate_ack": {"status": 409, "response": {"task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "request_id": "01m5qw97p9xgyxzwfwmzbczkd1", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_status": {"status": 200, "response": {"task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "can_continue": false, "cancel_requested": true, "reason": "run_cancelled", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "cancelled", "run_closed_reason": "cancelled", "run_closed_at": "2026-10-24T21:15:19.139787Z", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:18.868117Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first": {"workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5qw98677c10kwqv7h4s9x4e", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5qw986qacg06ps3paph5624", "task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker"}, "stale_fail": {"status": 200, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "outcome": "failed", "recorded": true, "reason": null, "next_task_id": "01m5qw98qqqdsbe49ye8879md3", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5qw98677c10kwqv7h4s9x4e", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5qw986qacg06ps3paph5624", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m"}, "stale_request": 202, "stale_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9922842rpf9krn13t5dn", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_own_task": {"status": 409, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_acknowledgement_fence_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_new_task": {"status": 409, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt_old_task": {"status": 409, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt": {"status": 409, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_fenced", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_status": {"status": 200, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T21:20:20.353227Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T21:20:20.353020Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "cancellation_history_event_id": "01m5qw99b2xay0h93x1sskjqbc", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_history": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:19.630000Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw986a3jv3c1jqyh29q0hh", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw986a3jv3c1jqyh29q0hh", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:19.624680Z", "applied_at": "2026-10-24T21:15:19.624687Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:19.635924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "workflow_command_id": "01m5qw986a3jv3c1jqyh29q0hh", "command": {"id": "01m5qw986a3jv3c1jqyh29q0hh", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:19.624680Z", "applied_at": "2026-10-24T21:15:19.624687Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:19.891980Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw986qacg06ps3paph5624", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:19.639387Z", "leased_at": "2026-10-24T21:15:19.757708Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:16:19.757637Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:19.645942Z", "last_dispatched_at": "2026-10-24T21:15:19.645942Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:20.037645Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.030742Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:20:20.030966Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 5, "event_type": "ActivityRetryScheduled", "timestamp": "2026-10-24T21:15:20.187201Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "retry_task_id": "01m5qw98qqqdsbe49ye8879md3", "retry_of_task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "retry_available_at": "2026-10-24T21:15:20.178836Z", "retry_backoff_seconds": 0, "retry_after_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "retry_after_attempt": 1, "max_attempts": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "exception_type": "parity.retryable", "activity_attempt": {"id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "status": "failed", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "closed_at": "2026-10-24T21:15:20.179184Z"}, "exception_class": "RuntimeException", "message": "retry me", "code": 0, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "status": "pending", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.030742Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "completed", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 6, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:20.359625Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "status": "running", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.353020Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "task": {"id": "01m5qw98qqqdsbe49ye8879md3", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:20.178836Z", "leased_at": "2026-10-24T21:15:20.353020Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:20:20.353227Z", "attempt_count": 2, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:20.219594Z", "last_dispatched_at": "2026-10-24T21:15:20.219594Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:20.508622Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:20.501658Z", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "lineage": [{"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e"}]}, "command": {"id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw98677c10kwqv7h4s9x4e", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyMC41MDE2NThaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5OTFOVFRDN0JCWDhaVjVLWjVaUR5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzk5MU5UVEM3QkJYOFpWNUtaNVpRMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXnByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LXN0YWxlKHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5ODY3N2MxMGt3cXY3aDRzOXg0ZSJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjIwLjUwMTY1OFomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyMC41MDE2NThaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzk5MU5UVEM3QkJYOFpWNUtaNVpRKHdvcmtmbG93X2luc3RhbmNlX2lkCl5wcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1zdGFsZR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzk4Njc3YzEwa3dxdjdoNHM5eDRlAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-stale/runs/01m5qw98677c10kwqv7h4s9x4e/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:97314033b81ef0fcb18ea2e9b0903b78227292617d2066fa5e74e3a5b31b3c0c", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:20.501658Z", "applied_at": "2026-10-24T21:15:20.501658Z"}}}, {"sequence": 8, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:20.802210Z", "principal": null, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "cancelled_at": "2026-10-24T21:15:20.795645Z", "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "status": "cancelled", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.353020Z", "closed_at": "2026-10-24T21:15:20.795645Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "activity_attempt": {"id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "task_id": "01m5qw98qqqdsbe49ye8879md3", "attempt_number": 2, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "started_at": "2026-10-24T21:15:20.353020Z", "closed_at": "2026-10-24T21:15:20.795645Z"}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:20.807285Z", "principal": null, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:20.501658Z", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "lineage": [{"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e"}]}, "task": {"id": "01m5qw9922842rpf9krn13t5dn", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:20.501658Z", "leased_at": "2026-10-24T21:15:20.679571Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:15:30.679486Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:20.550270Z", "last_dispatched_at": "2026-10-24T21:15:20.550270Z"}}}], "terminated_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-terminated", "run_id": "01m5qw9a29sq7n9y3p7j80psk2", "queue": "pr1-activity-cancellation-acknowledgement-terminated-queue", "worker": "pr1-activity-cancellation-acknowledgement-terminated-worker", "workflow_task_id": "01m5qw9a2qpwwjd0t2pt6c85by", "task_id": "01m5qw9aav3vemzppj1s2hd037", "activity_attempt_id": "01M5QW9AFB23KJQ4BBBQ1ZQ7V8", "activity_execution_id": "01m5qw9aapb0scpkta377s39ya", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker"}, "terminated_request": 202, "terminated_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9amdz410vdkgdh3sh41y", "workflow_run_id": "01m5qw9a29sq7n9y3p7j80psk2", "request_id": "01M5QW9AM19VRHEZSXDP4ZTD9K", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminate": 200, "terminated_ack": {"status": 200, "response": {"task_id": "01m5qw9aav3vemzppj1s2hd037", "activity_attempt_id": "01M5QW9AFB23KJQ4BBBQ1ZQ7V8", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker", "request_id": "01M5QW9AM19VRHEZSXDP4ZTD9K", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5qw9b78rej9s74b5rxrdyjq", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminated_history": ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityStarted", "CooperativeCancellationRequested", "ActivityCancelled", "CooperativeCancellationDelivered", "TerminateRequested", "WorkflowTerminated", "ActivityCancellationAcknowledged"], "boundary_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-boundary", "run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "worker": "pr1-activity-cancellation-acknowledgement-boundary-worker", "workflow_task_id": "01m5qw9bkj8asvajdrtsaka0sk", "task_id": "01m5qw9bvq80x04md6pmhc1x1h", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker"}, "boundary_complete": 200, "boundary_timer": 200, "boundary_request": 202, "boundary_task_id": "01m5qw9cjkmv2nag3wyztp5se1", "boundary_history_before": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}], "boundary_resolved_activity": {"status": 409, "response": {"delivered": false, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": null, "sequence": null, "call_kind": null, "sequence_span": null, "operation_sequence": null, "operation_sequence_span": null, "reason": "cancellation_delivery_not_eligible", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_timer_as_activity": {"status": 409, "response": {"delivered": false, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": null, "sequence": null, "call_kind": null, "sequence_span": null, "operation_sequence": null, "operation_sequence_span": null, "reason": "cancellation_delivery_shape_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}], "boundary_pending": {"status": 200, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-boundary", "run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "accepted": true, "duplicate": true, "run_status": "waiting", "cancellation_request": {"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}}, "acknowledgement_capability": "absent"}, "boundary_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "call_kind": "timer", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_history_after_delivery": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}, {"sequence": 8, "event_type": "TimerCancelled", "timestamp": "2026-10-24T21:15:24.751043Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "cancelled_at": "2026-10-24T21:15:24.750414Z", "task": {"id": "01m5qw9cjkmv2nag3wyztp5se1", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:24.105115Z", "leased_at": "2026-10-24T21:15:24.292591Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:15:34.292508Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:24.153076Z", "last_dispatched_at": "2026-10-24T21:15:24.153076Z"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:24.757076Z", "principal": null, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "sequence": 2, "call_kind": "timer", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "task": {"id": "01m5qw9cjkmv2nag3wyztp5se1", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:24.105115Z", "leased_at": "2026-10-24T21:15:24.292591Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:15:34.292508Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:24.153076Z", "last_dispatched_at": "2026-10-24T21:15:24.153076Z"}}}]} \ No newline at end of file diff --git a/tests/Fixtures/ServerParityExpected/message-streams-observation.json b/tests/Fixtures/ServerParityExpected/message-streams-observation.json new file mode 100644 index 000000000..215042c4c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/message-streams-observation.json @@ -0,0 +1,1323 @@ +{ + "recorded_from": "published Server 2.11.1 (ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102) on SQLite, workflow id probe-5", + "message_streams": { + "applicable": true, + "workflow_id": "probe-5-ms", + "cluster_contract": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "worker_capability": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + }, + "supported": true + }, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "list_before": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 0, + "streams": [] + } + }, + "show_before": { + "status": 404, + "response": { + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "reason": "message_stream_not_found" + } + }, + "show_bad_name": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "list_unknown": { + "status": 404, + "response": { + "message": "Workflow not found.", + "reason": "instance_not_found" + } + }, + "poll1": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [] + }, + "complete1_refusals": { + "wait_ahead": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + }, + "cursor_ahead": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "bad_name": { + "status": 422, + "response": { + "message": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "repeated_wait": { + "status": 422, + "response": { + "message": "A stream may have only one pending wait per workflow task completion.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + }, + "validation_errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + } + } + }, + "old_protocol": { + "status": 409, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "rejected", + "reason": "message_streams_unavailable", + "requested_version": "1.14", + "minimum_protocol_version": "1.15", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + } + } + }, + "complete1": { + "status": 200, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "list_waiting": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 1, + "streams": [ + { + "stream_name": "orders", + "last_position": 0, + "cursor_position": 0, + "pending_count": 0, + "pending_wait": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "after_position": 0, + "since": "2026-10-23T19:04:33.000000Z" + }, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": null, + "last_input_message_id": null, + "last_input_at": null + } + ] + } + }, + "appends": { + "m1": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "m1_duplicate": { + "status": 200, + "response": { + "accepted": true, + "duplicate": true, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "duplicate", + "reason": null + } + }, + "m1_conflict": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "rejected", + "reason": "message_identity_conflict" + } + }, + "m2_sdk": { + "status": null, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "no_id": { + "status": 422, + "response": { + "message": "The message id field is required.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field is required." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field is required." + ] + } + } + }, + "bad_id": { + "status": 422, + "response": { + "message": "The message id field format is invalid.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field format is invalid." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field format is invalid." + ] + } + } + }, + "long_id": { + "status": 422, + "response": { + "message": "The message id field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + } + } + }, + "bad_input": { + "status": 422, + "response": { + "message": "The input field must be an array.", + "reason": "validation_failed", + "errors": { + "input": [ + "The input field must be an array." + ] + }, + "validation_errors": { + "input": [ + "The input field must be an array." + ] + } + } + }, + "bad_stream": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "unknown_workflow": { + "status": 404, + "response": { + "accepted": false, + "workflow_id": "probe-5-ms-missing", + "stream_name": "orders", + "outcome": "rejected", + "reason": "instance_not_found" + } + }, + "other_stream": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "reserved_signal": { + "status": 422, + "response": { + "workflow_id": "probe-5-ms", + "signal_name": "__durable_workflow_message_stream", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport.", + "accepted": false, + "control_plane": { + "schema": "durable-workflow.v2.control-plane-response", + "version": 1, + "operation": "signal", + "workflow_id": "probe-5-ms", + "contract": { + "schema": "durable-workflow.v2.control-plane-response.contract", + "version": 1, + "legacy_field_policy": "reject_non_canonical", + "legacy_fields": { + "query": "query_name", + "signal": "signal_name", + "update": "update_name", + "wait_policy": "wait_for" + }, + "required_fields": [ + "workflow_id", + "operation_name", + "operation_name_field" + ], + "success_fields": [ + "outcome" + ], + "rejection_fields": [ + "workflow_id", + "run_id", + "signal_name", + "command_status", + "command_source", + "target_scope", + "outcome", + "reason", + "rejection_reason", + "rejection_category", + "message", + "command_contract_source", + "command_contract_backfill_needed", + "command_contract_backfill_available", + "declared_signals", + "signal_admission", + "retryable", + "error_id" + ], + "rejection_reasons": [ + "instance_not_found", + "historical_run_command_rejected", + "unknown_signal", + "invalid_signal_arguments", + "run_not_active", + "instance_not_started", + "selected_run_not_current", + "configured_workflow_type_invalid", + "v1_projection_read_only", + "backend_lock_pressure", + "control_plane_internal_error" + ] + }, + "operation_name": "__durable_workflow_message_stream", + "operation_name_field": "signal_name", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport." + } + } + }, + "m3": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "cursor_position": 1, + "outcome": "accepted", + "reason": null + } + }, + "closed": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m4", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + }, + "closed_duplicate": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + } + }, + "list_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "stream": { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + } + }, + "poll2": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + } + ] + }, + "complete2_refusals": { + "beyond": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "negative": { + "status": 422, + "response": { + "message": "The message_stream_cursors.0.through_position field must be at least 0.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + } + } + }, + "wait_behind": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + } + }, + "complete2": { + "status": 200, + "response": { + "task_id": "01m5n2d3eb25p3d5t00y35nhhk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [ + "01m5n2d516v62fqj8j4dzkb82t" + ], + "reason": null + } + }, + "history_after2": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + } + ], + "list_after2": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 1, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "poll3": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete3": { + "status": 200, + "response": { + "task_id": "01m5n2d516v62fqj8j4dzkb82t", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "completed", + "created_task_ids": [ + "01m5n2d5gnm3yatnb4jaqtvfnk" + ], + "reason": null + } + }, + "list_after_continue": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 2, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "poll4": { + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete4": { + "status": 200, + "response": { + "task_id": "01m5n2d5gnm3yatnb4jaqtvfnk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "continued_run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "list_final": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 3, + "pending_count": 0, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "first_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 1, + "new_position": 2, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 1, + "new_position": 2, + "task_status": "completed" + } + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 2, + "new_position": 3, + "task_status": "completed" + } + ] + } +} diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs new file mode 100644 index 000000000..24398e2cc --- /dev/null +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -0,0 +1,100 @@ +// Server #721: the SQLite published-SDK patch cohort stops PHP's maintenance +// loop for its PHP record. Every failure path must leave the loop restored or +// fail loudly, and never record a restart that did not stay running. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const helper = resolve('scripts/conformance/server-parity/published-sdk-patch/maintenance.sh'); + +// One container's state in a file; every call is logged. +// SCENARIO=stop-nonzero: the stop takes effect but exits 1. +// SCENARIO=immediate-exit: start exits 0, the container runs for one +// inspection and then exits. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +test "\${!#}" = dw-parity-7-2-sqlite-php-maintenance || exit 3 +case "$1" in + inspect) + state=$(cat "$dir/state") + if test "$state" = running-once; then echo exited > "$dir/state"; state=running; fi + echo "$state" ;; + stop) echo exited > "$dir/state"; test "$SCENARIO" != stop-nonzero ;; + start) + if test "$SCENARIO" = immediate-exit; then echo running-once > "$dir/state"; else echo running > "$dir/state"; fi ;; + *) exit 2 ;; +esac +`; + +function run(scenario, command, scope = 'dw-parity-7-2-sqlite') { + const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', ['-c', + `set -euo pipefail; . "$HELPER"; run_without_maintenance "$SCHEDULE" -- ${command}`], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05', + GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', RESOURCE_SCOPE: scope}, + }); + const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); + const calls = read('calls').trim().split('\n'); + return { + status: result.status, + stderr: result.stderr, + schedule: read('schedule.txt').trim().split('\n'), + starts: calls.filter(call => call.startsWith('start ')).length, + stopIndex: calls.findIndex(call => call.startsWith('stop ')), + lastStart: calls.findLastIndex(call => call.startsWith('start ')), + state: read('state').trim(), + }; +} + +test('a normal window stops, runs, and restores a loop that stays running', () => { + const result = run('normal', 'true'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.schedule, ['before: running', 'during php record: exited', 'after php record: running for 0.05s x3']); + assert.equal(result.starts, 1, 'restoration is disarmed after the sustained check'); + assert.equal(result.state, 'running'); +}); + +test('a stop that takes effect but exits nonzero still restores', () => { + const result = run('stop-nonzero', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.lastStart > result.stopIndex, 'a start follows the stop'); + assert.equal(result.state, 'running'); + assert.deepEqual(result.schedule, ['before: running']); +}); + +test('a failed PHP record restores the loop', () => { + const result = run('normal', 'false'); + assert.notEqual(result.status, 0); + assert.equal(result.state, 'running'); + assert.ok(result.starts >= 1); + assert.ok(!result.schedule.some(line => line.startsWith('after php record'))); +}); + +test('a restart that exits at once stays armed and fails without recording running', () => { + const result = run('immediate-exit', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.schedule.includes('after php record: exited'), result.schedule.join('\n')); + assert.ok(!result.schedule.some(line => line.startsWith('after php record: running'))); + // Three attempts after the record, then three more from the still-armed trap. + assert.equal(result.starts, 6); + assert.match(result.stderr, /did not stay running/); +}); + +test('a container outside the job scope is never touched', () => { + const result = run('normal', 'true', 'dw-parity-other-job'); + assert.notEqual(result.status, 0); + assert.equal(result.starts, 0); + assert.equal(result.stopIndex, -1); + assert.equal(result.state, 'running'); +}); diff --git a/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs new file mode 100644 index 000000000..8066eb17e --- /dev/null +++ b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkActivityCancellationAcknowledgement} from '../../scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/activity-cancellation-acknowledgement.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json', import.meta.url))); +const id = 'pr1-activity-cancellation-acknowledgement'; +const projection = {scope: 'activity_cancellation_acknowledgement', cleanup_timeout_seconds: 60}; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', activity_cancellation_acknowledgement: state}; +}; +const check = state => checkActivityCancellationAcknowledgement(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), projection); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', activity_cancellation_acknowledgement: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), projection); +}); + +// The correct retried-attempt outcome (Workflow #923 fixed): the delivery +// cancels the live task and its attempt acknowledges. +const fixedRetry = state => { + const cancelled = state.stale_history.find(event => event.event_type === 'ActivityCancelled'); + cancelled.payload.task.id = state.stale_second.task_id; + state.stale_second_attempt.status = 200; + Object.assign(state.stale_second_attempt.response, {acknowledged: true, reason: null, history_event_id: '01m5pcebxrbwvbd8n7t8ezc0kz'}); +}; + +test('the correct retried-attempt acknowledgement is accepted', () => { + assert.deepStrictEqual(check(observation(fixedRetry)), projection); +}); + +for (const [label, change, message] of [ + ['a refused live attempt whose delivery cancelled its own task', state => { + state.stale_history.find(event => event.event_type === 'ActivityCancelled').payload.task.id = state.stale_second.task_id; + }, /the live attempt acknowledges/], + ['the reference defect without its leased live task', state => { + state.stale_second_status.response.task_status = 'cancelled'; + }, /reference defect: the live task stays leased/], + ['an acknowledgement that does not advertise the capability', state => { + state.acknowledged.acknowledgement_capability = false; + }, /acknowledged answers in worker protocol 1.20 and advertises acknowledgement/], + ['cluster info without the capability', state => { + state.discovery.activity_cancellation_acknowledgement = 'absent'; + }, /cluster info advertises/], + ['a refusal that records history', state => { + state.history_after_refusals.push(structuredClone(state.history_before_ack.at(-1))); + }, /refusals record nothing/], + ['a status receipt before delivery', state => { + state.status_before_delivery.response.cancellation_acknowledgement = {}; + }, /a pending request does not stop the attempt/], + ['a receipt that already reads stopped', state => { + state.status_before_ack.response.cancellation_acknowledgement.callback_state = 'stopped'; + }, /the receipt before acknowledgement/], + ['a duplicate that records a second event', state => { + state.duplicate.response.history_event_id = '01m5pce6htfpz4bjzf62naz6cz'; + }, /duplicate/], + ['an acknowledgement event without its evidence source', state => { + delete state.history_after_ack.at(-1).payload.evidence_source; + }, /acknowledgement payload/], + ['an acknowledgement snapshot that keeps a lease', state => { + state.history_after_ack.at(-1).payload.task.lease_expires_at = '2026-10-24T07:24:08.525107Z'; + }, /without a lease/], + ['a 1.19 claim that acknowledges', state => { + state.legacy_ack = structuredClone(state.acknowledged); + }, /a 1\.19 claim cannot acknowledge/], + ['a 1.19 request refused by validation instead', state => { + state.protocol_1_19 = structuredClone(state.missing_fields); + }, /a 1\.19 request/], + ['an immediate cancel with a receipt', state => { + state.immediate_status.response.cancellation_acknowledgement = {}; + }, /immediate cancel status, no receipt/], + ['a stale first attempt that acknowledges', state => { + state.stale_first_attempt_own_task.status = 200; + }, /stale_first_attempt_own_task/], + ['a terminated run whose acknowledgement precedes its terminal event', state => { + state.terminated_history = [...state.terminated_history.slice(0, -3), 'ActivityCancellationAcknowledged', + 'TerminateRequested', 'WorkflowTerminated']; + }, /follows the terminal event/], + ['a cleanup budget other than the requested one', state => { + state.request.response.cancellation_request.cleanup_deadline_at = '2026-10-24T07:21:08.747589Z'; + }, /the cleanup budget runs from the request/], + // The defect in Server #745's first native head: an activity delivery at + // an activity that already completed was recorded instead of refused. + ['a delivery recorded at an activity that resolved before the request', state => { + state.boundary_resolved_activity = structuredClone(state.boundary_delivery); + Object.assign(state.boundary_resolved_activity.response, {sequence: 1, call_kind: 'activity'}); + }, /an activity that resolved before the request is not eligible/], + ['a resolved activity refused as a shape mismatch', state => { + state.boundary_resolved_activity.response.reason = 'cancellation_delivery_shape_mismatch'; + }, /an activity that resolved before the request is not eligible/], + ['an activity delivery accepted at a timer', state => { + state.boundary_timer_as_activity.status = 200; + state.boundary_timer_as_activity.response.delivered = true; + }, /an activity delivery at a timer is a shape mismatch/], + ['a boundary refusal that records history', state => { + state.boundary_history_after_refusals.push(structuredClone(state.boundary_history_before.at(-1))); + }, /boundary refusals record nothing/], + ['a refusal that pins the delivery sequence', state => { + state.boundary_pending.response.cancellation_request.delivery_sequence = 1; + }, /the request is still undelivered after the refusals/], + ['a timer delivery that leaves the timer pending', state => { + state.boundary_history_after_delivery.splice(-2, 1); + }, /delivery only appends|the timer is cancelled under the delivery/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} diff --git a/tests/Unit/ServerParityMessageStreamsContractTest.mjs b/tests/Unit/ServerParityMessageStreamsContractTest.mjs new file mode 100644 index 000000000..f98be161b --- /dev/null +++ b/tests/Unit/ServerParityMessageStreamsContractTest.mjs @@ -0,0 +1,81 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkMessageStreams} from '../../scripts/conformance/server-parity/message-streams-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/message-streams.json', import.meta.url))); +// What published Server 2.11.1 answered for this fixture's probe. +const recorded = JSON.parse(readFileSync(new URL('../Fixtures/ServerParityExpected/message-streams-observation.json', import.meta.url))); +const model = () => structuredClone({mode: 'http', message_streams: recorded.message_streams}); +const projection = {scope: 'message_streams', first_run_events: 17, continued_run_events: 9}; +const check = observation => checkMessageStreams(fixture, observation, 'probe-5'); +const stream = (state, step, name) => state[step].response.streams.find(entry => entry.stream_name === name); + +test('published PHP message streams pass', () => assert.deepStrictEqual(check(model()), projection)); +test('embedded explicitly declares HTTP message streams inapplicable', () => assert.deepStrictEqual(check( + {mode: 'embedded', message_streams: {applicable: false, reason: 'embedded_has_no_http_message_streams'}}), projection)); +test('the recorded observation keeps the published reference', () => { + assert.match(recorded.recorded_from, /^published Server 2\.11\.1 /); + assert.equal(recorded.message_streams.workflow_id, 'probe-5-ms'); +}); + +for (const [label, mutate] of [ + ['a missing observation', s => { for (const key of Object.keys(s)) delete s[key]; }], + ['no contract in cluster info', s => s.cluster_contract = null], + ['another contract version', s => s.cluster_contract.version = 2], + ['the capability as a plain flag', s => s.worker_capability = false], + ['an unsupported capability', s => s.worker_capability.supported = false], + ['a stream before any append', s => s.list_before.response.count = 1], + ['an unknown stream answered', s => s.show_before = {status: 200, response: {workflow_id: s.workflow_id, stream: null}}], + ['a bad stream name served', s => s.show_bad_name = {status: 404, response: {reason: 'message_stream_not_found'}}], + ['an unknown workflow listed', s => s.list_unknown = {status: 200, response: {workflow_id: 'x', count: 0, streams: []}}], + ['a wait ahead of the cursor accepted', s => s.complete1_refusals.wait_ahead = s.complete1], + ['a cursor past the stream accepted', s => s.complete1_refusals.cursor_ahead.status = 200], + ['two waits on one stream accepted', s => s.complete1_refusals.repeated_wait.response.errors = {}], + ['stream metadata before protocol 1.15 accepted', s => s.complete1_refusals.old_protocol.status = 200], + ['the wait not recorded', s => stream(s, 'list_waiting', 'orders').pending_wait = null], + ['a wait time without microseconds', s => stream(s, 'list_waiting', 'orders').pending_wait.since = '2026-10-23T18:00:00Z'], + ['the first message at position 0', s => s.appends.m1.response.position = 0], + ['an accepted append without its run', s => delete s.appends.m1.response.run_id], + ['a duplicate appended again', s => s.appends.m1_duplicate = {status: 202, response: {...s.appends.m2_sdk.response, message_id: 'm1', position: 3}}], + ['a conflicting payload accepted', s => Object.assign(s.appends.m1_conflict, {status: 200})], + ['a missing message id accepted', s => s.appends.no_id.status = 202], + ['a long message id accepted', s => s.appends.long_id.status = 202], + ['input that is not a list accepted', s => s.appends.bad_input.status = 202], + ['an append to an unknown workflow with the service shape', s => s.appends.unknown_workflow.response.duplicate = false], + ['the runtime signal accepted from a client', s => s.appends.reserved_signal = {status: 202, response: {outcome: 'signal_received'}}], + ['a refused stream name counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 5], + ['a malformed append not counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 3], + ['a raw refused message id kept', s => stream(s, 'list_after_appends', 'orders').last_input_message_id = 'bad id!'], + ['a wait kept after an append', s => { stream(s, 'list_after_appends', 'orders').pending_wait = stream(s, 'list_waiting', 'orders').pending_wait; }], + ['streams listed out of name order', s => s.list_after_appends.response.streams.reverse()], + ['a show unlike the list', s => s.show_after_appends.response.stream.duplicate_count = 0], + ['messages delivered out of order', s => s.poll2.history.reverse()], + ['two messages in one signal', s => s.poll2.history[0].deliveries.push(s.poll2.history[1].deliveries[0])], + ['a payload delivered by reference', s => s.poll2.history[0].deliveries[0].payload_envelope = ['codec', 'external_storage']], + ['a negative cursor accepted', s => s.complete2_refusals.negative.status = 200], + ['a wait behind the new cursor accepted', s => s.complete2_refusals.wait_behind.status = 200], + ['the next message task not reported', s => s.complete2.response.created_task_ids = []], + ['no stream cursor event', s => s.history_after2 = s.history_after2.filter(event => event.stream_key !== 'orders')], + ['the stream cursor before the task completed', s => { + s.history_after2.find(event => event.stream_key === 'orders').task_status = 'leased'; + }], + ['the cursor not moved', s => stream(s, 'list_after2', 'orders').cursor_position = 0], + ['the third message without the cursor', s => s.appends.m3.response.cursor_position = 0], + ['the cursor not carried to the continued run', s => s.poll4.history.shift()], + ['a consumed message delivered again', s => s.poll4.history.splice(1, 0, structuredClone(s.poll2.history[0]))], + ['unconsumed messages out of append order', s => s.poll4.history.push(s.poll4.history.splice(1, 1)[0])], + ['the continuation as the same run', s => s.continued_run_id = s.run_id], + ['an append to a closed instance accepted', s => s.appends.closed.status = 202], + ['a closed duplicate answered as a duplicate', s => Object.assign(s.appends.closed_duplicate.response, {reason: null, outcome: 'duplicate'})], + ['SignalApplied for a consumed stream message', s => s.first_run_events.splice(7, 0, 'SignalApplied')], + ['unconsumed signals not rejected', s => s.first_run_events = s.first_run_events.filter(type => type !== 'SignalRejected')], + ['a continued run without its final cursor', s => s.continued_run_events.pop()], + ['diagnostics with payloads', s => { stream(s, 'list_final', 'orders').payload = [{n: 3}]; }], +]) { + test(`refuses ${label}`, () => { + const observation = model(); + mutate(observation.message_streams); + assert.throws(() => check(observation)); + }); +} diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index e855bf2c3..3fe4475d4 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -39,7 +39,7 @@ const NAMES = ['backend_capabilities', 'run_summary_projection', 'selected_run_p 'command_contract_snapshots', 'task_transport', 'activity_path', 'routing_health', 'durable_resume_paths', 'worker_compatibility', 'scheduler_role', 'long_poll_wake_acceleration']; -// PHP 2.5.14's recorded stages, with PHP's thresholds and sizes. +// Published PHP's recorded stages, with PHP's thresholds and sizes. const WAITING = {runs: {total: 1, current: 1, running: 1, waiting: 1}, tasks: {open: 1, ready: 1, delayed: 1}, backlog: {delayed_tasks: 1, tasks_added_last_minute: 1, tasks_dispatched_last_minute: 1}, starts: {pending_runs: 1, pending_commands: 1}, history: {events: 2, max_event_count: 2}, @@ -179,9 +179,16 @@ test('published PHP health read after the repair its metrics caught midway passe setChecks(midRepair(observation), {task_transport: 'ok', routing_health: 'ok'}); assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); }); -test('rejects PHP health behind metrics caught mid-repair', () => { +// Health reads its backlog with its own query, after its task counts: the run +// may be projected again in between. +test('published PHP health whose backlog was read after the run was projected again passes', () => { const observation = model(); midRepair(observation); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health whose recount is overdue after metrics caught mid-repair and a dispatch count past it', () => { + const observation = model(); + setChecks(midRepair(observation), {routing_health: 'ok', durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); }); test('rejects native metrics showing a repair native never runs', () => { @@ -189,6 +196,47 @@ test('rejects native metrics showing a repair native never runs', () => { setChecks(midRepair(observation, 'rust'), {durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); }); +// The task saved as redispatched before its run is projected again +// (TaskDispatcher::markDispatched): no longer overdue, and the run still needs +// repair. Seen on #673 run 39995620421, mysql-8.0. +const dispatched = (observation, runtime = 'php') => { + const stage = midRepair(observation, runtime); + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: 0, unhealthy: 0}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: false, oldest_unhealthy_at: false}); + stage.sections.backlog.counts.unhealthy_tasks = 0; + setChecks(stage, {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}); + return stage; +}; +test('published PHP read after the redispatch and before the run is projected again passes', () => { + const observation = model(); + dispatched(observation); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects native showing a redispatch with a run still needing repair', () => { + const observation = model(); + dispatched(observation, 'rust'); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +// The tasks section read while overdue, the backlog after the redispatch. +// Seen on #673 run 39995620421, postgres-17. +test('published PHP metrics whose backlog was read after the redispatch pass', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.sections.backlog.counts.unhealthy_tasks = 0; + setChecks(stage, {task_transport: 'ok', routing_health: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health still overdue after the backlog showed the redispatch', () => { + const observation = model(); + observation.operator_metrics.stages.waiting_after_redispatch.sections.backlog.counts.unhealthy_tasks = 0; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects a PHP backlog overdue again after the tasks showed the redispatch', () => { + const observation = model(); + const stage = dispatched(observation); + stage.sections.backlog.counts.unhealthy_tasks = 1; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); test('rejects a run needing repair without a repaired task', () => { const observation = model(); const stage = midRepair(observation); @@ -201,11 +249,97 @@ test('rejects a repair warning at another stage', () => { setChecks(observation.operator_metrics.stages.waiting, {durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); }); -test('rejects a repaired task still reported overdue', () => { +// Repaired and still overdue in tasks, with no run needing repair: only a +// backlog read after the run was projected again explains it, and the health +// read after that cannot still warn about the task. +test('rejects health still overdue after a backlog read past the repair', () => { const observation = model(); observation.operator_metrics.stages.waiting_after_redispatch.sections.tasks.counts.max_repair_count = 1; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); + +// Workflow aca7928484's taskMetrics reads each value with its own query, so +// the repair loop can move on inside the tasks section. +const straddle = (observation, {runs = 0, stamp, overdue, repaired, unhealthy, backlogUnhealthy, backlogRepair, + runStamp = false, runtime = 'php'}) => { + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = runtime; + stage.sections.runs.counts.repair_needed = runs; + stage.sections.runs.timestamps.oldest_repair_needed_at = runStamp; + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: overdue, max_repair_count: repaired, unhealthy}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: stamp, oldest_unhealthy_at: stamp}); + Object.assign(stage.sections.backlog.counts, {unhealthy_tasks: backlogUnhealthy, repair_needed_runs: backlogRepair}); + return stage; +}; +// Security's counterexample on #708: runs at 0, overdue at 0, the repair +// count at 1, the recount and backlog at 2. +const securityStraddle = {stamp: true, overdue: 1, repaired: 1, unhealthy: 0, backlogUnhealthy: 0, backlogRepair: 1}; +test('published PHP tasks read across a repair inside taskMetrics pass', () => { + for (const health of [{task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}, + {task_transport: 'ok', routing_health: 'ok'}]) { + const observation = model(); + setChecks(straddle(observation, securityStraddle), health); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); + } +}); +test('rejects native tasks read across a repair', () => { + const observation = model(); + setChecks(straddle(observation, {...securityStraddle, runtime: 'rust'}), + {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); }); +test('published PHP timestamps kept from before a redispatch inside taskMetrics pass', () => { + for (const backlogRepair of [1, 0]) { + const observation = model(); + setChecks(straddle(observation, {stamp: true, overdue: 0, repaired: 1, unhealthy: 0, backlogUnhealthy: 0, backlogRepair}), + {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: backlogRepair ? 'warning' : 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); + } +}); +test('published PHP run timestamp read before the repair its count shows passes', () => { + const observation = model(); + setChecks(straddle(observation, {runs: 1, stamp: true, overdue: 1, repaired: 1, unhealthy: 1, backlogUnhealthy: 1, + backlogRepair: 1}), {durable_resume_paths: 'warning'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('published PHP health whose dispatch count was read before the redispatch passes', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {task_transport: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health whose recount is overdue after its dispatch count was not', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {routing_health: 'ok'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects a PHP recount overdue after the dispatch count showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: true, overdue: 0, repaired: 1, unhealthy: 1, backlogUnhealthy: 1, backlogRepair: 1}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a PHP backlog overdue after the tasks recount showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: true, overdue: 1, repaired: 1, unhealthy: 0, backlogUnhealthy: 1, backlogRepair: 1}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a PHP overdue count after its timestamp showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: false, overdue: 1, repaired: 0, unhealthy: 1, backlogUnhealthy: 1, backlogRepair: 0}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /repair timestamps/); +}); +test('rejects a PHP run timestamp from a repair its later reads have not reached', () => { + const observation = model(); + straddle(observation, {runStamp: true, stamp: true, overdue: 1, repaired: 0, unhealthy: 1, backlogUnhealthy: 1, + backlogRepair: 0}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /repair timestamps/); +}); +test('rejects a native timestamp without its count', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = 'rust'; + stage.sections.tasks.timestamps.oldest_dispatch_overdue_since = false; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /present exactly when/); +}); // Published PHP on MariaDB answers health with 500 once an activity exists. const mariaDbHealthDefect = (observation, runtime, stages) => { observation.operator_metrics.database_family = 'mariadb'; @@ -249,7 +383,10 @@ for (const [label, mutate] of [ ['a closed run still actionable', s => s.stages.completed.sections.command_contracts.counts.actionable_backfill_needed_runs = 1], ['history events miscounted', s => s.stages.completed.sections.history.counts.events = 7], ['a worker never counted', s => s.stages.activity_running.active_workers = 0], - ['a transport warning missed', s => s.stages.waiting_after_redispatch.checks[5].status = 'ok'], + ['a transport warning missed by native', s => { + s.stages.waiting_after_redispatch.runtime = 'rust'; + s.stages.waiting_after_redispatch.checks[5].status = 'ok'; + }], ['an unexpected warning', s => s.stages.completed.checks[9].status = 'warning'], ['a check error', s => s.stages.empty.checks[0].status = 'error'], ['a missing check', s => s.stages.empty.checks.pop()], @@ -265,3 +402,40 @@ for (const [label, mutate] of [ assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); }); } + +// Every document the redispatch stage could report, from the eight values +// that depend on the repair state: exactly those some forward-only sequence +// of PHP's per-query reads produces pass, with health read after the run was +// projected again; every other one is rejected. +test('PHP redispatch documents pass exactly when a forward-only read order produces them', () => { + const overdue = state => state <= 1; + const needsRepair = state => state === 1 || state === 2; + const document = reads => [needsRepair(reads[0]), needsRepair(reads[1]), overdue(reads[2]), overdue(reads[3]), + reads[4] >= 1, overdue(reads[5]), overdue(reads[6]), needsRepair(reads[7])].map(Number).join(''); + const forward = new Set(); + const walk = (reads, from) => { + if (reads.length === 8) { + forward.add(document(reads)); + return; + } + for (let state = from; state <= 3; state++) walk([...reads, state], state); + }; + walk([], 0); + let passed = 0; + for (let bits = 0; bits < 256; bits++) { + const [runStamp, runs, stamp, overdueTasks, repaired, unhealthy, backlogUnhealthy, backlogRepair] = + [...bits.toString(2).padStart(8, '0')].map(Number); + const observation = model(); + const stage = straddle(observation, {runStamp: runStamp === 1, runs, stamp: stamp === 1, overdue: overdueTasks, + repaired, unhealthy, backlogUnhealthy, backlogRepair}); + setChecks(stage, {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'ok'}); + const key = [runStamp, runs, stamp, overdueTasks, repaired, unhealthy, backlogUnhealthy, backlogRepair].join(''); + if (forward.has(key)) { + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection, key); + passed++; + } else { + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts|repair timestamps/, key); + } + } + assert.equal(passed, forward.size); +});