From e606ee750cffb9e7dfb698b4239c1a4e23c6e31f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 18:25:49 +0000 Subject: [PATCH 01/23] Accept published PHP's operator metrics read across its redispatch steps PHP's repair pass raises the task's repair count and projects the run in one transaction, then stamps the dispatch and projects the run again in a separate step. Operator metrics reads runs, tasks and backlog without a shared snapshot, so one response can straddle those steps. Model the four PHP states and accept reads that never go backwards in read order; native must still report one consistent state. --- ...0-23-operator-metrics-redispatch-states.md | 47 ++++++++ docs/rust-server-port.md | 4 +- .../operator-metrics-contract.mjs | 114 ++++++++++++------ ...erverParityOperatorMetricsContractTest.mjs | 41 +++++++ 4 files changed, 171 insertions(+), 35 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md diff --git a/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md new file mode 100644 index 000000000..7d55b3ea1 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md @@ -0,0 +1,47 @@ +# October 23, operator metrics after PHP redispatches an overdue task + +- **Found by.** On PR #673's parity run 39995620421, the Shared Server + fixtures failed on PostgreSQL 17 and MySQL 8.0. The diff came before + any PHP pass line, so published PHP 2.11.1 failed the + `operator-metrics` contract, not native. The failing stage was + `waiting_after_redispatch`. +- **The race in published PHP.** PHP's repair pass redispatches the + overdue ready task in two steps: + - `TaskWatchdog::recoverExistingTask` raises the task's + `repair_count` and projects the run summary, in one transaction. The + run is marked `repair_needed`. + - `TaskDispatcher::markDispatched` then stamps the dispatch, and + `refreshRunSummary` clears `repair_needed` in a separate step. + + `OperatorMetrics::snapshot` reads runs, then tasks, then activities, + then the backlog, with no shared snapshot. `HealthCheck` takes its own + snapshot after the metrics request. So one response can mix states. +- **The four PHP states.** In order, the overdue task is: + 0. overdue and unrepaired; + 1. overdue, repaired, run marked `repair_needed`; + 2. dispatched, run still marked `repair_needed`; + 3. dispatched, run summary refreshed. + + The old contract accepted states 0, 1 and 3 read consistently. It + refused state 2 (the MySQL 8.0 shape) and a backlog read after the + redispatch while the tasks were read before it (the PostgreSQL 17 + shape). +- **Contract decision.** For published PHP only, each read in the + metrics response may show the earliest state that matches it, provided + the states never go backwards in PHP's read order: runs, then tasks, + then each backlog count. The health response must be at or after the + last metrics state, and its warnings follow the same rule. A PHP + response that goes backwards, such as health still overdue after the + backlog showed the redispatch, still fails. Native runs no repair loop, + so native must still report the overdue task consistently. A native + response that shows a redispatch must show it as one settled state. +- **How it was checked.** Five new contract tests cover: + - the MySQL 8.0 shape (state 2), which passes for PHP and fails for + native; + - the PostgreSQL 17 shape (backlog after the redispatch), which passes; + - two PHP responses that go backwards, which fail. + + Both CI shapes fail against the old contract. All 39 server-parity + unit test files pass. +- **Not changed.** The fixture, its stages and the other stages' expected + values. Native's behaviour is unchanged. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 96247ce9f..b8c65ecad 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -587,7 +587,9 @@ October 21, operator metrics and health: - PHP's maintenance loop redispatches an overdue ready task, after which PHP reports it repaired, not overdue. Native runs no repair loop, so the task stays overdue until a worker claims it. The fixture accepts either - PHP state. + PHP state. Since October 23 it also accepts PHP's reads that straddle + the redispatch, provided they never go backwards; see + [the redispatch-states checkpoint](rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md). - The operator dashboards, capacity evidence, and the repair, retention, activity-timeout and payload-cleanup pass endpoints remain. diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index 69df4275e..d95d31fcf 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -70,18 +70,33 @@ const EXPECTED = { backlog: {tasks_added_last_minute: 4}, history: {events: 8, max_event_count: 8}, command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, closed_backfill_needed_runs: 1}}, }; -// PHP's maintenance loop redispatches an overdue ready task (TaskRepair), so -// past the redispatch window published PHP may already show the task repaired -// instead of overdue. Native runs no repair loop and keeps reporting it overdue. -const REPAIRED = {...EXPECTED.waiting_after_redispatch, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, - backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1}}; -const REPAIRED_WARNINGS = ['command_contract_snapshots']; -// PHP's TaskWatchdog repairs the task and projects the run before -// TaskDispatcher redispatches it and projects the run again. Until then the -// task is still overdue and the run reads as needing repair. -const REPAIRING = {...EXPECTED.waiting_after_redispatch, runs: {...waitingRun, repair_needed: 1}, - backlog: {...EXPECTED.waiting_after_redispatch.backlog, repair_needed_runs: 1}}; -const REPAIRING_WARNINGS = ['command_contract_snapshots', 'task_transport', 'routing_health', 'durable_resume_paths']; +// Past the redispatch window, published PHP's maintenance loop moves the +// overdue ready task through four states: +// 0 overdue; +// 1 repairing: TaskWatchdog::recoverExistingTask sets the repair count and +// projects the run as needing repair, in one transaction; +// 2 dispatched: TaskDispatcher::markDispatched saves the task as dispatched, +// and its run is not yet projected again; +// 3 redispatched: the run is projected again. +// The task is overdue in states 0 and 1, carries a repair count from state 1, +// and its run reads as needing repair in states 1 and 2. A stage makes many +// reads: the operator metrics document reads runs, then tasks, then the +// backlog counts, and the health request comes after it. Each read sees the +// state at its own moment, so the states it sees never go backwards in that +// order. Native runs no repair loop: it keeps the task overdue, or shows it +// redispatched once repaired. +const PHP_REPAIR_STATES = [0, 1, 2, 3]; +const overdueAt = state => state <= 1; +const runNeedsRepairAt = state => state === 1 || state === 2; +const runsAt = state => (runNeedsRepairAt(state) ? {...waitingRun, repair_needed: 1} : waitingRun); +const tasksAt = state => ({open: 1, ready: 1, ready_due_or_delayed: 1, + ...(overdueAt(state) ? {dispatch_overdue: 1, unhealthy: 1} : {}), ...(state >= 1 ? {max_repair_count: 1} : {})}); +const backlogAt = (unhealthy, repairNeeded) => ({runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1, + ...(overdueAt(unhealthy) ? {unhealthy_tasks: 1} : {}), ...(runNeedsRepairAt(repairNeeded) ? {repair_needed_runs: 1} : {})}); +const warningsAt = (task, run) => ['command_contract_snapshots', + ...(overdueAt(task) ? ['task_transport', 'routing_health'] : []), ...(runNeedsRepairAt(run) ? ['durable_resume_paths'] : [])]; +const REDISPATCHED = {...EXPECTED.waiting_after_redispatch, runs: runsAt(3), + tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, backlog: backlogAt(3, 3)}; // Checks other than ok at each stage (the uncompared check aside). const WARNINGS = { empty: [], @@ -93,15 +108,50 @@ const WARNINGS = { completed: [], }; -// Past the redispatch window published PHP is in one of these states, in -// order: overdue, repairing, or redispatched. Metrics and health are separate -// requests, so its health may be at the state its metrics show or a later one. -// Native runs no repair loop and is held to the state its metrics show. -const REPAIR_STATES = [ - {counts: EXPECTED.waiting_after_redispatch, warnings: WARNINGS.waiting_after_redispatch}, - {counts: REPAIRING, warnings: REPAIRING_WARNINGS}, - {counts: REPAIRED, warnings: REPAIRED_WARNINGS}, -]; +const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); + +// Published PHP past the redispatch window: the expected counts and the +// warnings its health may report. The metrics reads (runs, then tasks, then +// each backlog count) take the earliest states that never go backwards and +// match. The health request sees that last state or a later one. With no +// match, the overdue state is expected so the difference is reported. +export function phpRedispatchExpectation(counts) { + const base = EXPECTED.waiting_after_redispatch; + let best = null; + for (const runs of PHP_REPAIR_STATES) { + for (const tasks of PHP_REPAIR_STATES.filter(state => state >= runs)) { + if (!isDeepStrictEqual(nonzero(counts.runs), runsAt(runs)) || !isDeepStrictEqual(nonzero(counts.tasks), tasksAt(tasks))) continue; + const later = PHP_REPAIR_STATES.filter(state => state >= tasks); + for (const unhealthy of later) { + for (const repairNeeded of later) { + if (!isDeepStrictEqual(nonzero(counts.backlog), backlogAt(unhealthy, repairNeeded))) continue; + const last = Math.max(unhealthy, repairNeeded); + if (best === null || last < best.last) best = {runs, tasks, unhealthy, repairNeeded, last}; + } + } + } + } + if (best === null) { + return {counts: {...base, tasks: tasksAt(0)}, warnings: [WARNINGS.waiting_after_redispatch]}; + } + // HealthCheck reads one snapshot after the metrics request, again runs, + // then tasks, then backlog. Task checks warn on an overdue task in tasks or + // backlog; durable resume paths warns on a run needing repair in runs or + // backlog. + const warnings = []; + for (const runs of PHP_REPAIR_STATES.filter(state => state >= best.last)) { + for (const tasks of PHP_REPAIR_STATES.filter(state => state >= runs)) { + for (const backlog of PHP_REPAIR_STATES.filter(state => state >= tasks)) { + const task = overdueAt(tasks) || overdueAt(backlog) ? 0 : 3; + const run = runNeedsRepairAt(runs) || runNeedsRepairAt(backlog) ? 1 : 3; + const list = warningsAt(task, run); + if (!warnings.some(seen => isDeepStrictEqual(seen, list))) warnings.push(list); + } + } + } + return {counts: {...base, runs: runsAt(best.runs), tasks: tasksAt(best.tasks), + backlog: backlogAt(best.unhealthy, best.repairNeeded)}, warnings}; +} function normalized(sections) { const counts = {}; @@ -152,18 +202,15 @@ function checkStage(name, stage, namespace, family) { } const counts = normalized(raw); const redispatch = name === 'waiting_after_redispatch'; - const php = redispatch && stage.runtime === 'php'; - const repaired = redispatch && counts.tasks.max_repair_count > 0; - if (repaired) delete counts.tasks.max_repair_count; - const states = redispatch ? REPAIR_STATES : [{counts: EXPECTED[name], warnings: WARNINGS[name]}]; - // A repair count means PHP repaired the task: mid-repair (PHP only) or redispatched. - const possible = !repaired ? [0] : php ? [1, 2] : [2]; - const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); - const matches = index => Object.entries(counts) - .every(([section, values]) => isDeepStrictEqual(nonzero(values), states[index].counts[section] ?? {})); - const state = possible.find(matches) ?? possible[possible.length - 1]; + let expected = {counts: EXPECTED[name], warnings: [WARNINGS[name]]}; + if (redispatch && stage.runtime === 'php') { + expected = phpRedispatchExpectation(counts); + } else if (redispatch && counts.tasks.max_repair_count > 0) { + delete counts.tasks.max_repair_count; + expected = {counts: REDISPATCHED, warnings: [warningsAt(3, 3)]}; + } for (const [section, values] of Object.entries(counts)) { - assert.deepStrictEqual(nonzero(values), states[state].counts[section] ?? {}, `${label} ${section} counts`); + assert.deepStrictEqual(nonzero(values), expected.counts[section] ?? {}, `${label} ${section} counts`); } assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, `${label} active workers`); @@ -172,8 +219,7 @@ function checkStage(name, stage, namespace, family) { const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); const warnings = statuses.filter(check => check.status !== 'ok').map(check => check.name); - const allowed = (php ? states.slice(state) : [states[state]]).map(candidate => candidate.warnings); - const settled = allowed.find(list => isDeepStrictEqual(list, warnings)) ?? states[state].warnings; + const settled = expected.warnings.find(list => isDeepStrictEqual(list, warnings)) ?? expected.warnings[0]; assert.deepStrictEqual(warnings, settled, `${label} warnings`); const warning = settled.length > 0 || stage.checks.some(check => check.status === 'warning'); assert.deepStrictEqual([stage.health_status, stage.health.status, stage.health.healthy], [200, warning ? 'warning' : 'ok', true], diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index e855bf2c3..d185863cb 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -189,6 +189,47 @@ test('rejects native metrics showing a repair native never runs', () => { setChecks(midRepair(observation, 'rust'), {durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); }); +// The task saved as redispatched before its run is projected again +// (TaskDispatcher::markDispatched): no longer overdue, and the run still needs +// repair. Seen on #673 run 39995620421, mysql-8.0. +const dispatched = (observation, runtime = 'php') => { + const stage = midRepair(observation, runtime); + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: 0, unhealthy: 0}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: false, oldest_unhealthy_at: false}); + stage.sections.backlog.counts.unhealthy_tasks = 0; + setChecks(stage, {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}); + return stage; +}; +test('published PHP read after the redispatch and before the run is projected again passes', () => { + const observation = model(); + dispatched(observation); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects native showing a redispatch with a run still needing repair', () => { + const observation = model(); + dispatched(observation, 'rust'); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +// The tasks section read while overdue, the backlog after the redispatch. +// Seen on #673 run 39995620421, postgres-17. +test('published PHP metrics whose backlog was read after the redispatch pass', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.sections.backlog.counts.unhealthy_tasks = 0; + setChecks(stage, {task_transport: 'ok', routing_health: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health still overdue after the backlog showed the redispatch', () => { + const observation = model(); + observation.operator_metrics.stages.waiting_after_redispatch.sections.backlog.counts.unhealthy_tasks = 0; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects a PHP backlog overdue again after the tasks showed the redispatch', () => { + const observation = model(); + const stage = dispatched(observation); + stage.sections.backlog.counts.unhealthy_tasks = 1; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); test('rejects a run needing repair without a repaired task', () => { const observation = model(); const stage = midRepair(observation); From 3fa5cd528895c6bb28b16b8e32065a6a25527251 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 18:30:15 +0000 Subject: [PATCH 02/23] Bring the Rust port log's current state up to October 23 --- docs/rust-server-port.md | 68 +++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index b8c65ecad..32dcdfba1 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -14,7 +14,7 @@ qualify a later tree. ## Current state -Updated October 22, 2026. +Updated October 23, 2026. - **Merged since October 20.** - **Backup-first takeover** on SQLite, PostgreSQL and MySQL/MariaDB: @@ -54,45 +54,55 @@ Updated October 22, 2026. fingerprints and all-of barriers; - cooperative waits, child cancellation and parent-close policies, continue-as-new messages, history pages, declared signals and build IDs. + - **The reference moved to published Server 2.11.1** + ([#679](https://github.com/durable-workflow/server/pull/679)), with + activity lease renewal ([#680](https://github.com/durable-workflow/server/pull/680)) + and the service catalog reads ([#681](https://github.com/durable-workflow/server/pull/681)). + - **The October 23 trains.** + [#675](https://github.com/durable-workflow/server/pull/675) covered + definition routing, worker admission, poll waits and the schedule probe; + [#678](https://github.com/durable-workflow/server/pull/678), namespace + delete; [#676](https://github.com/durable-workflow/server/pull/676), + PHP's health read during its own repair. + [#696](https://github.com/durable-workflow/server/pull/696) covered + command batches, named-namespace commands, completion checks, external + inputs, the operator surface (system metrics, Prometheus, repair and + payload cleanup passes), activity leases, deployments, priority and + fairness, standalone activities, archive and redrive. - **GHSA-6q48-6c7w-9gw2 trust remediation.** [#671](https://github.com/durable-workflow/server/pull/671) moves the Rust locks to the fixed webpki roots. [#672](https://github.com/durable-workflow/server/pull/672) pins the image's TLS toolchains and checks every trust store. - **Open.** - - [#675](https://github.com/durable-workflow/server/pull/675): train of - [#667](https://github.com/durable-workflow/server/pull/667) (definition - routing), [#669](https://github.com/durable-workflow/server/pull/669) (poll - waits), [#670](https://github.com/durable-workflow/server/pull/670) (worker - admission) and [#674](https://github.com/durable-workflow/server/pull/674) - (schedule fixture probe). - - [#676](https://github.com/durable-workflow/server/pull/676): accept - published PHP's health read during or after its own task repair. + - [#707](https://github.com/durable-workflow/server/pull/707): cluster info + payload discovery, takeover storage refusal and the operator dashboard by + workflow type. + - [#708](https://github.com/durable-workflow/server/pull/708): accept + published PHP's operator metrics read across its redispatch steps. - [#673](https://github.com/durable-workflow/server/pull/673): pip, Composer and Cargo trust in CI and conformance, in Security review. [#677](https://github.com/durable-workflow/server/issues/677) owns the remaining clients, with an explicit non-completion boundary. -- **Shared Server fixtures on main.** Embedded passes. Native differs on two - fixtures: - - **`signals-before-worker`:** native records `SignalRejected` after a - terminal event. - - **`worker-listing`:** native reports `_worker_protocol_version` in - `process_metrics`. - - In both, native follows published Server 2.10.0 while the fixtures still - expect the frozen 2.5.14 reference. Separately, the published PHP reference - sometimes differs from itself on `operator-metrics` (postgres-17) because of - its own repair window, which #676 handles. Moving the reference needs a - recorded contract decision, not an edited expectation. -- **Parity surface.** Native serves 97 of published PHP's 175 API routes. That - count compares `php artisan route:list --path=api` on this tree with the - native router. The missing routes are listed below. +- **Shared Server fixtures.** Native, embedded and published PHP 2.11.1 are + compared on SQLite, MySQL 8.0, MariaDB 10.11 and PostgreSQL 17. The known + failures come from timing races inside the published PHP reference, not + from native: + - `activity-deadline-receipts` (fixed by #696); + - `operator-metrics` at `waiting_after_redispatch` (#708). + + Each was a recorded contract decision; no expectation was edited to make + native pass. +- **Parity surface.** Native serves 125 of published PHP 2.11.1's 174 API + routes. That count compares PHP's `route:list` table + (`rust/src/runtime/php-routes.v2.11.1.json`) with the native router. The + 49 missing routes are listed below, by family. - **Next action.** - 1. Verify the Rust suites on #675 and #676, then merge them. - 2. Record the reference decision for Server 2.10.0 and re-record the affected - fixtures from that published release, so main's Shared Server fixtures run - green. - 3. Port the missing families below. + 1. Land #707 and #708. + 2. Port message streams, which all three published SDKs declare from + worker protocol 1.15. Native refuses a worker that asks for them, + with `message_streams_unavailable`. + 3. Port the remaining worker protocol families below. Packaging the native binary into the server image remains an open gate. From 41bb1e89e70c70d34d5471361de2da3bea23ae62 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 19:17:21 +0000 Subject: [PATCH 03/23] Serve message streams like Server 2.11.1 Appends, diagnostics, delivery as the runtime-reserved signal, worker cursor acknowledgements and waits on workflow task completions, and the hand-over of the cursor and unconsumed messages to a continued run, with the message-streams parity fixture recorded from published PHP 2.11.1. A signal consumed by a condition wait now records no SignalApplied unless it is declared, the instance cursor event carries its task snapshot, and a completion reports the next message task, as PHP does. Retention holds runs whose instance has inbound streams until stream cleanup is ported. --- .github/workflows/server-parity.yml | 4 +- .../2026-10-23-message-streams.md | 64 + docs/rust-server-port.md | 14 +- rust/src/runtime/continuations.rs | 10 +- rust/src/runtime/execution.rs | 89 +- rust/src/runtime/http.rs | 54 +- rust/src/runtime/message_streams.rs | 719 +++++++++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/retention.rs | 7 +- rust/src/runtime/store.rs | 223 ++- rust/tests/execution.rs | 2 + rust/tests/execution/message_streams.rs | 178 +++ .../conformance/server-parity/contract.mjs | 3 + .../message-streams-contract.mjs | 191 +++ .../server-parity/message-streams-probe.php | 163 ++ scripts/conformance/server-parity/probe.php | 6 + .../ServerParity/message-streams.json | 182 +++ .../message-streams-observation.json | 1323 +++++++++++++++++ ...ServerParityMessageStreamsContractTest.mjs | 81 + 19 files changed, 3229 insertions(+), 85 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-message-streams.md create mode 100644 rust/src/runtime/message_streams.rs create mode 100644 rust/tests/execution/message_streams.rs create mode 100644 scripts/conformance/server-parity/message-streams-contract.mjs create mode 100644 scripts/conformance/server-parity/message-streams-probe.php create mode 100644 tests/Fixtures/ServerParity/message-streams.json create mode 100644 tests/Fixtures/ServerParityExpected/message-streams-observation.json create mode 100644 tests/Unit/ServerParityMessageStreamsContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 679c1d450..c37129d0e 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -41,6 +41,7 @@ on: - tests/Unit/ServerParityPayloadCleanupContractTest.mjs - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -95,6 +96,7 @@ on: - tests/Unit/ServerParityPayloadCleanupContractTest.mjs - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1518,7 +1520,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md new file mode 100644 index 000000000..c369f493f --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md @@ -0,0 +1,64 @@ +# October 23, message streams + +- **The gap.** All three published SDKs declare `message-streams` from + worker protocol 1.15. Native refused a worker that asked for them, did not + serve the append and diagnostics routes, and ignored + `message_stream_cursors` and `message_stream_waits` on a completion. +- **How PHP 2.11.1 does it.** Learned from its source, then observed live + against the published image: + - Each accepted message is delivered to the current run as the + runtime-reserved signal `__durable_workflow_message_stream`, in the + append's transaction. The signal skips the declared-signal and argument + checks but not the pending-signal and payload ceilings. + - SDK receives open a condition wait keyed by the stream cursor. A stream + message wakes that wait. + - When a signal wakes a condition wait, PHP consumes it without + `SignalApplied`, unless the run's durable contract declares that signal. + This holds for any undeclared signal, not only stream messages. + - The stream cursor's `MessageCursorAdvanced` is recorded after the + completion, with the completed task's snapshot. + - On continue-as-new, the next run receives a payload-free cursor + checkpoint for each acknowledged stream, then every unconsumed message + again, by append order. The old run rejects the signals it had not + consumed. +- **What native does now.** + - `POST /api/workflows/{id}/message-streams/{stream}/messages`, with PHP's + validation order, refusals and diagnostics of malformed input. + - `GET /api/workflows/{id}/message-streams` and + `GET /api/workflows/{id}/message-streams/{stream}`. + - Completion cursors and waits: validated before anything changes, + recorded in the completion's transaction, handed over on continue-as-new. + - The worker capability is PHP's contract with `supported: true`, and + cluster info publishes `message_streams_contract`. + - Continue-as-new no longer refuses a run with message streams. +- **Native changes found on the way.** + - A signal consumed by a condition wait no longer records `SignalApplied` + unless it is declared, and keeps the wait's sequence. + - The instance-level `MessageCursorAdvanced` carries the task snapshot, as + PHP's does. + - A completion reports the run's next message task in + `created_task_ids`, as PHP does. +- **How it was checked.** + - The `message-streams` fixture: 15 appends, 8 refused completions, the + deliveries each worker read, both runs' full event sequences, and the + diagnostics at each step. + - Published PHP 2.11.1 and native pass it on SQLite. The probe's output + is the same size on both, and both runs' event sequences are identical. + - The contract test refuses 50 changed observations, among them a + `SignalApplied` for a consumed message, a missing next message task, a + redelivered consumed message and a missing cursor checkpoint. +- **Not done yet** (Server #325 follow-ups): + - **External payload references.** Native refuses a reference as stream + input with 501 `message_stream_external_payload_unavailable`. PHP keeps + the reference and resolves it when history reaches a worker. + - **Cleanup and upgrade.** Retention cleanup of streams + (`MessageStreamRetentionCleanup`) is not ported. Takeover of PHP data + with streams is still refused. + - **Cluster `capabilities` map.** Native has none yet, so + `capabilities.message_streams` is absent; that map is the cluster info + work. + - **History presentation, outside streams.** Task snapshots lack PHP's + `last_dispatch_attempt_at` and `last_dispatched_at`. A consumed signal's + `SignalReceived` lacks `workflow_sequence` in poll history. + - **Condition fingerprint.** Native requires `condition_definition_fingerprint` + on `open_condition_wait`; PHP accepts a wait without one. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 96247ce9f..db8edcd57 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -148,7 +148,10 @@ any other unknown path or method answers PHP's own 404 or 405 - **Run operations.** - Imports from embedded v2 and Waterline v1. - **Other families.** - - Message streams. + - Message streams are served + ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), + except external payload references as input, stream retention cleanup + and takeover of PHP data with streams. - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every @@ -265,9 +268,12 @@ contract fixture before its correction. Add a fixture for each before changing them. - **Retention of runs with external payloads or message streams.** Published PHP deletes a pruned run's external payload objects and releases - its message stream items. Native does not clean either up yet. It skips - such a run with `external_payload_cleanup_unavailable` or - `message_stream_cleanup_unavailable` and keeps its details. + its message stream items. It also compacts its instance's consumed inbound + stream items, and removes the instance's streams with its last run. Native + does not clean any of these up yet. It skips such a run, including any run + whose instance has inbound streams, with + `external_payload_cleanup_unavailable` or + `message_stream_cleanup_unavailable`, and keeps its details. - **Deeply nested JSON bodies.** Published PHP decodes JSON up to 512 levels deep. Native checks every body as PHP does before any route, but its handlers parse with serde_json's 128-level limit. A body nested 129 to 512 diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 7a5d4d62c..e8590924a 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -105,17 +105,15 @@ where } // Published PHP continues a run whatever it has open. Its activities, // timers, children and waits stay with the closed predecessor, and no - // parent-close policy applies. Streams, service calls and live - // queries are not transferred by native yet. + // parent-close policy applies. Message streams pass to the next run + // when the completion records its stream acknowledgements. Durable + // output streams, service calls and live queries are not transferred + // by native yet. for (sql, selected) in [ ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), ), - ( - "SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index fa6a57a4f..5c2772790 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -585,6 +585,89 @@ impl Runtime { Ok(view) } + delegate!(message_stream_workflow_bound(workflow_id: &str) -> Result); + delegate!(record_malformed_message(workflow_id: &str, stream: &str, message_id: &Value) -> Result<()>); + delegate!(message_stream_diagnostics(workflow_id: &str, stream: Option<&str>) -> Result>); + + /// `MessageStreamController::append`: the stream name, the workflow in + /// this namespace, the request rules and the payload envelope, in PHP's + /// order. A refused request or payload counts as malformed input. + pub(crate) async fn append_message(&self, workflow_id: &str, stream: &str, body: Value, context: Value) -> Result<(StatusCode, Value)> { + super::message_streams::check_path_name(stream)?; + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"accepted":false,"workflow_id":workflow_id,"stream_name":stream,"outcome":"rejected","reason":"instance_not_found"}), + }); + } + let body = super::laravel_rules::normalize(&body); + let blob = match self.message_payload(&body).await { + Ok(blob) => blob, + Err(error) => { + if matches!(&error, super::RuntimeError::Protocol { status, .. } | super::RuntimeError::Refused { status, .. } if *status == StatusCode::UNPROCESSABLE_ENTITY) { + self.record_malformed_message(workflow_id, stream, &body["message_id"]).await?; + } + return Err(error); + } + }; + let message_id = body["message_id"].as_str().unwrap_or_default().to_owned(); + let (codec, structural) = (self.signal_codec.clone(), self.structural); + match &*self.storage { + Storage::Sqlite(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::Postgres(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::MySql(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + } + } + + /// The request rules, then `AvroPayloadEnvelopeResolver::resolve`: no + /// input is an empty argument list, plain values are encoded, and an + /// inline envelope keeps its exact bytes. + async fn message_payload(&self, body: &Value) -> Result { + super::message_streams::check_append(body)?; + if body.get("input").is_none_or(Value::is_null) { + return super::queries::empty_arguments(self.signal_codec.clone()).await; + } + super::input_envelope::transport(body)?; + if let Some(blob) = self.plain_arguments(body).await? { + return Ok(blob); + } + if self.prepare_external(&body["input"]).await?.is_some() { + // PHP keeps the reference and resolves it for workers; native + // does not resolve stream payload references yet. + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_IMPLEMENTED, + response: json!({"accepted":false,"workflow_id":null,"message_id":body["message_id"],"outcome":"rejected", + "reason":"message_stream_external_payload_unavailable"}), + }); + } + self.decodable(&body["input"]["blob"]).await?; + super::envelope(body, "input") + } + + /// `MessageStreamController::index` and `show`. + pub(crate) async fn message_streams(&self, workflow_id: &str, stream: Option<&str>) -> Result { + if let Some(stream) = stream { + super::message_streams::check_path_name(stream)?; + } + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Workflow not found.","reason":"instance_not_found"}), + }); + } + let streams = self.message_stream_diagnostics(workflow_id, stream).await?; + match stream { + None => Ok(json!({"workflow_id":workflow_id,"count":streams.len(),"streams":streams})), + Some(stream) => match streams.into_iter().next() { + Some(found) => Ok(json!({"workflow_id":workflow_id,"stream":found})), + None => Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"workflow_id":workflow_id,"stream_name":stream,"reason":"message_stream_not_found"}), + }), + }, + } + } + /// `ActivityController::index`. pub(crate) async fn standalone_activities(&self, query: Value) -> Result { let request = super::standalone_activities::list_request(&query)?; @@ -1256,17 +1339,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } } diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 98febdb4c..3bdc13127 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -120,6 +120,12 @@ pub fn router(runtime: Runtime) -> Router { post(repair_run), ) .route("/api/workflows/{workflow_id}/debug", get(debug_current)) + .route("/api/workflows/{workflow_id}/message-streams", get(list_message_streams)) + .route("/api/workflows/{workflow_id}/message-streams/{stream_name}", get(show_message_stream)) + .route( + "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + post(append_message), + ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/debug", get(debug_run), @@ -919,6 +925,7 @@ async fn cluster() -> Json { "auth_composition_contract": auth_composition, "principal_attribution_contract": principal_attribution, "service_catalog_read_contract": service_catalog, + "message_streams_contract": super::message_streams::manifest(), "payload_codec": "avro", "qualified_for_php_takeover": false}), ) } @@ -960,6 +967,39 @@ fn present(body: &Value, field: &str) -> Option { body.get(field).filter(|value| !value.is_null()).cloned() } +/// `POST /api/workflows/{workflow_id}/message-streams/{stream_name}/messages`. +async fn append_message( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, stream)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let normalized = super::laravel_rules::normalize(&body); + let context = request.context(&body, &workflow_id, "message_stream_append", metadata(&[ + ("stream_name", Some(json!(stream))), + ("message_id", present(&normalized, "message_id")), + ("request_id", present(&normalized, "request_id")), + ])); + let (status, response) = runtime.append_message(&workflow_id, &stream, body, context).await?; + Ok((status, Json(response))) +} + +/// `GET /api/workflows/{workflow_id}/message-streams`. +async fn list_message_streams( + Extension(runtime): Extension, + Path(workflow_id): Path, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, None).await?)) +} + +/// `GET /api/workflows/{workflow_id}/message-streams/{stream_name}`. +async fn show_message_stream( + Extension(runtime): Extension, + Path((workflow_id, stream)): Path<(String, String)>, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, Some(&stream)).await?)) +} + fn signal_context(request: &CommandRequest, body: &Value, workflow_id: &str, name: &str) -> Value { request.context(body, workflow_id, "signal", metadata(&[ ("request_id", present(body, "request_id")), @@ -1853,7 +1893,9 @@ async fn complete_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { - // PHP's transport rules come before every guard, ownership included. + // PHP validates the message-stream fields first, then its transport + // rules, before every guard, ownership included. + super::message_streams::check_completion_fields(&body).map_err(worker_refusal)?; let body = super::completion_rules::checked(body).map_err(worker_refusal)?; if body["commands"].as_array().is_some_and(|commands| { commands @@ -1878,6 +1920,16 @@ async fn complete_workflow( "remediation":format!("Complete typed search-attribute updates only through server nodes advertising worker protocol {minimum} or newer.")}), }); } + if !super::message_streams::Completion::of(&body).is_empty() && !protocol_at_least(&headers, 15)? { + let minimum = super::message_streams::MINIMUM_PROTOCOL_VERSION; + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":body["workflow_task_attempt"], + "outcome":"rejected","reason":"message_streams_unavailable", + "requested_version":request_protocol(&headers)?,"minimum_protocol_version":minimum, + "remediation":format!("Send the X-Durable-Workflow-Protocol-Version header with worker protocol {minimum} or newer.")}), + }); + } let attempt = body["workflow_task_attempt"].clone(); runtime .complete_workflow(&task_id, body, Some(request.principal().clone()), headers.get("x-durable-workflow-protocol-version").and_then(|value| value.to_str().ok())) diff --git a/rust/src/runtime/message_streams.rs b/rust/src/runtime/message_streams.rs new file mode 100644 index 000000000..42e87ab1a --- /dev/null +++ b/rust/src/runtime/message_streams.rs @@ -0,0 +1,719 @@ +//! Message streams, as published Server 2.11.1 serves them +//! (`MessageStreamController`, `MessageStreamService`, +//! `MessageStreamsContract`). +//! +//! A client appends ordered messages to a named stream of a workflow +//! instance. The server assigns each its position and delivers it to the +//! current run as the runtime-reserved `__durable_workflow_message_stream` +//! signal, in the append's transaction. Workers acknowledge what they consumed +//! with `message_stream_cursors` on a workflow task completion, which records +//! `MessageCursorAdvanced`, and report open receives with +//! `message_stream_waits`. When a run continues as new, the next run receives +//! a payload-free cursor checkpoint and every unconsumed message again. +//! +//! An external payload reference as message input is refused for now: +//! published PHP keeps it by reference and resolves it when history crosses +//! to a worker, which native does not do yet. +use super::laravel_rules::{self, Rule}; +use super::store::{PreparedControlArguments, SignalWrite, Store, id, now}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::Semaphore; + +pub(super) const INTERNAL_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_SCHEMA: &str = "durable-workflow.v2.message-stream.message"; +const CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor"; +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.15"; +const MAX_BATCH: i64 = 100; +const NAME_MESSAGE: &str = "must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens."; + +/// `MessageStreamsContract::manifest`. +pub(super) fn manifest() -> Value { + json!({ + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": MINIMUM_PROTOCOL_VERSION, + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": INTERNAL_SIGNAL, + "message_schema": MESSAGE_SCHEMA, + "cursor_schema": CURSOR_SCHEMA, + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics", + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict", + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run", + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true, + }, + "batch": { + "minimum": 1, + "maximum": MAX_BATCH, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages", + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention", + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await", + }, + "conformance": { + "runtimes": ["php", "python", "rust"], + "targets": ["server", "cloud"], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order", + ], + "artifacts": "published", + }, + }) +} + +/// The worker capability: the manifest, supported. +pub(super) fn worker_capability() -> Value { + let mut capability = manifest(); + capability["supported"] = json!(true); + capability +} + +fn identifier(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) +} + +/// `/^[A-Za-z0-9._:-]{1,128}$/`. +pub(super) fn valid_name(name: &str) -> bool { + name.len() <= 128 && identifier(name) +} + +fn invalid(field: &str, message: String) -> RuntimeError { + laravel_rules::failure(vec![(field.to_owned(), vec![message])]).expect("one message") +} + +/// `validateStreamName`: a path stream name outside the pattern. +pub(super) fn check_path_name(name: &str) -> Result<()> { + if valid_name(name) { + Ok(()) + } else { + Err(invalid("stream_name", format!("The stream name {NAME_MESSAGE}"))) + } +} + +/// The append body's rules, after Laravel's normalization. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("message_id", &[Rule::Required, Rule::String, Rule::Max(191), Rule::Pattern(identifier)]), + ("input", &[Rule::Nullable, Rule::Array]), + ("request_id", &[Rule::Nullable, Rule::String, Rule::Max(255)]), +]; + +pub(super) fn check_append(body: &Value) -> Result<()> { + laravel_rules::validate(body, APPEND_RULES) +} + +/// `diagnosticMessageId`: a refused message id as diagnostics keep it. +pub(super) fn diagnostic_message_id(value: &Value) -> String { + match value.as_str() { + None => "[omitted]".into(), + Some(id) if id.len() <= 191 && identifier(id) => id.into(), + Some(id) => { + use sha2::Digest; + format!("sha256:{}", hex(sha2::Sha256::digest(id.as_bytes()))) + } + } +} + +/// The payload hash PHP stores: sha256 of `codec \0 blob`. +pub(super) fn payload_hash(codec: &str, blob: &str) -> String { + use sha2::Digest; + let mut digest = sha2::Sha256::new(); + digest.update(codec.as_bytes()); + digest.update([0]); + digest.update(blob.as_bytes()); + hex(digest.finalize()) +} + +fn hex(digest: impl AsRef<[u8]>) -> String { + digest.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The worker protocol's message-stream fields on a completion, checked as +/// PHP's first `validate` call does, before the completion's own rules. +const COMPLETION_RULES: &laravel_rules::Rules = &[ + ("message_stream_cursors", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_cursors.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_cursors.*.through_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), + ("message_stream_waits", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_waits.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_waits.*.after_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), +]; + +pub(super) fn check_completion_fields(body: &Value) -> Result<()> { + laravel_rules::validate(&laravel_rules::normalize(body), COMPLETION_RULES) +} + +/// A completion's cursor advances and pending waits, in request order. +pub(super) struct Completion { + pub(super) cursors: Vec, + pub(super) waits: Vec, +} + +impl Completion { + pub(super) fn of(body: &Value) -> Self { + let list = |field: &str| match &body[field] { + Value::Array(items) => items.clone(), + Value::Object(items) => items.values().cloned().collect(), + _ => Vec::new(), + }; + Self { cursors: list("message_stream_cursors"), waits: list("message_stream_waits") } + } + + pub(super) fn is_empty(&self) -> bool { + self.cursors.is_empty() && self.waits.is_empty() + } +} + +/// `metadataStreamName`. +fn metadata_name(item: &Value, field: &str) -> Result { + match item["stream_name"].as_str() { + Some(name) if valid_name(name) => Ok(name.to_owned()), + _ => Err(invalid(field, format!("The stream_name {NAME_MESSAGE}"))), + } +} + +fn non_negative(value: &Value) -> Option { + value.as_i64().filter(|value| *value >= 0) +} + +/// The rejection PHP's service answers for an append it cannot accept. +fn rejection(status: StatusCode, reason: &str, workflow_id: &str, stream: &str, message_id: &str, extra: Value) -> RuntimeError { + let mut response = json!({"accepted":false,"duplicate":false,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":null,"outcome":"rejected","reason":reason}); + if let Value::Object(extra) = extra { + response.as_object_mut().expect("object").extend(extra); + } + RuntimeError::Protocol { status, response } +} + +/// The value a stream delivery signal carries: one map in a list. +fn map(entries: Vec<(&str, crate::codec::Value)>) -> crate::codec::Value { + crate::codec::Value::Map(entries.into_iter().map(|(key, value)| (key.to_owned(), value)).collect::>()) +} + +/// A runtime signal's encoded arguments and command payload. +async fn runtime_arguments(codec: Arc, delivery: crate::codec::Value) -> Result { + let values = vec![delivery]; + let arguments = super::queries::encode_arguments(codec.clone(), crate::codec::Value::Array(values.clone())).await?; + let command = super::queries::encode_arguments( + codec.clone(), + map(vec![ + ("name", crate::codec::Value::String(INTERNAL_SIGNAL.into())), + ("arguments", crate::codec::Value::Array(values.clone())), + ("validation_errors", crate::codec::Value::Array(vec![])), + ]), + ) + .await?; + let value = super::queries::encode_arguments(codec, values[0].clone()).await?; + Ok(PreparedControlArguments { arguments, values, value, command }) +} + +/// `CommandContext::controlPlane`: the context of a delivery with no request. +fn control_plane_context() -> Value { + json!({"caller":{"type":"control_plane","label":"Control Plane"},"auth":{"status":"not_applicable","method":"none"}}) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `NamespaceWorkflowScope::workflowBound`. + pub(crate) async fn message_stream_workflow_bound(&self, workflow_id: &str) -> Result { + Ok(Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// The stream row, created at position zero when absent. + async fn locked_stream(&self, tx: &mut Transaction<'_, DB>, workflow_id: &str, stream: &str) -> Result { + const SELECT: &str = "SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3"; + if let Some(row) = Self::query(SELECT) + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(stream) + .fetch_optional(&mut **tx) + .await? + { + return Ok(row); + } + let at = now(); + Self::query("INSERT INTO workflow_inbound_streams(namespace,workflow_instance_id,stream_name,last_position,cursor_position,duplicate_count,malformed_count,created_at,updated_at) VALUES ($1,$2,$3,0,0,0,0,$4,$5)") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + Ok(Self::query(SELECT).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_one(&mut **tx).await?) + } + + /// The instance's current run, if it is still open. + async fn active_current_run(tx: &mut Transaction<'_, DB>, workflow_id: &str) -> Result> { + let Some(run) = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1") + .bind(workflow_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let open = matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting"); + Ok(open.then_some(run)) + } + + /// `MessageStreamService::append`, in one transaction. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn append_message( + &self, + workflow_id: &str, + stream: &str, + message_id: &str, + payload_codec: &str, + payload_blob: &str, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() { + return Err(rejection(StatusCode::NOT_FOUND, "instance_not_found", workflow_id, stream, message_id, json!({}))); + } + let Some(run) = Self::active_current_run(&mut tx, workflow_id).await? else { + return Err(rejection(StatusCode::CONFLICT, "run_not_active", workflow_id, stream, message_id, json!({}))); + }; + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let stream_id = DB::number(&row, "id")?; + let cursor = DB::number(&row, "cursor_position")?; + let hash = payload_hash(payload_codec, payload_blob); + let at = now(); + if let Some(existing) = Self::query("SELECT position,payload_hash FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).fetch_optional(&mut *tx).await? + { + let same = DB::string(&existing, "payload_hash")? == hash; + let outcome = if same { "duplicate" } else { "message_identity_conflict" }; + Self::query("UPDATE workflow_inbound_streams SET duplicate_count=duplicate_count+1,last_input_outcome=$1,last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(outcome).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + tx.commit().await?; + let response = json!({"accepted":same,"duplicate":same,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":DB::number(&existing,"position")?,"cursor_position":cursor, + "outcome":if same { "duplicate" } else { "rejected" },"reason":if same { Value::Null } else { json!("message_identity_conflict") }}); + return Ok((if same { StatusCode::OK } else { StatusCode::CONFLICT }, response)); + } + let position = DB::number(&row, "last_position")? + 1; + Self::query("INSERT INTO workflow_inbound_stream_items(stream_id,namespace,workflow_instance_id,stream_name,message_id,position,payload_codec,payload_blob,payload_hash,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)") + .bind(stream_id).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).bind(message_id).bind(position) + .bind(payload_codec).bind(payload_blob).bind(&hash).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,last_input_outcome='accepted',last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(position).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + let delivery = Self::message_delivery(stream, message_id, position, payload_codec, payload_blob); + let run_id = DB::string(&run, "id")?; + match Self::deliver_runtime_signal(&mut tx, workflow_id, delivery, context, codec, limits).await? { + Ok(command_id) => { + Self::mark_delivered(&mut tx, stream_id, message_id, &run_id, &command_id).await?; + } + Err(reason) => { + Self::query("DELETE FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,last_input_outcome='message_stream_not_supported' WHERE id=$2") + .bind(position - 1).bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + self.wake.notify_waiters(); + return Err(rejection(StatusCode::CONFLICT, "message_stream_not_supported", workflow_id, stream, message_id, + json!({"detail":reason,"remediation":"Use a PHP, Python, or Rust worker that advertises the message_streams capability."}))); + } + } + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::ACCEPTED, json!({"accepted":true,"duplicate":false,"workflow_id":workflow_id,"run_id":run_id, + "stream_name":stream,"message_id":message_id,"position":position,"cursor_position":cursor, + "outcome":"accepted","reason":null}))) + } + + /// `MessageStreamService::recordMalformed`: counts a refused append + /// against the stream of an open run, keeping no payload. + pub(crate) async fn record_malformed_message(&self, workflow_id: &str, stream: &str, message_id: &Value) -> Result<()> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() || Self::active_current_run(&mut tx, workflow_id).await?.is_none() { + return Ok(()); + } + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET malformed_count=malformed_count+1,last_input_outcome='malformed',last_input_message_id=$1,last_input_at=$2,updated_at=$3 WHERE id=$4") + .bind(diagnostic_message_id(message_id)).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + /// `MessageStreamService::diagnostics`: every stream of the instance, or + /// the named one, by name, without payloads. + pub(crate) async fn message_stream_diagnostics(&self, workflow_id: &str, stream: Option<&str>) -> Result> { + let rows = match stream { + Some(stream) => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_all(&self.pool).await?, + None => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).fetch_all(&self.pool).await?, + }; + let time = |row: &DB::Row, field: &str| -> Result { + Ok(super::task_queues::json_time(DB::optional_instant(row, field)?)) + }; + rows.iter() + .map(|row| { + let last = DB::number(row, "last_position")?; + let cursor = DB::number(row, "cursor_position")?; + let wait = match DB::optional_string(row, "waiting_run_id")? { + Some(run_id) => json!({"run_id":run_id,"after_position":DB::optional_number(row,"waiting_after_position")?, + "since":time(row,"waiting_since")?}), + None => Value::Null, + }; + Ok(json!({"stream_name":DB::string(row,"stream_name")?,"last_position":last,"cursor_position":cursor, + "pending_count":(last - cursor).max(0),"pending_wait":wait, + "duplicate_count":DB::number(row,"duplicate_count")?,"malformed_count":DB::number(row,"malformed_count")?, + "last_input_outcome":DB::optional_string(row,"last_input_outcome")?, + "last_input_message_id":DB::optional_string(row,"last_input_message_id")?, + "last_input_at":time(row,"last_input_at")?})) + }) + .collect() + } + + /// `MessageStreamService::validateCompletion`: the worker's cursor + /// advances and waits, checked before the completion changes anything. + pub(super) async fn validate_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + completion: &Completion, + ) -> Result<()> { + let mut proposed = BTreeMap::new(); + for (index, cursor) in completion.cursors.iter().enumerate() { + let field = |name: &str| format!("message_stream_cursors.{index}.{name}"); + let stream = metadata_name(cursor, &field("stream_name"))?; + if proposed.contains_key(&stream) { + return Err(invalid(&field("stream_name"), "A stream may be acknowledged only once per workflow task completion.".into())); + } + let Some(through) = non_negative(&cursor["through_position"]) else { + return Err(invalid(&field("through_position"), "The through_position must be a non-negative integer.".into())); + }; + let row = Self::query("SELECT id,last_position,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await?; + let Some(row) = row.filter(|row| DB::number(row, "last_position").is_ok_and(|last| through <= last)) else { + return Err(invalid(&field("through_position"), "The cursor cannot advance beyond the last accepted stream position.".into())); + }; + let current = DB::number(&row, "cursor_position")?; + if through > current { + let delivered = Self::scalar("SELECT COUNT(*) FROM workflow_inbound_stream_items WHERE stream_id=$1 AND position>$2 AND position<=$3 AND delivered_run_id=$4") + .bind(DB::number(&row, "id")?).bind(current).bind(through).bind(run_id).fetch_one(&mut **tx).await?; + if delivered != through - current { + return Err(invalid(&field("through_position"), "The cursor must advance contiguously through messages delivered to this workflow run.".into())); + } + } + proposed.insert(stream, current.max(through)); + } + let mut waiting = std::collections::BTreeSet::new(); + for (index, wait) in completion.waits.iter().enumerate() { + let field = |name: &str| format!("message_stream_waits.{index}.{name}"); + let stream = metadata_name(wait, &field("stream_name"))?; + if !waiting.insert(stream.clone()) { + return Err(invalid(&field("stream_name"), "A stream may have only one pending wait per workflow task completion.".into())); + } + let Some(after) = non_negative(&wait["after_position"]) else { + return Err(invalid(&field("after_position"), "The after_position must be a non-negative integer.".into())); + }; + let expected = match proposed.get(&stream) { + Some(cursor) => *cursor, + None => Self::query("SELECT cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await? + .map(|row| DB::number(&row, "cursor_position")).transpose()?.unwrap_or(0), + }; + if after != expected { + return Err(invalid(&field("after_position"), "The pending wait must start after the durable stream cursor.".into())); + } + } + Ok(()) + } + + /// `MessageStreamService::recordCompletion`, after the completion has + /// been applied in the same transaction: consumed messages, each moved + /// cursor with `MessageCursorAdvanced`, pending waits, then the hand-over + /// to a run that continued this one. + pub(super) async fn record_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + task_id: &str, + completion: &Completion, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + for cursor in &completion.cursors { + let stream = cursor["stream_name"].as_str().unwrap_or_default(); + let through = cursor["through_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + let previous = DB::number(&row, "cursor_position")?; + if through <= previous { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET consumed_run_id=$1,consumed_task_id=$2,consumed_at=$3,updated_at=$4 WHERE stream_id=$5 AND position>$6 AND position<=$7") + .bind(run_id).bind(task_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .bind(previous).bind(through).execute(&mut **tx).await?; + Self::query("UPDATE workflow_inbound_streams SET cursor_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,updated_at=$2 WHERE id=$3") + .bind(through).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?).execute(&mut **tx).await?; + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":stream,"previous_position":previous, + "new_position":through,"task":Self::task_snapshot(&task)?}), Some(task_id), None).await?; + } + for wait in &completion.waits { + let stream = wait["stream_name"].as_str().unwrap_or_default(); + let after = wait["after_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + if DB::number(&row, "last_position")? > after { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET waiting_run_id=$1,waiting_after_position=$2,waiting_since=$3,updated_at=$4 WHERE id=$5") + .bind(run_id).bind(after).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut **tx).await?; + } + let current = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1") + .bind(instance_id).fetch_one(&mut **tx).await?; + let current = DB::optional_string(¤t, "current_run_id")?; + if let Some(current) = current.filter(|current| current != run_id) { + self.carry_streams_forward(tx, instance_id, ¤t, codec, limits).await?; + } + Ok(()) + } + + /// `carryForward`: a cursor checkpoint for each acknowledged stream the + /// current run has not received, then every unconsumed message the + /// current run has not received, by append order. + async fn carry_streams_forward( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + current: &str, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + let context = control_plane_context(); + let streams = Self::query("SELECT id,stream_name,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND cursor_position>0 AND (cursor_checkpoint_run_id IS NULL OR cursor_checkpoint_run_id<>$3) ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + for stream in streams { + let checkpoint = map(vec![ + ("schema", crate::codec::Value::String(CURSOR_SCHEMA.into())), + ("stream_name", crate::codec::Value::String(DB::string(&stream, "stream_name")?)), + ("through_position", crate::codec::Value::Long(DB::number(&stream, "cursor_position")?)), + ]); + if Self::deliver_runtime_signal(tx, instance_id, checkpoint, &context, codec.clone(), limits).await?.is_err() { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_cursor_carry_failed" }); + } + Self::query("UPDATE workflow_inbound_streams SET cursor_checkpoint_run_id=$1 WHERE id=$2") + .bind(current).bind(DB::number(&stream, "id")?).execute(&mut **tx).await?; + } + loop { + let items = Self::query("SELECT stream_id,stream_name,message_id,position,payload_codec,payload_blob FROM workflow_inbound_stream_items WHERE namespace=$1 AND workflow_instance_id=$2 AND consumed_at IS NULL AND (delivered_run_id IS NULL OR delivered_run_id<>$3) ORDER BY id LIMIT 100") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + if items.is_empty() { + return Ok(()); + } + for item in items { + let (stream, message_id) = (DB::string(&item, "stream_name")?, DB::string(&item, "message_id")?); + let delivery = Self::message_delivery(&stream, &message_id, DB::number(&item, "position")?, + &DB::string(&item, "payload_codec")?, &DB::string(&item, "payload_blob")?); + match Self::deliver_runtime_signal(tx, instance_id, delivery, &context, codec.clone(), limits).await? { + Ok(command_id) => { + Self::mark_delivered(tx, DB::number(&item, "stream_id")?, &message_id, current, &command_id).await?; + } + Err(_) => { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_carry_failed" }); + } + } + } + } + } + + /// The message a stream delivers: its schema, identity, position and the + /// inline payload envelope. + fn message_delivery(stream: &str, message_id: &str, position: i64, payload_codec: &str, payload_blob: &str) -> crate::codec::Value { + use crate::codec::Value as Payload; + map(vec![ + ("schema", Payload::String(MESSAGE_SCHEMA.into())), + ("stream_name", Payload::String(stream.into())), + ("message_id", Payload::String(message_id.into())), + ("position", Payload::Long(position)), + ("payload_envelope", map(vec![ + ("codec", Payload::String(payload_codec.into())), + ("blob", Payload::String(payload_blob.into())), + ])), + ]) + } + + async fn mark_delivered(tx: &mut Transaction<'_, DB>, stream_id: i64, message_id: &str, run_id: &str, command_id: &str) -> Result<()> { + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET delivered_run_id=$1,workflow_command_id=$2,delivered_at=$3,updated_at=$4 WHERE stream_id=$5 AND message_id=$6") + .bind(run_id).bind(command_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id).bind(message_id) + .execute(&mut **tx).await?; + Ok(()) + } + + /// `WorkflowStub::attemptRuntimeSignalWithArguments` for the instance's + /// current run: no declared-signal admission or argument contract, but + /// the pending-signal and payload ceilings still apply. The accepted + /// command's id, or the refusal's reason. + async fn deliver_runtime_signal( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + delivery: crate::codec::Value, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result> { + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1") + .bind(workflow_id).fetch_one(&mut **tx).await?; + let run_id = DB::string(&instance, "current_run_id")?; + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(&run_id).fetch_one(&mut **tx).await?; + if !matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting") { + return Ok(Err("run_not_active")); + } + let prepared = runtime_arguments(codec.clone(), delivery).await?; + let command_id = id(); + let signal_id = id(); + let command_sequence = DB::number(&run, "last_command_sequence")? + 1; + let message_sequence = DB::number(&instance, "last_message_sequence")? + 1; + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: &run_id, workflow_type: &workflow_type, name: INTERNAL_SIGNAL, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context }; + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") + .bind(&run_id).fetch_one(&mut **tx).await?; + let exceeded = super::memo_limits::Exceeded::pending_signals(limits.pending_signals, pending) + .or_else(|| super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments)); + if let Some(exceeded) = exceeded { + let oversized = exceeded.kind == "payload_size"; + let (values, errors, diagnostics) = if oversized { + (vec![], json!({"arguments":[exceeded.message]}), exceeded.fields()) + } else { + (prepared.values.clone(), json!([]), json!({})) + }; + let command = super::store::prepare_rejected_command(codec.clone(), INTERNAL_SIGNAL.into(), values, + errors.clone(), diagnostics, "signal_codec_unavailable").await?; + let arguments = if oversized { super::queries::empty_arguments(codec).await? } else { prepared.arguments.clone() }; + let row_errors = if oversized { errors } else { json!([]) }; + Self::record_rejected_signal(tx, &write, None, "structural_limit_exceeded", &command, &arguments, &row_errors).await?; + return Ok(Err("structural_limit_exceeded")); + } + Self::record_accepted_signal(tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.summary_changed(&run_id); + Ok(Ok(command_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_follow_phps_pattern() { + assert!(valid_name("orders")); + assert!(valid_name("a.b_c:d-e")); + assert!(valid_name(&"x".repeat(128))); + assert!(!valid_name(&"x".repeat(129))); + assert!(!valid_name("")); + assert!(!valid_name("bad name")); + assert!(!valid_name("bad/name")); + } + + #[test] + fn refused_message_ids_are_bounded_for_diagnostics() { + assert_eq!(diagnostic_message_id(&Value::Null), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!(7)), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!("m3")), "m3"); + let long = "a".repeat(192); + assert_eq!(diagnostic_message_id(&json!(long)).len(), "sha256:".len() + 64); + assert!(diagnostic_message_id(&json!("bad id!")).starts_with("sha256:")); + } + + #[test] + fn payload_hash_covers_codec_and_blob() { + assert_ne!(payload_hash("avro", "a"), payload_hash("avro", "b")); + assert_ne!(payload_hash("avro", "ab"), payload_hash("avroa", "b")); + assert_eq!(payload_hash("avro", "x").len(), 64); + } + + #[test] + fn the_worker_capability_is_the_supported_manifest() { + let capability = worker_capability(); + assert_eq!(capability["supported"], true); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); + assert_eq!(capability["transport"]["internal_signal"], INTERNAL_SIGNAL); + assert_eq!(manifest()["batch"]["maximum"], 100); + } + + #[test] + fn completion_fields_follow_laravels_rules() { + assert!(check_completion_fields(&json!({})).is_ok()); + assert!(check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":0}]})).is_ok()); + let Err(RuntimeError::Protocol { response, .. }) = + check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":-1}]})) + else { + panic!("expected a validation failure"); + }; + assert_eq!(response["message"], "The message_stream_cursors.0.through_position field must be at least 0."); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index d60cb0473..ff9968292 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -117,6 +117,7 @@ mod service_catalog; mod sqlite; mod start_definition; mod start_request; +mod message_streams; mod storage_test; mod store; mod system_metrics; diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index 9e888fde5..b1a8e1047 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -78,9 +78,12 @@ pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the -/// skip reason. An abandoned activity's hold is read from history. +/// skip reason. An abandoned activity's hold is read from history. A run +/// whose instance has inbound message streams is held too: native does not +/// compact or remove stream state as PHP's MessageStreamRetentionCleanup +/// does yet, so it keeps the run rather than orphan the streams. pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)", 2, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 93c299d12..a0c7a0fe5 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,8 +15,21 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// One signal command's identity and sequences in its run, as both the +/// public signal route and the runtime's message-stream deliveries write it. +pub(super) struct SignalWrite<'a> { + pub(super) workflow_id: &'a str, + pub(super) run_id: &'a str, + pub(super) workflow_type: &'a str, + pub(super) name: &'a str, + pub(super) command_id: &'a str, + pub(super) signal_id: &'a str, + pub(super) command_sequence: i64, + pub(super) message_sequence: i64, + pub(super) context: &'a Value, +} /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). -const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_STREAM_SIGNAL: &str = super::message_streams::INTERNAL_SIGNAL; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; /// `nextEligibleSignal`: a run's received signals in PHP's order: command /// sequence (unsequenced last), receipt time, then id. @@ -779,25 +792,9 @@ where "signal_codec_unavailable", ) .await?; - let rejected_at = now(); - Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(outcome).bind(&workflow_type) - .bind(&prepared.command).bind(reason).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(rejected_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; - Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name) - .bind(outcome).bind(command_sequence).bind(&row_arguments).bind(DB::document(&row_errors)).bind(reason) - .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut *tx).await?; - Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) - .await?; - Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) - .await?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_rejected_signal(&mut tx, &write, outcome, reason, &prepared.command, &row_arguments, &row_errors).await?; tx.commit().await?; let mut response = json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run, "requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence, @@ -812,40 +809,88 @@ where // signal wait reservation or notification is added here. return Err(super::RuntimeError::Protocol { status, response }); } - let wait = Self::open_wait(&mut tx, ¤t_run).await?; - let mut wait_id = format!("signal-command:{command_id}"); - if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == name) { + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_accepted_signal(&mut tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.commit().await?; + self.wake.notify_waiters(); + Ok( + json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), + ) + } + + /// A refused signal's durable admission record: its rejected command, + /// its rejected signal row and the advanced sequences. No history, task + /// or wait reservation is added. + pub(super) async fn record_rejected_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + outcome: Option<&str>, + reason: &str, + command: &str, + arguments: &str, + errors: &Value, + ) -> Result<()> { + let rejected_at = now(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(outcome).bind(write.workflow_type) + .bind(command).bind(reason).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(rejected_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name) + .bind(outcome).bind(write.command_sequence).bind(arguments).bind(DB::document(errors)).bind(reason) + .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await + } + + /// `recordAcceptedSignalCommand`, `recordAcceptedSignal` and + /// `recordSignalReceivedHistory`: the accepted command, the received + /// signal reserving an open wait for its name, the advanced sequences, + /// `SignalReceived`, then the run's next message task if none is open. + pub(super) async fn record_accepted_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + command: &str, + arguments: &str, + queue: &str, + ) -> Result<()> { + let wait = Self::open_wait(tx, write.run_id).await?; + let mut wait_id = format!("signal-command:{}", write.command_id); + if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == write.name) { let reserved = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_wait_id=$2 AND status='received'") - .bind(¤t_run).bind(text(wait, "signal_wait_id")?).fetch_one(&mut *tx).await?; + .bind(write.run_id).bind(text(wait, "signal_wait_id")?).fetch_one(&mut **tx).await?; if reserved == 0 { wait_id = text(wait, "signal_wait_id")?.to_owned(); } } let accepted_at = now(); Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,command_sequence,message_sequence,accepted_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','accepted','signal_received',$5,'avro',$6,$7,$8,$9,$10)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(DB::string(&run, "workflow_type")?) - .bind(&prepared.command).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(accepted_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.workflow_type) + .bind(command).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(accepted_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,signal_wait_id,status,outcome,command_sequence,payload_codec,arguments,received_at) VALUES ($1,$2,$3,$4,$5,$6,$7,'received','signal_received',$8,'avro',$9,$10)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name).bind(&wait_id) - .bind(command_sequence).bind(&prepared.arguments).bind(DB::bind_time(accepted_at)).execute(&mut *tx).await?; + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name).bind(&wait_id) + .bind(write.command_sequence).bind(arguments).bind(DB::bind_time(accepted_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await?; + Self::append(tx, write.run_id, "SignalReceived", json!({"workflow_command_id":write.command_id,"signal_id":write.signal_id, + "workflow_instance_id":write.workflow_id,"workflow_run_id":write.run_id,"signal_name":write.name,"signal_wait_id":wait_id, + "arguments":wire(arguments),"payload_codec":"avro"}), None, Some(write.command_id)).await?; + Self::enqueue_next_message(tx, write.run_id, queue).await + } + + async fn advance_signal_sequences(tx: &mut Transaction<'_, DB>, write: &SignalWrite<'_>) -> Result<()> { Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) + .bind(write.command_sequence) + .bind(write.run_id) + .execute(&mut **tx) .await?; Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) + .bind(write.message_sequence) + .bind(write.workflow_id) + .execute(&mut **tx) .await?; - Self::append(&mut tx, ¤t_run, "SignalReceived", json!({"workflow_command_id":command_id,"signal_id":signal_id,"workflow_instance_id":workflow_id,"workflow_run_id":current_run,"signal_name":name,"signal_wait_id":wait_id,"arguments":wire(&prepared.arguments),"payload_codec":"avro"}), None, Some(&command_id)).await?; - Self::enqueue_next_message(&mut tx, ¤t_run, &DB::string(&run, "queue")?).await?; - tx.commit().await?; - self.wake.notify_waiters(); - Ok( - json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), - ) + Ok(()) } pub(super) async fn authored_sequence( @@ -1043,6 +1088,17 @@ where run_id: &str, queue: &str, ) -> Result<()> { + Self::next_message_task(tx, run_id, queue).await.map(drop) + } + + /// `PendingMessageTask::createForRun`: the run's next update or signal + /// task, when it is open, has no open workflow task and holds a message + /// that can advance it. The created task's id. + pub(super) async fn next_message_task( + tx: &mut Transaction<'_, DB>, + run_id: &str, + queue: &str, + ) -> Result> { let status = Self::query("SELECT status FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) @@ -1051,12 +1107,12 @@ where DB::string(&status, "status")?.as_str(), "pending" | "running" | "waiting" ) { - return Ok(()); + return Ok(None); } let open = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(run_id).fetch_one(&mut **tx).await?; if open != 0 { - return Ok(()); + return Ok(None); } // `nextReadyUpdate`: the earliest accepted update, unless an earlier // signal is still unapplied. That signal then goes first. @@ -1074,7 +1130,7 @@ where .await? .is_none() { - Self::create_update_task( + let task_id = Self::create_update_task( tx, run_id, queue, @@ -1082,7 +1138,7 @@ where &DB::string(&update, "workflow_command_id")?, ) .await?; - return Ok(()); + return Ok(Some(task_id)); } let (wait, condition_open, declared) = Self::signal_delivery(tx, run_id).await?; let signals = Self::query(RECEIVED_SIGNALS).bind(run_id).fetch_all(&mut **tx).await?; @@ -1092,10 +1148,10 @@ where if !Self::signal_can_advance(wait.as_ref(), condition_open, declared.as_deref(), &name, &signal_wait_id) { continue; } - Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; - break; + let task_id = Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; + return Ok(Some(task_id)); } - Ok(()) + Ok(None) } async fn apply_task_signal( @@ -1124,15 +1180,20 @@ where let wait = open.clone().filter(|wait| { wait.get("signal_name").is_none() || (wait["signal_name"] == name && wait["signal_wait_id"] == wait_id) }); - if wait.is_none() - && !Self::declared_signals(tx, run_id) - .await? - .is_some_and(|names| names.contains(&name)) - { + let declared = Self::declared_signals(tx, run_id) + .await? + .is_some_and(|names| names.contains(&name)); + if wait.is_none() && !declared { let reason = if open.is_some() { "signal_wait_mismatch" } else { "signal_wait_not_open" }; return Err(refuse(StatusCode::CONFLICT, reason)); } let direct = wait.as_ref().is_some_and(|wait| wait.get("signal_name").is_some()); + // `recordAppliedSignalForSignalResume` applies a signal its open + // signal wait matches, or one the durable contract declares, with + // `SignalApplied`. Otherwise `markConditionWaitSignalConsumed`: the + // open condition wait consumes it at the wait's sequence, without + // that event (an undeclared or runtime-reserved signal). + let applies = direct || declared; let condition_timer = match &wait { Some(wait) if !direct => Self::condition_timer(tx, run_id, wait).await?, _ => None, @@ -1156,8 +1217,9 @@ where .bind(&command_id) .execute(&mut **tx) .await?; + let recorded_sequence = if direct || !declared { sequence } else { None }; Self::query("UPDATE workflow_signal_records SET status='applied',workflow_sequence=$1,applied_at=$2,closed_at=$3 WHERE id=$4") - .bind(DB::optional_integer(sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; + .bind(DB::optional_integer(recorded_sequence)).bind(DB::bind_time(applied_at)).bind(DB::bind_time(applied_at)).bind(signal_id).execute(&mut **tx).await?; let run = Self::query( "SELECT workflow_instance_id,message_cursor_position FROM workflow_runs WHERE id=$1", ) @@ -1177,7 +1239,7 @@ where .bind(run_id) .execute(&mut **tx) .await?; - Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position}), Some(task_id), None).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position,"task":Self::task_snapshot(task)?}), Some(task_id), None).await?; if let Some(wait) = wait.as_ref().filter(|_| direct) { Self::cancel_wait_timeout(tx, run_id, wait, task_id, applied_at).await?; } @@ -1185,15 +1247,17 @@ where if direct { applied["sequence"] = json!(sequence); } - Self::append( - tx, - run_id, - "SignalApplied", - applied, - Some(task_id), - Some(&command_id), - ) - .await?; + if applies { + Self::append( + tx, + run_id, + "SignalApplied", + applied, + Some(task_id), + Some(&command_id), + ) + .await?; + } if !direct && let Some(mut satisfied) = wait && !condition_timer.as_ref().is_some_and(|(_, fired)| *fired) @@ -2106,7 +2170,9 @@ where external: super::external_resolution::PreparedCommands, worker: Option, protocol: Option<&str>, + limits: super::memo_limits::StructuralLimits, ) -> Result { + let transition_codec_for_streams = transition_codec.clone(); let commands = body["commands"] .as_array() .filter(|c| !c.is_empty() && c.len() <= 100) @@ -2262,6 +2328,17 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + // `MessageStreamService::validateCompletion`, before anything changes. + let streams = super::message_streams::Completion::of(&body); + let stream_instance = Self::query("SELECT workflow_instance_id FROM workflow_runs WHERE id=$1") + .bind(DB::string(&task, "workflow_run_id")?) + .fetch_one(&mut *tx) + .await?; + let stream_instance = DB::string(&stream_instance, "workflow_instance_id")?; + if !streams.is_empty() { + self.validate_stream_completion(&mut tx, &stream_instance, &DB::string(&task, "workflow_run_id")?, &streams) + .await?; + } // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. @@ -2654,10 +2731,14 @@ where .bind(task_id) .execute(&mut *tx) .await?; - Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; Self::resume_undelivered_cancellation(&mut tx, &run_id, &DB::string(&run, "queue")?) .await?; - Self::enqueue_next_message(&mut tx, &run_id, &DB::string(&run, "queue")?).await?; + // PHP reports the run's next message task with the tasks the + // commands created. + if let Some(next) = Self::next_message_task(&mut tx, &run_id, &DB::string(&run, "queue")?).await? { + created.push(next); + } + Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; // PHP's executor leaves every run a completed task did not close // waiting, even when the task recorded only markers or side effects. Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1 AND status IN ('pending','running')") @@ -2672,6 +2753,12 @@ where if let Some(worker) = &worker { Self::attribute_worker_terminal(&mut tx, &run_id, task_id, worker).await?; } + // `MessageStreamService::recordCompletion`: only a continued run + // changes the instance's current run, which then receives the streams. + if !streams.is_empty() || commands.iter().any(|command| command["type"] == "continue_as_new") { + self.record_stream_completion(&mut tx, &stream_instance, &run_id, task_id, &streams, transition_codec_for_streams, limits) + .await?; + } tx.summary_changed(&run_id); tx.commit().await?; self.wake.notify_waiters(); @@ -3960,7 +4047,7 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); value["supported_workflow_task_commands"] diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index c3ce551e4..551fa00cf 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -109,6 +109,8 @@ mod parent_close_policies; mod history_pages; #[path = "execution/declared_signals.rs"] mod declared_signals; +#[path = "execution/message_streams.rs"] +mod message_streams; #[path = "execution/poll_waits.rs"] mod poll_waits; #[path = "execution/child_cascade.rs"] diff --git a/rust/tests/execution/message_streams.rs b/rust/tests/execution/message_streams.rs new file mode 100644 index 000000000..254cfb6f8 --- /dev/null +++ b/rust/tests/execution/message_streams.rs @@ -0,0 +1,178 @@ +use super::*; + +// Server 2.11.1 message streams: ordered, deduplicated appends delivered to +// the current run as the runtime-reserved signal; worker acknowledgements and +// waits on workflow task completions; the hand-over to a continued run. + +const STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; + +fn stream_worker() -> Value { + json!({"worker_id":"streams","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["message_streams"]}) +} + +/// An SDK receive: a condition wait keyed by the stream cursor. +fn receive(cursor: i64) -> Value { + json!([{"type":"open_condition_wait","condition_key":format!("message-stream:orders:{cursor}"), + "condition_definition_fingerprint":format!("sha256:{}", "a".repeat(64))}]) +} + +async fn append(app: &Router, workflow: &str, stream: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/workflows/{workflow}/message-streams/{stream}/messages"), body).await +} + +async fn complete(app: &Router, task: &Value, commands: Value, extra: Value, protocol: &str) -> (StatusCode, Value) { + let mut body = completion(task, commands); + body.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + request_protocol(app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()), body, protocol).await +} + +async fn streams(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}/message-streams"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +/// The stream deliveries a history holds: (schema tail, stream, message id or cursor). +fn deliveries(history: &Value) -> Vec<(String, String, String)> { + history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "SignalReceived" && event["payload"]["signal_name"] == STREAM_SIGNAL) + .map(|event| { + let blob = event["payload"]["arguments"]["blob"].as_str().unwrap(); + let Payload::Array(values) = durable_workflow_server::codec::ValueCodec::new().unwrap().decode(blob).unwrap() else { + panic!("one argument list"); + }; + let Payload::Map(delivery) = &values[0] else { panic!("one delivery map") }; + let text = |key: &str| match delivery.get(key) { + Some(Payload::String(value)) => value.clone(), + Some(Payload::Long(value)) => value.to_string(), + other => panic!("{key}: {other:?}"), + }; + let schema = text("schema"); + let marker = if schema.ends_with(".cursor") { text("through_position") } else { text("message_id") }; + (schema.rsplit('.').next().unwrap().to_owned(), text("stream_name"), marker) + }) + .collect() +} + +fn types(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +#[tokio::test] +async fn appended_messages_reach_the_run_and_follow_it_when_it_continues() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + let started = start(&app, "streams-flow").await; + let task = poll(&app, "streams", "workflow").await; + let waits = json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}); + let (status, body) = complete(&app, &task, receive(0), waits, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let listed = streams(&app, "streams-flow").await; + assert_eq!(listed["streams"][0]["pending_wait"]["after_position"], 0, "{listed}"); + assert_eq!(listed["streams"][0]["pending_wait"]["run_id"], started["run_id"]); + + let (status, first) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &first["position"], &first["run_id"]), (StatusCode::ACCEPTED, &json!(1), &started["run_id"]), "{first}"); + let (status, again) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &again["duplicate"], &again["position"]), (StatusCode::OK, &json!(true), &json!(1)), "{again}"); + assert!(again.get("run_id").is_none(), "a duplicate names no run: {again}"); + let (status, conflict) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":9}]})).await; + assert_eq!((status, &conflict["reason"]), (StatusCode::CONFLICT, &json!("message_identity_conflict")), "{conflict}"); + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m2","input":[{"n":2}]})).await.1["position"], 2); + let listed = streams(&app, "streams-flow").await; + let orders = &listed["streams"][0]; + assert_eq!((&orders["last_position"], &orders["pending_count"], &orders["duplicate_count"]), (&json!(2), &json!(2), &json!(2)), "{listed}"); + assert!(orders["pending_wait"].is_null(), "an append clears the wait: {listed}"); + + // The condition wait wakes for the first message; the worker reads both. + let task = poll(&app, "streams", "workflow").await; + assert_eq!(deliveries(&task["history_events"]), [("message".into(), "orders".into(), "m1".into()), ("message".into(), "orders".into(), "m2".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}); + let (status, body) = complete(&app, &task, receive(1), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["created_task_ids"].as_array().unwrap().len(), 1, "the next message task is reported: {body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(!types(&history).contains(&"SignalApplied".to_owned()), "the condition wait consumes the message: {history}"); + let cursors: Vec<_> = history.as_array().unwrap().iter().filter(|event| event["event_type"] == "MessageCursorAdvanced").collect(); + assert_eq!(cursors.len(), 2, "{history}"); + assert_eq!(cursors[0]["payload"]["stream_key"], "instance:streams-flow"); + assert_eq!(cursors[0]["payload"]["task"]["status"], "leased"); + assert_eq!((&cursors[1]["payload"]["stream_key"], &cursors[1]["payload"]["new_position"]), (&json!("orders"), &json!(1))); + assert_eq!(cursors[1]["payload"]["task"]["status"], "completed"); + assert_eq!(streams(&app, "streams-flow").await["streams"][0]["cursor_position"], 1); + + // Continue as new acknowledging the second message: the next run gets the + // cursor and the third message, which was still unconsumed. + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m3","input":[{"n":3}]})).await.1["position"], 3); + let task = poll(&app, "streams", "workflow").await; + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":2}]}); + let (status, body) = complete(&app, &task, json!([{"type":"continue_as_new","arguments":envelope(Payload::Array(vec![]))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "streams", "workflow").await; + assert_ne!(task["run_id"], started["run_id"], "{task}"); + assert_eq!(deliveries(&task["history_events"]), [("cursor".into(), "orders".into(), "2".into()), ("message".into(), "orders".into(), "m3".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":3}]}); + let (status, body) = complete(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::String("done".into()))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let orders = &streams(&app, "streams-flow").await["streams"][0]; + assert_eq!((&orders["cursor_position"], &orders["pending_count"]), (&json!(3), &json!(0)), "{orders}"); + let (status, closed) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &closed["reason"]), (StatusCode::CONFLICT, &json!("run_not_active")), "{closed}"); +} + +#[tokio::test] +async fn refused_appends_and_acknowledgements_change_nothing_but_the_counts() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + start(&app, "streams-refused").await; + let task = poll(&app, "streams", "workflow").await; + for (extra, field) in [ + (json!({"message_stream_waits":[{"stream_name":"orders","after_position":1}]}), "message_stream_waits.0.after_position"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}), "message_stream_cursors.0.through_position"), + (json!({"message_stream_waits":[{"stream_name":"bad name","after_position":0}]}), "message_stream_waits.0.stream_name"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":-1}]}), "message_stream_cursors.0.through_position"), + ] { + let (status, body) = complete(&app, &task, receive(0), extra, "1.20").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert!(body["errors"].get(field).is_some(), "{field}: {body}"); + } + let (status, body) = complete(&app, &task, receive(0), json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}), "1.14").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("message_streams_unavailable")), "{body}"); + assert_eq!(streams(&app, "streams-refused").await["count"], 0, "a refused completion records no stream"); + + for (body, field) in [(json!({"input":[1]}), "message_id"), (json!({"message_id":"bad id!"}), "message_id"), + (json!({"message_id":"x".repeat(192)}), "message_id"), (json!({"message_id":"m1","input":"text"}), "input")] { + let (status, refused) = append(&app, "streams-refused", "orders", body).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert!(refused["errors"].get(field).is_some(), "{refused}"); + } + let (status, refused) = append(&app, "streams-refused", "bad%20name", json!({"message_id":"m1"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + let (status, refused) = append(&app, "streams-missing", "orders", json!({"message_id":"m1"})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::NOT_FOUND, &json!("instance_not_found")), "{refused}"); + let orders = &streams(&app, "streams-refused").await["streams"][0]; + assert_eq!((&orders["malformed_count"], &orders["last_position"]), (&json!(4), &json!(0)), "{orders}"); + // The last refused append kept a valid id; the refused stream name and + // the unknown workflow count nowhere. + assert_eq!((&orders["last_input_outcome"], &orders["last_input_message_id"]), (&json!("malformed"), &json!("m1")), "{orders}"); + let (status, refused) = request(&app, "POST", &format!("/api/workflows/streams-refused/signal/{STREAM_SIGNAL}"), json!([1])).await; + assert_eq!((status, &refused["reason"]), (StatusCode::UNPROCESSABLE_ENTITY, &json!("runtime_reserved_signal")), "{refused}"); + let (status, missing) = request(&app, "GET", "/api/workflows/streams-refused/message-streams/other", Value::Null).await; + assert_eq!((status, &missing["reason"]), (StatusCode::NOT_FOUND, &json!("message_stream_not_found")), "{missing}"); +} + +#[tokio::test] +async fn cluster_info_and_workers_see_the_contract() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, info) = request(&app, "GET", "/api/cluster/info", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{info}"); + assert_eq!(info["message_streams_contract"]["schema"], "durable-workflow.v2.message-streams.contract"); + assert_eq!(info["message_streams_contract"]["transport"]["internal_signal"], STREAM_SIGNAL); + let capability = &info["worker_protocol"]["server_capabilities"]["message_streams"]; + assert_eq!(capability["supported"], true, "{capability}"); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 85f39f1ac..a04c2efdc 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -19,6 +19,7 @@ import {checkPrometheusMetrics} from './prometheus-metrics-contract.mjs'; import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; +import {checkMessageStreams} from './message-streams-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -155,6 +156,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const payloadCleanup = fixture.payload_cleanup ? checkPayloadCleanup(fixture, observation, workflowId) : null; const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; + const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -708,6 +710,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(payloadCleanup ? {payload_cleanup: payloadCleanup} : {}), ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), + ...(messageStreams ? {message_streams: messageStreams} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/message-streams-contract.mjs b/scripts/conformance/server-parity/message-streams-contract.mjs new file mode 100644 index 000000000..829508c2a --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-contract.mjs @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; + +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/; +const MESSAGE = 'durable-workflow.v2.message-stream.message'; +const CURSOR = 'durable-workflow.v2.message-stream.cursor'; + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + assert.equal(read.status, 422, `${label} status`); + assert.deepStrictEqual(read.response, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, `${label} body`); +} + +// One stream's diagnostics: exact counts, the pending wait by run and a +// microsecond UTC time, and the last input. +function checkStream(entry, expected, runId, label) { + const {pending_wait: wait, last_input_at: inputAt, ...rest} = entry; + const {wait: expectedWait = null, input = null, ...counts} = expected; + assert.deepStrictEqual(rest, {...counts, pending_count: counts.last_position - counts.cursor_position, + duplicate_count: counts.duplicate_count ?? 0, malformed_count: counts.malformed_count ?? 0, + last_input_outcome: input?.outcome ?? null, last_input_message_id: input?.message_id ?? null}, `${label} counts`); + if (expectedWait === null) { + assert.equal(wait, null, `${label} has no pending wait`); + } else { + assert.equal(wait?.run_id, runId, `${label} pending wait run`); + assert.equal(wait?.after_position, expectedWait.after_position, `${label} pending wait position`); + assert.match(wait?.since ?? '', TIME, `${label} pending wait time`); + } + if (input === null) { + assert.equal(inputAt, null, `${label} has no input yet`); + } else { + assert.match(inputAt ?? '', TIME, `${label} last input time`); + } +} + +function checkList(read, workflowId, streams, runId, label) { + assert.equal(read.status, 200, `${label} status`); + assert.equal(read.response.workflow_id, workflowId, `${label} workflow`); + assert.equal(read.response.count, Object.keys(streams).length, `${label} count`); + assert.deepStrictEqual(read.response.streams.map(entry => entry.stream_name), Object.keys(streams), `${label} streams by name`); + for (const entry of read.response.streams) { + checkStream(entry, {stream_name: entry.stream_name, ...streams[entry.stream_name]}, runId, `${label} ${entry.stream_name}`); + } +} + +function accepted(read, workflowId, runId, stream, messageId, position, cursor, label) { + assert.equal(read.status, 202, `${label} status`); + assert.deepStrictEqual(read.response, {accepted: true, duplicate: false, workflow_id: workflowId, run_id: runId, + stream_name: stream, message_id: messageId, position, cursor_position: cursor, outcome: 'accepted', reason: null}, `${label} body`); +} + +function completed(read, runId, status, created, label) { + assert.equal(read.status, 200, `${label} status`); + const {task_id: taskId, created_task_ids: createdIds, ...rest} = read.response; + assert.equal(typeof taskId, 'string', `${label} task`); + assert.deepStrictEqual(rest, {workflow_task_attempt: 1, outcome: 'completed', recorded: true, run_id: runId, + run_status: status, reason: null}, `${label} outcome`); + assert.equal(createdIds.length, created, `${label} created tasks, the next message task included`); +} + +// A message delivery or cursor checkpoint as the worker reads it. +function delivery(stream, messageId, position, payload) { + return [{schema: MESSAGE, stream_name: stream, message_id: messageId, position, payload_envelope: ['blob', 'codec'], payload}]; +} + +// Reviewed message streams (published Server 2.11.1). +export function checkMessageStreams(fixture, observation, workflowId) { + const definition = fixture.message_streams; + const state = observation.message_streams; + // The same for every target: embedded applications have no HTTP message streams. + const projection = {scope: 'message_streams', first_run_events: definition.first_run_events.length, + continued_run_events: definition.continued_run_events.length}; + assert.ok(state && typeof state === 'object', 'actual message stream observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_message_streams'}, + 'embedded message streams explicitly inapplicable'); + return projection; + } + const id = `${workflowId}-ms`; + const {stream, other_stream: other, refusals: text, messages} = definition; + const run = state.run_id; + assert.equal(state.workflow_id, id, 'stream workflow'); + assert.deepStrictEqual(state.cluster_contract, definition.manifest, 'cluster info publishes the contract'); + assert.deepStrictEqual(state.worker_capability, {...definition.manifest, supported: true}, 'workers see the supported contract'); + + assert.deepStrictEqual(state.list_before, {status: 200, response: {workflow_id: id, count: 0, streams: []}}, 'no streams yet'); + assert.deepStrictEqual(state.show_before, {status: 404, response: {workflow_id: id, stream_name: stream, reason: 'message_stream_not_found'}}, + 'an unknown stream'); + refused(state.show_bad_name, 'stream_name', text.stream_name, 'a stream name outside the pattern'); + assert.deepStrictEqual(state.list_unknown, {status: 404, response: {message: 'Workflow not found.', reason: 'instance_not_found'}}, + 'an unknown workflow'); + assert.deepStrictEqual(state.poll1.history, [], 'nothing delivered before the first append'); + + const first = state.complete1_refusals; + refused(first.wait_ahead, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait ahead of the cursor'); + refused(first.cursor_ahead, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(first.bad_name, 'message_stream_waits.0.stream_name', text.metadata_stream_name, 'a wait stream name outside the pattern'); + refused(first.repeated_wait, 'message_stream_waits.1.stream_name', text.duplicate_wait, 'two waits on one stream'); + assert.equal(first.old_protocol.status, 409, 'stream metadata before protocol 1.15'); + const {task_id: oldTask, ...oldProtocol} = first.old_protocol.response; + assert.equal(typeof oldTask, 'string', 'the refused task'); + assert.deepStrictEqual(oldProtocol, {workflow_task_attempt: 1, outcome: 'rejected', reason: 'message_streams_unavailable', + requested_version: '1.14', minimum_protocol_version: '1.15', remediation: text.old_protocol_remediation}, 'the protocol refusal'); + completed(state.complete1, run, 'waiting', 0, 'the first completion with a wait'); + checkList(state.list_waiting, id, {[stream]: {last_position: 0, cursor_position: 0, wait: {after_position: 0}}}, run, 'the waiting stream'); + + const appends = state.appends; + accepted(appends.m1, id, run, stream, 'm1', 1, 0, 'the first message'); + assert.deepStrictEqual(appends.m1_duplicate, {status: 200, response: {accepted: true, duplicate: true, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'duplicate', reason: null}}, 'the same message again'); + assert.deepStrictEqual(appends.m1_conflict, {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'rejected', reason: 'message_identity_conflict'}}, 'the id with another payload'); + assert.deepStrictEqual(appends.m2_sdk.response, {accepted: true, duplicate: false, workflow_id: id, run_id: run, stream_name: stream, + message_id: 'm2', position: 2, cursor_position: 0, outcome: 'accepted', reason: null}, 'the published SDK append'); + refused(appends.no_id, 'message_id', text.message_id_required, 'no message id'); + refused(appends.bad_id, 'message_id', text.message_id_format, 'a message id outside the pattern'); + refused(appends.long_id, 'message_id', text.message_id_max, 'a message id too long'); + refused(appends.bad_input, 'input', text.input_array, 'input that is not a list'); + refused(appends.bad_stream, 'stream_name', text.stream_name, 'an append to a stream name outside the pattern'); + assert.deepStrictEqual(appends.unknown_workflow, {status: 404, response: {accepted: false, workflow_id: `${id}-missing`, + stream_name: stream, outcome: 'rejected', reason: 'instance_not_found'}}, 'an append to an unknown workflow'); + accepted(appends.other_stream, id, run, other, 'a1', 1, 0, 'a message without input on another stream'); + assert.equal(appends.reserved_signal.status, 422, 'the runtime signal is not a public signal'); + assert.equal(appends.reserved_signal.response.reason, 'runtime_reserved_signal', 'the reserved signal refusal'); + // Four refused appends to the stream are malformed; the last kept the + // valid id m3. The refused stream name counts nowhere. + const appended = { + [other]: {last_position: 1, cursor_position: 0, input: {outcome: 'accepted', message_id: 'a1'}}, + [stream]: {last_position: 2, cursor_position: 0, duplicate_count: 2, malformed_count: 4, input: {outcome: 'malformed', message_id: 'm3'}}, + }; + checkList(state.list_after_appends, id, appended, run, 'after the appends'); + assert.equal(state.show_after_appends.status, 200, 'one stream'); + assert.deepStrictEqual(state.show_after_appends.response, {workflow_id: id, + stream: state.list_after_appends.response.streams.find(entry => entry.stream_name === stream)}, 'one stream as listed'); + + assert.equal(state.poll2.run_id, run, 'the woken run'); + assert.deepStrictEqual(state.poll2.history, [ + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm1', 1, messages.m1)}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm2', 2, messages.m2)}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + ], 'each message is one runtime signal, in append order'); + const second = state.complete2_refusals; + refused(second.beyond, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(second.negative, 'message_stream_cursors.0.through_position', text.negative_cursor, 'a negative cursor'); + refused(second.wait_behind, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait behind the new cursor'); + // Two messages are still unconsumed, so the next message task follows. + completed(state.complete2, run, 'waiting', 1, 'the first acknowledgement'); + assert.deepStrictEqual(state.history_after2.filter(event => event.event_type === 'MessageCursorAdvanced'), [ + {event_type: 'MessageCursorAdvanced', stream_key: 'instance', previous_position: 0, new_position: 1, task_status: 'leased'}, + {event_type: 'MessageCursorAdvanced', stream_key: stream, previous_position: 0, new_position: 1, task_status: 'completed'}, + ], 'the instance cursor while the task runs, then the stream cursor once it completed'); + checkList(state.list_after2, id, {...appended, [stream]: {...appended[stream], cursor_position: 1}}, run, 'after the acknowledgement'); + + accepted(appends.m3, id, run, stream, 'm3', 3, 1, 'the third message'); + assert.equal(state.poll3.run_id, run, 'the same run again'); + assert.deepStrictEqual(state.poll3.history.filter(event => event.event_type === 'SignalReceived').at(-1), + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, 'the third message delivered'); + completed(state.complete3, run, 'completed', 1, 'continue as new'); + checkList(state.list_after_continue, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 2, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'after continue as new'); + + const continued = state.continued_run_id; + assert.equal(state.poll4.run_id, continued, 'the continued run'); + assert.notEqual(continued, run, 'a new run'); + assert.deepStrictEqual(state.poll4.history, [ + {event_type: 'SignalReceived', deliveries: [{schema: CURSOR, stream_name: stream, through_position: 2}]}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, + ], 'the continued run receives the cursor, then every unconsumed message by append order'); + completed(state.complete4, continued, 'completed', 0, 'the continued run completes'); + checkList(state.list_final, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 3, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'the final streams'); + for (const [name, messageId] of [['closed', 'm4'], ['closed_duplicate', 'm1']]) { + assert.deepStrictEqual(appends[name], {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: messageId, position: null, outcome: 'rejected', reason: 'run_not_active'}}, `${name}: the instance has no open run`); + } + + assert.deepStrictEqual(state.first_run_events, definition.first_run_events, + 'the first run: deliveries consumed by the condition waits without SignalApplied, the rest rejected at the close'); + assert.deepStrictEqual(state.continued_run_events, definition.continued_run_events, 'the continued run'); + assert.deepStrictEqual(state.first_run_streams.filter(event => event.event_type === 'MessageCursorAdvanced').map(event => event.stream_key), + ['instance', stream, 'instance', stream], 'first run cursors'); + assert.ok(!JSON.stringify(state.list_final).includes('"n":'), 'diagnostics never carry payloads'); + for (const event of [...state.first_run_streams, ...state.continued_run_streams]) { + if (event.event_type === 'SignalReceived') { + assert.equal(event.deliveries.length, 1, 'one delivery per runtime signal'); + } + } + + return projection; +} diff --git a/scripts/conformance/server-parity/message-streams-probe.php b/scripts/conformance/server-parity/message-streams-probe.php new file mode 100644 index 000000000..bd1869b16 --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-probe.php @@ -0,0 +1,163 @@ +payloadCodec(); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new MessageStreamsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, ?string $worker = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + $worker !== null ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ?? '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $append = static fn (string $streamName, array $body): array => $raw('POST', + "workflows/{$id}/message-streams/".rawurlencode($streamName).'/messages', $body); + $complete = static fn (array $task, array $body, string $protocol = '1.20'): array => $raw('POST', + "worker/workflow-tasks/{$task['task_id']}/complete", + ['lease_owner' => $task['lease_owner'] ?? $worker, 'workflow_task_attempt' => $task['workflow_task_attempt']] + $body, $protocol); + $wait = static fn (int $cursor): array => ['type' => 'open_condition_wait', + 'condition_key' => "message-stream:{$stream}:{$cursor}", + 'condition_definition_fingerprint' => 'sha256:'.hash('sha256', $definition['condition_fingerprint_source'])]; + // A stream's deliveries and cursor moves in history, decoded. + $streamEvents = static function (array $events) use ($codec, $id): array { + $rows = []; + foreach ($events as $event) { + $payload = $event['payload'] ?? []; + if (($event['event_type'] ?? null) === 'SignalReceived' + && ($payload['signal_name'] ?? null) === '__durable_workflow_message_stream') { + $deliveries = $codec->decodeEnvelope($payload['arguments']); + foreach ($deliveries as &$delivery) { + if (is_array($delivery['payload_envelope'] ?? null)) { + $envelope = $delivery['payload_envelope']; + $delivery['payload_envelope'] = array_keys($envelope); + sort($delivery['payload_envelope']); + $delivery['payload'] = $codec->decodeEnvelope($envelope); + } + } + unset($delivery); + $rows[] = ['event_type' => 'SignalReceived', 'deliveries' => $deliveries]; + } elseif (($event['event_type'] ?? null) === 'MessageCursorAdvanced') { + $key = $payload['stream_key'] ?? null; + $rows[] = ['event_type' => 'MessageCursorAdvanced', + 'stream_key' => $key === "instance:{$id}" ? 'instance' : $key, + 'previous_position' => $payload['previous_position'] ?? null, + 'new_position' => $payload['new_position'] ?? null, + 'task_status' => $payload['task']['status'] ?? null]; + } + } + + return $rows; + }; + $history = static function (string $runId) use ($raw, $id): array { + $read = $raw('GET', "workflows/{$id}/runs/{$runId}/history"); + + return $read['response']['events'] ?? []; + }; + $poll = static function () use ($client, $worker, $queue, $streamEvents): array { + $task = $client->pollWorkflowTask($worker, $queue, 0); + if (! is_array($task)) { + throw new RuntimeException('No workflow task was ready for the message stream run.'); + } + + return [$task, ['run_id' => $task['run_id'] ?? null, 'history' => $streamEvents($task['history_events'] ?? [])]]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $info = $raw('GET', 'cluster/info')['response']; + $state['cluster_contract'] = $info['message_streams_contract'] ?? null; + $state['worker_capability'] = $info['worker_protocol']['server_capabilities']['message_streams'] ?? null; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $queue, [$fixture['input']])->selectedRunId; + $state['list_before'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_before'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + $state['show_bad_name'] = $raw('GET', "workflows/{$id}/message-streams/".rawurlencode($definition['bad_stream'])); + $state['list_unknown'] = $raw('GET', "workflows/{$id}-missing/message-streams"); + $client->registerWorker($worker, $queue, [$definition['workflow_type']], []); + [$task, $state['poll1']] = $poll(); + $state['complete1_refusals'] = [ + 'wait_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 1]]]), + 'cursor_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]), + 'bad_name' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => 'bad name', 'after_position' => 0]]]), + 'repeated_wait' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [ + ['stream_name' => $stream, 'after_position' => 0], ['stream_name' => $stream, 'after_position' => 0]]]), + 'old_protocol' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]], '1.14'), + ]; + $state['complete1'] = $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]); + $state['list_waiting'] = $raw('GET', "workflows/{$id}/message-streams"); + $messages = $definition['messages']; + $state['appends'] = [ + 'm1' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_duplicate' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_conflict' => $append($stream, ['message_id' => 'm1', 'input' => $messages['conflict']]), + // The published SDK client's own append. + 'm2_sdk' => ['status' => null, 'response' => $client->appendMessageStream($id, $stream, 'm2', $messages['m2'])], + 'no_id' => $append($stream, ['input' => $messages['m3']]), + 'bad_id' => $append($stream, ['message_id' => 'bad id!', 'input' => $messages['m3']]), + 'long_id' => $append($stream, ['message_id' => str_repeat('a', $definition['long_message_id_length']), 'input' => $messages['m3']]), + 'bad_input' => $append($stream, ['message_id' => 'm3', 'input' => 'text']), + 'bad_stream' => $append($definition['bad_stream'], ['message_id' => 'm3', 'input' => $messages['m3']]), + 'unknown_workflow' => $raw('POST', "workflows/{$id}-missing/message-streams/{$stream}/messages", ['message_id' => 'm1', 'input' => [1]]), + 'other_stream' => $append($definition['other_stream'], ['message_id' => 'a1']), + 'reserved_signal' => $raw('POST', "workflows/{$id}/signal/__durable_workflow_message_stream", ['input' => [1]]), + ]; + $state['list_after_appends'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_after_appends'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + [$task, $state['poll2']] = $poll(); + $state['complete2_refusals'] = [ + 'beyond' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]), + 'negative' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => -1]]]), + 'wait_behind' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]], + 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]), + ]; + $state['complete2'] = $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]); + $state['history_after2'] = $streamEvents($history($state['run_id'])); + $state['list_after2'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['m3'] = $append($stream, ['message_id' => 'm3', 'input' => $messages['m3']]); + [$task, $state['poll3']] = $poll(); + $state['complete3'] = $complete($task, ['commands' => [['type' => 'continue_as_new', 'arguments' => $codec->envelope([$fixture['input']])]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 2]]]); + $state['list_after_continue'] = $raw('GET', "workflows/{$id}/message-streams"); + [$task, $state['poll4']] = $poll(); + $state['complete4'] = $complete($task, ['commands' => [['type' => 'complete_workflow', 'result' => $codec->envelope('done')]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]); + $state['continued_run_id'] = $state['poll4']['run_id']; + $state['list_final'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['closed'] = $append($stream, ['message_id' => 'm4', 'input' => [1]]); + $state['appends']['closed_duplicate'] = $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]); + $eventTypes = static fn (array $events): array => array_map(static fn (array $event): string => $event['event_type'], $events); + $firstRun = $history($state['run_id']); + $continuedRun = $history($state['continued_run_id']); + $state['first_run_events'] = $eventTypes($firstRun); + $state['first_run_streams'] = $streamEvents($firstRun); + $state['continued_run_events'] = $eventTypes($continuedRun); + $state['continued_run_streams'] = $streamEvents($continuedRun); + + return $state; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index b6339da20..2f4ea2339 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -437,6 +437,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $payloadCleanup = isset($fixture['payload_cleanup']) ? finishHttpPayloadCleanup($client, $fixture, $workflowId, $url) : null; $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; + $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -543,6 +544,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'payload_cleanup' => $payloadCleanup, 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, + 'message_streams' => $messageStreams, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -853,6 +855,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/payload-cleanup-probe.php'; require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; + require __DIR__.'/message-streams-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1032,6 +1035,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } + if ($mode === 'embedded' && isset($fixture['message_streams'])) { + $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/tests/Fixtures/ServerParity/message-streams.json b/tests/Fixtures/ServerParity/message-streams.json new file mode 100644 index 000000000..49b0ea7ef --- /dev/null +++ b/tests/Fixtures/ServerParity/message-streams.json @@ -0,0 +1,182 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "message-streams", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "message_streams": { + "workflow_type": "parity.v1.message_streams", + "stream": "orders", + "other_stream": "audit", + "bad_stream": "bad name!", + "condition_fingerprint_source": "message-stream:orders", + "manifest": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "messages": { + "m1": [ + { + "n": 1 + } + ], + "m2": [ + { + "n": 2 + } + ], + "m3": [ + { + "n": 3 + } + ], + "conflict": [ + { + "n": 9 + } + ] + }, + "long_message_id_length": 192, + "refusals": { + "stream_name": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "metadata_stream_name": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "message_id_required": "The message id field is required.", + "message_id_format": "The message id field format is invalid.", + "message_id_max": "The message id field must not be greater than 191 characters.", + "input_array": "The input field must be an array.", + "wait_after_cursor": "The pending wait must start after the durable stream cursor.", + "cursor_beyond": "The cursor cannot advance beyond the last accepted stream position.", + "duplicate_wait": "A stream may have only one pending wait per workflow task completion.", + "negative_cursor": "The message_stream_cursors.0.through_position field must be at least 0.", + "old_protocol_remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ] + }, + "contract": "Message streams (published Server 2.11.1, MessageStreamController and MessageStreamService over the Workflow package's runtime-reserved signal). Cluster info publishes the message streams contract and the worker capability the same contract with supported true. A client appends ordered messages to a named stream of a workflow instance: the first message is position 1, a repeated message id with the same payload is a duplicate at its original position, and with another payload a 409 message_identity_conflict. Each accepted message reaches the current run as a __durable_workflow_message_stream SignalReceived whose single argument holds the message schema, stream name, message id, position and the inline payload envelope. Refused requests are counted as malformed without keeping their payload, by a diagnostic message id. Diagnostics list every stream of the instance by name with positions, pending counts, the pending wait, duplicate and malformed counts and the last input, without payloads. A worker on protocol 1.15 or newer reports open receives with message_stream_waits and consumed messages with message_stream_cursors on a workflow task completion; the server checks both before the completion changes anything, records MessageCursorAdvanced for each moved cursor and clears the wait. An open condition wait is woken by a stream message, which it consumes without SignalApplied. When the run continues as new, the next run receives a payload-free cursor checkpoint for each acknowledged stream and every unconsumed message again, in append order; signals the old run had not consumed are rejected. Appends to a closed run are refused with run_not_active, duplicates included. Embedded applications have no HTTP message streams." +} diff --git a/tests/Fixtures/ServerParityExpected/message-streams-observation.json b/tests/Fixtures/ServerParityExpected/message-streams-observation.json new file mode 100644 index 000000000..215042c4c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/message-streams-observation.json @@ -0,0 +1,1323 @@ +{ + "recorded_from": "published Server 2.11.1 (ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102) on SQLite, workflow id probe-5", + "message_streams": { + "applicable": true, + "workflow_id": "probe-5-ms", + "cluster_contract": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "worker_capability": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + }, + "supported": true + }, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "list_before": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 0, + "streams": [] + } + }, + "show_before": { + "status": 404, + "response": { + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "reason": "message_stream_not_found" + } + }, + "show_bad_name": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "list_unknown": { + "status": 404, + "response": { + "message": "Workflow not found.", + "reason": "instance_not_found" + } + }, + "poll1": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [] + }, + "complete1_refusals": { + "wait_ahead": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + }, + "cursor_ahead": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "bad_name": { + "status": 422, + "response": { + "message": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "repeated_wait": { + "status": 422, + "response": { + "message": "A stream may have only one pending wait per workflow task completion.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + }, + "validation_errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + } + } + }, + "old_protocol": { + "status": 409, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "rejected", + "reason": "message_streams_unavailable", + "requested_version": "1.14", + "minimum_protocol_version": "1.15", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + } + } + }, + "complete1": { + "status": 200, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "list_waiting": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 1, + "streams": [ + { + "stream_name": "orders", + "last_position": 0, + "cursor_position": 0, + "pending_count": 0, + "pending_wait": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "after_position": 0, + "since": "2026-10-23T19:04:33.000000Z" + }, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": null, + "last_input_message_id": null, + "last_input_at": null + } + ] + } + }, + "appends": { + "m1": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "m1_duplicate": { + "status": 200, + "response": { + "accepted": true, + "duplicate": true, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "duplicate", + "reason": null + } + }, + "m1_conflict": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "rejected", + "reason": "message_identity_conflict" + } + }, + "m2_sdk": { + "status": null, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "no_id": { + "status": 422, + "response": { + "message": "The message id field is required.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field is required." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field is required." + ] + } + } + }, + "bad_id": { + "status": 422, + "response": { + "message": "The message id field format is invalid.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field format is invalid." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field format is invalid." + ] + } + } + }, + "long_id": { + "status": 422, + "response": { + "message": "The message id field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + } + } + }, + "bad_input": { + "status": 422, + "response": { + "message": "The input field must be an array.", + "reason": "validation_failed", + "errors": { + "input": [ + "The input field must be an array." + ] + }, + "validation_errors": { + "input": [ + "The input field must be an array." + ] + } + } + }, + "bad_stream": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "unknown_workflow": { + "status": 404, + "response": { + "accepted": false, + "workflow_id": "probe-5-ms-missing", + "stream_name": "orders", + "outcome": "rejected", + "reason": "instance_not_found" + } + }, + "other_stream": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "reserved_signal": { + "status": 422, + "response": { + "workflow_id": "probe-5-ms", + "signal_name": "__durable_workflow_message_stream", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport.", + "accepted": false, + "control_plane": { + "schema": "durable-workflow.v2.control-plane-response", + "version": 1, + "operation": "signal", + "workflow_id": "probe-5-ms", + "contract": { + "schema": "durable-workflow.v2.control-plane-response.contract", + "version": 1, + "legacy_field_policy": "reject_non_canonical", + "legacy_fields": { + "query": "query_name", + "signal": "signal_name", + "update": "update_name", + "wait_policy": "wait_for" + }, + "required_fields": [ + "workflow_id", + "operation_name", + "operation_name_field" + ], + "success_fields": [ + "outcome" + ], + "rejection_fields": [ + "workflow_id", + "run_id", + "signal_name", + "command_status", + "command_source", + "target_scope", + "outcome", + "reason", + "rejection_reason", + "rejection_category", + "message", + "command_contract_source", + "command_contract_backfill_needed", + "command_contract_backfill_available", + "declared_signals", + "signal_admission", + "retryable", + "error_id" + ], + "rejection_reasons": [ + "instance_not_found", + "historical_run_command_rejected", + "unknown_signal", + "invalid_signal_arguments", + "run_not_active", + "instance_not_started", + "selected_run_not_current", + "configured_workflow_type_invalid", + "v1_projection_read_only", + "backend_lock_pressure", + "control_plane_internal_error" + ] + }, + "operation_name": "__durable_workflow_message_stream", + "operation_name_field": "signal_name", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport." + } + } + }, + "m3": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "cursor_position": 1, + "outcome": "accepted", + "reason": null + } + }, + "closed": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m4", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + }, + "closed_duplicate": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + } + }, + "list_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "stream": { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + } + }, + "poll2": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + } + ] + }, + "complete2_refusals": { + "beyond": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "negative": { + "status": 422, + "response": { + "message": "The message_stream_cursors.0.through_position field must be at least 0.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + } + } + }, + "wait_behind": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + } + }, + "complete2": { + "status": 200, + "response": { + "task_id": "01m5n2d3eb25p3d5t00y35nhhk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [ + "01m5n2d516v62fqj8j4dzkb82t" + ], + "reason": null + } + }, + "history_after2": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + } + ], + "list_after2": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 1, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "poll3": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete3": { + "status": 200, + "response": { + "task_id": "01m5n2d516v62fqj8j4dzkb82t", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "completed", + "created_task_ids": [ + "01m5n2d5gnm3yatnb4jaqtvfnk" + ], + "reason": null + } + }, + "list_after_continue": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 2, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "poll4": { + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete4": { + "status": 200, + "response": { + "task_id": "01m5n2d5gnm3yatnb4jaqtvfnk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "continued_run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "list_final": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 3, + "pending_count": 0, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "first_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 1, + "new_position": 2, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 1, + "new_position": 2, + "task_status": "completed" + } + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 2, + "new_position": 3, + "task_status": "completed" + } + ] + } +} diff --git a/tests/Unit/ServerParityMessageStreamsContractTest.mjs b/tests/Unit/ServerParityMessageStreamsContractTest.mjs new file mode 100644 index 000000000..f98be161b --- /dev/null +++ b/tests/Unit/ServerParityMessageStreamsContractTest.mjs @@ -0,0 +1,81 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkMessageStreams} from '../../scripts/conformance/server-parity/message-streams-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/message-streams.json', import.meta.url))); +// What published Server 2.11.1 answered for this fixture's probe. +const recorded = JSON.parse(readFileSync(new URL('../Fixtures/ServerParityExpected/message-streams-observation.json', import.meta.url))); +const model = () => structuredClone({mode: 'http', message_streams: recorded.message_streams}); +const projection = {scope: 'message_streams', first_run_events: 17, continued_run_events: 9}; +const check = observation => checkMessageStreams(fixture, observation, 'probe-5'); +const stream = (state, step, name) => state[step].response.streams.find(entry => entry.stream_name === name); + +test('published PHP message streams pass', () => assert.deepStrictEqual(check(model()), projection)); +test('embedded explicitly declares HTTP message streams inapplicable', () => assert.deepStrictEqual(check( + {mode: 'embedded', message_streams: {applicable: false, reason: 'embedded_has_no_http_message_streams'}}), projection)); +test('the recorded observation keeps the published reference', () => { + assert.match(recorded.recorded_from, /^published Server 2\.11\.1 /); + assert.equal(recorded.message_streams.workflow_id, 'probe-5-ms'); +}); + +for (const [label, mutate] of [ + ['a missing observation', s => { for (const key of Object.keys(s)) delete s[key]; }], + ['no contract in cluster info', s => s.cluster_contract = null], + ['another contract version', s => s.cluster_contract.version = 2], + ['the capability as a plain flag', s => s.worker_capability = false], + ['an unsupported capability', s => s.worker_capability.supported = false], + ['a stream before any append', s => s.list_before.response.count = 1], + ['an unknown stream answered', s => s.show_before = {status: 200, response: {workflow_id: s.workflow_id, stream: null}}], + ['a bad stream name served', s => s.show_bad_name = {status: 404, response: {reason: 'message_stream_not_found'}}], + ['an unknown workflow listed', s => s.list_unknown = {status: 200, response: {workflow_id: 'x', count: 0, streams: []}}], + ['a wait ahead of the cursor accepted', s => s.complete1_refusals.wait_ahead = s.complete1], + ['a cursor past the stream accepted', s => s.complete1_refusals.cursor_ahead.status = 200], + ['two waits on one stream accepted', s => s.complete1_refusals.repeated_wait.response.errors = {}], + ['stream metadata before protocol 1.15 accepted', s => s.complete1_refusals.old_protocol.status = 200], + ['the wait not recorded', s => stream(s, 'list_waiting', 'orders').pending_wait = null], + ['a wait time without microseconds', s => stream(s, 'list_waiting', 'orders').pending_wait.since = '2026-10-23T18:00:00Z'], + ['the first message at position 0', s => s.appends.m1.response.position = 0], + ['an accepted append without its run', s => delete s.appends.m1.response.run_id], + ['a duplicate appended again', s => s.appends.m1_duplicate = {status: 202, response: {...s.appends.m2_sdk.response, message_id: 'm1', position: 3}}], + ['a conflicting payload accepted', s => Object.assign(s.appends.m1_conflict, {status: 200})], + ['a missing message id accepted', s => s.appends.no_id.status = 202], + ['a long message id accepted', s => s.appends.long_id.status = 202], + ['input that is not a list accepted', s => s.appends.bad_input.status = 202], + ['an append to an unknown workflow with the service shape', s => s.appends.unknown_workflow.response.duplicate = false], + ['the runtime signal accepted from a client', s => s.appends.reserved_signal = {status: 202, response: {outcome: 'signal_received'}}], + ['a refused stream name counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 5], + ['a malformed append not counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 3], + ['a raw refused message id kept', s => stream(s, 'list_after_appends', 'orders').last_input_message_id = 'bad id!'], + ['a wait kept after an append', s => { stream(s, 'list_after_appends', 'orders').pending_wait = stream(s, 'list_waiting', 'orders').pending_wait; }], + ['streams listed out of name order', s => s.list_after_appends.response.streams.reverse()], + ['a show unlike the list', s => s.show_after_appends.response.stream.duplicate_count = 0], + ['messages delivered out of order', s => s.poll2.history.reverse()], + ['two messages in one signal', s => s.poll2.history[0].deliveries.push(s.poll2.history[1].deliveries[0])], + ['a payload delivered by reference', s => s.poll2.history[0].deliveries[0].payload_envelope = ['codec', 'external_storage']], + ['a negative cursor accepted', s => s.complete2_refusals.negative.status = 200], + ['a wait behind the new cursor accepted', s => s.complete2_refusals.wait_behind.status = 200], + ['the next message task not reported', s => s.complete2.response.created_task_ids = []], + ['no stream cursor event', s => s.history_after2 = s.history_after2.filter(event => event.stream_key !== 'orders')], + ['the stream cursor before the task completed', s => { + s.history_after2.find(event => event.stream_key === 'orders').task_status = 'leased'; + }], + ['the cursor not moved', s => stream(s, 'list_after2', 'orders').cursor_position = 0], + ['the third message without the cursor', s => s.appends.m3.response.cursor_position = 0], + ['the cursor not carried to the continued run', s => s.poll4.history.shift()], + ['a consumed message delivered again', s => s.poll4.history.splice(1, 0, structuredClone(s.poll2.history[0]))], + ['unconsumed messages out of append order', s => s.poll4.history.push(s.poll4.history.splice(1, 1)[0])], + ['the continuation as the same run', s => s.continued_run_id = s.run_id], + ['an append to a closed instance accepted', s => s.appends.closed.status = 202], + ['a closed duplicate answered as a duplicate', s => Object.assign(s.appends.closed_duplicate.response, {reason: null, outcome: 'duplicate'})], + ['SignalApplied for a consumed stream message', s => s.first_run_events.splice(7, 0, 'SignalApplied')], + ['unconsumed signals not rejected', s => s.first_run_events = s.first_run_events.filter(type => type !== 'SignalRejected')], + ['a continued run without its final cursor', s => s.continued_run_events.pop()], + ['diagnostics with payloads', s => { stream(s, 'list_final', 'orders').payload = [{n: 3}]; }], +]) { + test(`refuses ${label}`, () => { + const observation = model(); + mutate(observation.message_streams); + assert.throws(() => check(observation)); + }); +} From 6b6757e7e20aace841935a7ebc504e8737ded896 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 20:11:00 +0000 Subject: [PATCH 04/23] Follow PHP's per-query read order in the redispatch model Published Workflow aca7928484 reads each operator metrics value with its own query, so one section can straddle a repair. The PHP-only model now matches eight state-dependent reads in their real order, including the timestamps read before their counts, and reads each health warning from its own field. Native stays strict. --- ...0-23-operator-metrics-redispatch-states.md | 66 ++++++-- .../operator-metrics-contract.mjs | 143 +++++++++++------- ...erverParityOperatorMetricsContractTest.mjs | 141 ++++++++++++++++- 3 files changed, 273 insertions(+), 77 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md index 7d55b3ea1..1a82cab36 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-operator-metrics-redispatch-states.md @@ -26,22 +26,56 @@ refused state 2 (the MySQL 8.0 shape) and a backlog read after the redispatch while the tasks were read before it (the PostgreSQL 17 shape). -- **Contract decision.** For published PHP only, each read in the - metrics response may show the earliest state that matches it, provided - the states never go backwards in PHP's read order: runs, then tasks, - then each backlog count. The health response must be at or after the - last metrics state, and its warnings follow the same rule. A PHP - response that goes backwards, such as health still overdue after the - backlog showed the redispatch, still fails. Native runs no repair loop, - so native must still report the overdue task consistently. A native - response that shows a redispatch must show it as one settled state. -- **How it was checked.** Five new contract tests cover: - - the MySQL 8.0 shape (state 2), which passes for PHP and fails for - native; - - the PostgreSQL 17 shape (backlog after the redispatch), which passes; - - two PHP responses that go backwards, which fail. +- **First model, refused in review.** The first correction (head + `3fa5cd5288`) let each section of the response show one state, with + sections never going backwards. Security's review (#708 comment + 6373392835) showed this was not enough. Published Workflow `aca7928484` + reads each value with its own query, even inside one section. So a + forward-only change can split the `tasks` section: `dispatch_overdue` + at state 0, `max_repair_count` at state 1, and `unhealthy` at state 2. + The timestamps are read before the counts, so a timestamp can also + outlive its count. +- **Contract decision.** For published PHP only, the + `waiting_after_redispatch` stage follows PHP's actual query order. The + reads that depend on the repair state are, in order: + - runs: `oldest_repair_needed_at`, then `repair_needed`; + - tasks: `oldest_dispatch_overdue_since` (with `oldest_unhealthy_at`, + which PHP builds from the same timestamps), then `dispatch_overdue`, + then `max_repair_count`, then `unhealthy`, counted again; + - backlog: `unhealthy_tasks`, then `repair_needed_runs`. - Both CI shapes fail against the old contract. All 39 server-parity - unit test files pass. + The response passes when some sequence of states that never goes + backwards in that order produces exactly its counts and timestamps. The + health request reads its own snapshot afterwards, starting at or after + the response's last read. Each health warning follows the field it + reads: + - `task_transport` follows `tasks.unhealthy`; + - `routing_health` follows `tasks.dispatch_overdue`; + - `durable_resume_paths` follows `backlog.repair_needed_runs`. + + Native runs no repair loop, so native must still report the overdue + task consistently, with each timestamp present exactly when its count + is. A native response that shows a redispatch must show it as one + settled state. +- **How it was checked.** + - Security's counterexample passes for PHP and fails for native. So + does the timestamp kept from before a redispatch. + - New tests refuse each backward step: a recount that is overdue after + the dispatch count was not; a backlog that is overdue after the + recount was not; an overdue count after its timestamp showed the + redispatch; a run timestamp from a repair the later reads have not + reached; and health whose recount is overdue after its dispatch count + was not. + - An exhaustive test covers all 256 combinations of the eight values + that depend on the state. The 37 combinations that some forward-only + order produces pass; the other 219 fail. + - Two earlier tests assumed one read per section. They now accept + health whose backlog was read after the run was projected again, and + whose dispatch count was read before the redispatch. A native + response still fails both. + - The original CI shapes (MySQL 8.0 state 2, PostgreSQL 17 backlog + after the redispatch) still pass. The server-parity unit tests pass: + 5,502 of 5,504, with one intended skip and one test that needs Docker + inside its container. - **Not changed.** The fixture, its stages and the other stages' expected values. Native's behaviour is unchanged. diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index d95d31fcf..b3b0d1d69 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -1,8 +1,8 @@ import assert from 'node:assert/strict'; import {isDeepStrictEqual} from 'node:util'; -// Reviewed operator metrics and health of one namespace, as published PHP -// 2.5.14 reports them through a run's stages. Counts must match exactly, with +// Reviewed operator metrics and health of one namespace, as published Server +// 2.11.1 reports them through a run's stages. Counts must match exactly, with // two normalizations: // - Laravel compares "now" in whole seconds, so a task made available within // the current second counts as delayed. Ready-due and delayed tasks are @@ -79,24 +79,90 @@ const EXPECTED = { // and its run is not yet projected again; // 3 redispatched: the run is projected again. // The task is overdue in states 0 and 1, carries a repair count from state 1, -// and its run reads as needing repair in states 1 and 2. A stage makes many -// reads: the operator metrics document reads runs, then tasks, then the -// backlog counts, and the health request comes after it. Each read sees the -// state at its own moment, so the states it sees never go backwards in that -// order. Native runs no repair loop: it keeps the task overdue, or shows it -// redispatched once repaired. +// and its run reads as needing repair in states 1 and 2. +// Workflow aca7928484's OperatorMetrics::snapshot reads each value with its +// own query, so one document can see several states, never going backwards +// in query order. Its reads that depend on the state, in that order: +// runMetrics: oldest_repair_needed_at, then repair_needed; +// taskMetrics: oldest_dispatch_overdue_since (oldest_unhealthy_at is built +// from the same timestamps), then dispatch_overdue, then +// max_repair_count, then unhealthy, counted again; +// backlogMetrics: unhealthy_tasks, then repair_needed_runs. +// The health request reads its own snapshot afterwards, in the same order: +// task_transport warns on tasks.unhealthy, routing_health on +// tasks.dispatch_overdue, and durable_resume_paths on +// backlog.repair_needed_runs. Native runs no repair loop: it keeps the task +// overdue, or shows it redispatched once repaired. const PHP_REPAIR_STATES = [0, 1, 2, 3]; +const PHP_READS = 8; const overdueAt = state => state <= 1; const runNeedsRepairAt = state => state === 1 || state === 2; const runsAt = state => (runNeedsRepairAt(state) ? {...waitingRun, repair_needed: 1} : waitingRun); -const tasksAt = state => ({open: 1, ready: 1, ready_due_or_delayed: 1, - ...(overdueAt(state) ? {dispatch_overdue: 1, unhealthy: 1} : {}), ...(state >= 1 ? {max_repair_count: 1} : {})}); const backlogAt = (unhealthy, repairNeeded) => ({runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1, ...(overdueAt(unhealthy) ? {unhealthy_tasks: 1} : {}), ...(runNeedsRepairAt(repairNeeded) ? {repair_needed_runs: 1} : {})}); const warningsAt = (task, run) => ['command_contract_snapshots', ...(overdueAt(task) ? ['task_transport', 'routing_health'] : []), ...(runNeedsRepairAt(run) ? ['durable_resume_paths'] : [])]; const REDISPATCHED = {...EXPECTED.waiting_after_redispatch, runs: runsAt(3), tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, backlog: backlogAt(3, 3)}; +// The timestamps a PHP read may take apart from their counts. +const PHP_STAMPED = {runs: ['oldest_repair_needed_at'], tasks: ['oldest_dispatch_overdue_since', 'oldest_unhealthy_at']}; + +// Every way `count` reads can see the states without going backwards. +function forwardReads(count, from = 0) { + if (count === 0) return [[]]; + return PHP_REPAIR_STATES.filter(state => state >= from) + .flatMap(state => forwardReads(count - 1, state).map(rest => [state, ...rest])); +} +const PHP_DOCUMENT_READS = forwardReads(PHP_READS); + +// What a PHP document whose reads saw these states reports. +function phpDocument([runStamp, runCount, taskStamp, overdue, repairs, unhealthy, backlogUnhealthy, backlogRepair]) { + return { + counts: { + runs: runsAt(runCount), + tasks: {open: 1, ready: 1, ready_due_or_delayed: 1, ...(overdueAt(overdue) ? {dispatch_overdue: 1} : {}), + ...(repairs >= 1 ? {max_repair_count: 1} : {}), ...(overdueAt(unhealthy) ? {unhealthy: 1} : {})}, + backlog: backlogAt(backlogUnhealthy, backlogRepair), + }, + timestamps: { + runs: {oldest_repair_needed_at: runNeedsRepairAt(runStamp)}, + tasks: {oldest_dispatch_overdue_since: overdueAt(taskStamp), oldest_unhealthy_at: overdueAt(taskStamp)}, + }, + }; +} + +// The warnings of a PHP health request whose reads saw these states. +const phpWarnings = reads => ['command_contract_snapshots', ...(overdueAt(reads[5]) ? ['task_transport'] : []), + ...(overdueAt(reads[3]) ? ['routing_health'] : []), ...(runNeedsRepairAt(reads[7]) ? ['durable_resume_paths'] : [])]; + +const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); + +// Published PHP past the redispatch window: the expected counts, timestamps +// and the warnings its health may report. `timestamps` holds the presence of +// the PHP_STAMPED fields. The document's reads take the states that match +// and end earliest; the health request's reads start from there. With no +// match, the reads that match the counts alone are expected, so a timestamp +// difference is reported; with none either, the overdue state is expected. +export function phpRedispatchExpectation(counts, timestamps) { + const base = EXPECTED.waiting_after_redispatch; + const seen = {runs: nonzero(counts.runs), tasks: nonzero(counts.tasks), backlog: nonzero(counts.backlog)}; + const earliest = matches => matches.reduce((best, reads) => (best === null || reads[PHP_READS - 1] < best[PHP_READS - 1] ? reads : best), null); + const countMatches = PHP_DOCUMENT_READS.filter(reads => isDeepStrictEqual(phpDocument(reads).counts, seen)); + const best = earliest(countMatches.filter(reads => isDeepStrictEqual(phpDocument(reads).timestamps, timestamps))) + ?? earliest(countMatches); + if (best === null) { + const overdue = phpDocument(Array(PHP_READS).fill(0)); + return {counts: {...base, ...overdue.counts}, timestamps: overdue.timestamps, warnings: [WARNINGS.waiting_after_redispatch]}; + } + const document = phpDocument(best); + const warnings = []; + for (const reads of forwardReads(PHP_READS, best[PHP_READS - 1])) { + const list = phpWarnings(reads); + if (!warnings.some(listed => isDeepStrictEqual(listed, list))) warnings.push(list); + } + return {counts: {...base, ...document.counts}, timestamps: document.timestamps, warnings}; +} + // Checks other than ok at each stage (the uncompared check aside). const WARNINGS = { empty: [], @@ -108,51 +174,6 @@ const WARNINGS = { completed: [], }; -const nonzero = values => Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); - -// Published PHP past the redispatch window: the expected counts and the -// warnings its health may report. The metrics reads (runs, then tasks, then -// each backlog count) take the earliest states that never go backwards and -// match. The health request sees that last state or a later one. With no -// match, the overdue state is expected so the difference is reported. -export function phpRedispatchExpectation(counts) { - const base = EXPECTED.waiting_after_redispatch; - let best = null; - for (const runs of PHP_REPAIR_STATES) { - for (const tasks of PHP_REPAIR_STATES.filter(state => state >= runs)) { - if (!isDeepStrictEqual(nonzero(counts.runs), runsAt(runs)) || !isDeepStrictEqual(nonzero(counts.tasks), tasksAt(tasks))) continue; - const later = PHP_REPAIR_STATES.filter(state => state >= tasks); - for (const unhealthy of later) { - for (const repairNeeded of later) { - if (!isDeepStrictEqual(nonzero(counts.backlog), backlogAt(unhealthy, repairNeeded))) continue; - const last = Math.max(unhealthy, repairNeeded); - if (best === null || last < best.last) best = {runs, tasks, unhealthy, repairNeeded, last}; - } - } - } - } - if (best === null) { - return {counts: {...base, tasks: tasksAt(0)}, warnings: [WARNINGS.waiting_after_redispatch]}; - } - // HealthCheck reads one snapshot after the metrics request, again runs, - // then tasks, then backlog. Task checks warn on an overdue task in tasks or - // backlog; durable resume paths warns on a run needing repair in runs or - // backlog. - const warnings = []; - for (const runs of PHP_REPAIR_STATES.filter(state => state >= best.last)) { - for (const tasks of PHP_REPAIR_STATES.filter(state => state >= runs)) { - for (const backlog of PHP_REPAIR_STATES.filter(state => state >= tasks)) { - const task = overdueAt(tasks) || overdueAt(backlog) ? 0 : 3; - const run = runNeedsRepairAt(runs) || runNeedsRepairAt(backlog) ? 1 : 3; - const list = warningsAt(task, run); - if (!warnings.some(seen => isDeepStrictEqual(seen, list))) warnings.push(list); - } - } - } - return {counts: {...base, runs: runsAt(best.runs), tasks: tasksAt(best.tasks), - backlog: backlogAt(best.unhealthy, best.repairNeeded)}, warnings}; -} - function normalized(sections) { const counts = {}; for (const [name, section] of Object.entries(sections)) { @@ -185,8 +206,12 @@ function checkStage(name, stage, namespace, family) { assert.deepStrictEqual([stage.metrics_status, stage.metrics_namespace, defect ? namespace : stage.health_namespace, stage.expected_namespace], [200, namespace, namespace, namespace], `${label} namespace`); assert.deepStrictEqual(stage.metrics_keys, ['namespace', 'operator_metrics'], `${label} document`); + const phpRedispatch = name === 'waiting_after_redispatch' && stage.runtime === 'php'; for (const [section, pairs] of Object.entries(TIMESTAMPS)) { for (const [field, count] of Object.entries(pairs)) { + // PHP's repair loop may move between a timestamp's query and its + // count's; the redispatch model checks those timestamps. + if (phpRedispatch && PHP_STAMPED[section]?.includes(field)) continue; assert.equal(stage.sections[section].timestamps[field], stage.sections[section].counts[count] > 0, `${label} ${section}.${field} present exactly when ${count} is not zero`); } @@ -203,8 +228,11 @@ function checkStage(name, stage, namespace, family) { const counts = normalized(raw); const redispatch = name === 'waiting_after_redispatch'; let expected = {counts: EXPECTED[name], warnings: [WARNINGS[name]]}; - if (redispatch && stage.runtime === 'php') { - expected = phpRedispatchExpectation(counts); + let stamps = null; + if (phpRedispatch) { + stamps = Object.fromEntries(Object.entries(PHP_STAMPED).map(([section, fields]) => + [section, Object.fromEntries(fields.map(field => [field, raw[section].timestamps[field]]))])); + expected = phpRedispatchExpectation(counts, stamps); } else if (redispatch && counts.tasks.max_repair_count > 0) { delete counts.tasks.max_repair_count; expected = {counts: REDISPATCHED, warnings: [warningsAt(3, 3)]}; @@ -212,6 +240,7 @@ function checkStage(name, stage, namespace, family) { for (const [section, values] of Object.entries(counts)) { assert.deepStrictEqual(nonzero(values), expected.counts[section] ?? {}, `${label} ${section} counts`); } + if (stamps !== null) assert.deepStrictEqual(stamps, expected.timestamps, `${label} repair timestamps`); assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, `${label} active workers`); if (defect) return counts; diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index d185863cb..3fe4475d4 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -39,7 +39,7 @@ const NAMES = ['backend_capabilities', 'run_summary_projection', 'selected_run_p 'command_contract_snapshots', 'task_transport', 'activity_path', 'routing_health', 'durable_resume_paths', 'worker_compatibility', 'scheduler_role', 'long_poll_wake_acceleration']; -// PHP 2.5.14's recorded stages, with PHP's thresholds and sizes. +// Published PHP's recorded stages, with PHP's thresholds and sizes. const WAITING = {runs: {total: 1, current: 1, running: 1, waiting: 1}, tasks: {open: 1, ready: 1, delayed: 1}, backlog: {delayed_tasks: 1, tasks_added_last_minute: 1, tasks_dispatched_last_minute: 1}, starts: {pending_runs: 1, pending_commands: 1}, history: {events: 2, max_event_count: 2}, @@ -179,9 +179,16 @@ test('published PHP health read after the repair its metrics caught midway passe setChecks(midRepair(observation), {task_transport: 'ok', routing_health: 'ok'}); assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); }); -test('rejects PHP health behind metrics caught mid-repair', () => { +// Health reads its backlog with its own query, after its task counts: the run +// may be projected again in between. +test('published PHP health whose backlog was read after the run was projected again passes', () => { const observation = model(); midRepair(observation); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health whose recount is overdue after metrics caught mid-repair and a dispatch count past it', () => { + const observation = model(); + setChecks(midRepair(observation), {routing_health: 'ok', durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); }); test('rejects native metrics showing a repair native never runs', () => { @@ -242,11 +249,97 @@ test('rejects a repair warning at another stage', () => { setChecks(observation.operator_metrics.stages.waiting, {durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); }); -test('rejects a repaired task still reported overdue', () => { +// Repaired and still overdue in tasks, with no run needing repair: only a +// backlog read after the run was projected again explains it, and the health +// read after that cannot still warn about the task. +test('rejects health still overdue after a backlog read past the repair', () => { const observation = model(); observation.operator_metrics.stages.waiting_after_redispatch.sections.tasks.counts.max_repair_count = 1; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); + +// Workflow aca7928484's taskMetrics reads each value with its own query, so +// the repair loop can move on inside the tasks section. +const straddle = (observation, {runs = 0, stamp, overdue, repaired, unhealthy, backlogUnhealthy, backlogRepair, + runStamp = false, runtime = 'php'}) => { + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = runtime; + stage.sections.runs.counts.repair_needed = runs; + stage.sections.runs.timestamps.oldest_repair_needed_at = runStamp; + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: overdue, max_repair_count: repaired, unhealthy}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: stamp, oldest_unhealthy_at: stamp}); + Object.assign(stage.sections.backlog.counts, {unhealthy_tasks: backlogUnhealthy, repair_needed_runs: backlogRepair}); + return stage; +}; +// Security's counterexample on #708: runs at 0, overdue at 0, the repair +// count at 1, the recount and backlog at 2. +const securityStraddle = {stamp: true, overdue: 1, repaired: 1, unhealthy: 0, backlogUnhealthy: 0, backlogRepair: 1}; +test('published PHP tasks read across a repair inside taskMetrics pass', () => { + for (const health of [{task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}, + {task_transport: 'ok', routing_health: 'ok'}]) { + const observation = model(); + setChecks(straddle(observation, securityStraddle), health); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); + } +}); +test('rejects native tasks read across a repair', () => { + const observation = model(); + setChecks(straddle(observation, {...securityStraddle, runtime: 'rust'}), + {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'warning'}); assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); }); +test('published PHP timestamps kept from before a redispatch inside taskMetrics pass', () => { + for (const backlogRepair of [1, 0]) { + const observation = model(); + setChecks(straddle(observation, {stamp: true, overdue: 0, repaired: 1, unhealthy: 0, backlogUnhealthy: 0, backlogRepair}), + {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: backlogRepair ? 'warning' : 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); + } +}); +test('published PHP run timestamp read before the repair its count shows passes', () => { + const observation = model(); + setChecks(straddle(observation, {runs: 1, stamp: true, overdue: 1, repaired: 1, unhealthy: 1, backlogUnhealthy: 1, + backlogRepair: 1}), {durable_resume_paths: 'warning'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('published PHP health whose dispatch count was read before the redispatch passes', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {task_transport: 'ok'}); + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects PHP health whose recount is overdue after its dispatch count was not', () => { + const observation = model(); + setChecks(observation.operator_metrics.stages.waiting_after_redispatch, {routing_health: 'ok'}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /warnings/); +}); +test('rejects a PHP recount overdue after the dispatch count showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: true, overdue: 0, repaired: 1, unhealthy: 1, backlogUnhealthy: 1, backlogRepair: 1}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a PHP backlog overdue after the tasks recount showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: true, overdue: 1, repaired: 1, unhealthy: 0, backlogUnhealthy: 1, backlogRepair: 1}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts/); +}); +test('rejects a PHP overdue count after its timestamp showed the redispatch', () => { + const observation = model(); + straddle(observation, {stamp: false, overdue: 1, repaired: 0, unhealthy: 1, backlogUnhealthy: 1, backlogRepair: 0}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /repair timestamps/); +}); +test('rejects a PHP run timestamp from a repair its later reads have not reached', () => { + const observation = model(); + straddle(observation, {runStamp: true, stamp: true, overdue: 1, repaired: 0, unhealthy: 1, backlogUnhealthy: 1, + backlogRepair: 0}); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /repair timestamps/); +}); +test('rejects a native timestamp without its count', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + stage.runtime = 'rust'; + stage.sections.tasks.timestamps.oldest_dispatch_overdue_since = false; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /present exactly when/); +}); // Published PHP on MariaDB answers health with 500 once an activity exists. const mariaDbHealthDefect = (observation, runtime, stages) => { observation.operator_metrics.database_family = 'mariadb'; @@ -290,7 +383,10 @@ for (const [label, mutate] of [ ['a closed run still actionable', s => s.stages.completed.sections.command_contracts.counts.actionable_backfill_needed_runs = 1], ['history events miscounted', s => s.stages.completed.sections.history.counts.events = 7], ['a worker never counted', s => s.stages.activity_running.active_workers = 0], - ['a transport warning missed', s => s.stages.waiting_after_redispatch.checks[5].status = 'ok'], + ['a transport warning missed by native', s => { + s.stages.waiting_after_redispatch.runtime = 'rust'; + s.stages.waiting_after_redispatch.checks[5].status = 'ok'; + }], ['an unexpected warning', s => s.stages.completed.checks[9].status = 'warning'], ['a check error', s => s.stages.empty.checks[0].status = 'error'], ['a missing check', s => s.stages.empty.checks.pop()], @@ -306,3 +402,40 @@ for (const [label, mutate] of [ assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); }); } + +// Every document the redispatch stage could report, from the eight values +// that depend on the repair state: exactly those some forward-only sequence +// of PHP's per-query reads produces pass, with health read after the run was +// projected again; every other one is rejected. +test('PHP redispatch documents pass exactly when a forward-only read order produces them', () => { + const overdue = state => state <= 1; + const needsRepair = state => state === 1 || state === 2; + const document = reads => [needsRepair(reads[0]), needsRepair(reads[1]), overdue(reads[2]), overdue(reads[3]), + reads[4] >= 1, overdue(reads[5]), overdue(reads[6]), needsRepair(reads[7])].map(Number).join(''); + const forward = new Set(); + const walk = (reads, from) => { + if (reads.length === 8) { + forward.add(document(reads)); + return; + } + for (let state = from; state <= 3; state++) walk([...reads, state], state); + }; + walk([], 0); + let passed = 0; + for (let bits = 0; bits < 256; bits++) { + const [runStamp, runs, stamp, overdueTasks, repaired, unhealthy, backlogUnhealthy, backlogRepair] = + [...bits.toString(2).padStart(8, '0')].map(Number); + const observation = model(); + const stage = straddle(observation, {runStamp: runStamp === 1, runs, stamp: stamp === 1, overdue: overdueTasks, + repaired, unhealthy, backlogUnhealthy, backlogRepair}); + setChecks(stage, {task_transport: 'ok', routing_health: 'ok', durable_resume_paths: 'ok'}); + const key = [runStamp, runs, stamp, overdueTasks, repaired, unhealthy, backlogUnhealthy, backlogRepair].join(''); + if (forward.has(key)) { + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection, key); + passed++; + } else { + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST'), /counts|repair timestamps/, key); + } + } + assert.equal(passed, forward.size); +}); From 1414996fab53155cf324fa6300f1875c69e53c20 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 20:40:20 +0000 Subject: [PATCH 05/23] Accept condition waits without a key or fingerprint like PHP Published PHP treats an open_condition_wait's condition_key and condition_definition_fingerprint as optional strings of any form, trimmed and left out when empty. Native required both and a sha256 fingerprint. The wait projection and run summary now read a keyless wait as PHP does. --- .../2026-10-23-condition-wait-fields.md | 21 +++++++ rust/src/runtime/external_wait_summary.rs | 37 ++++++++--- rust/src/runtime/store.rs | 24 ++++--- rust/src/runtime/wait_projection.rs | 3 +- rust/tests/execution.rs | 2 + .../execution/keyless_condition_waits.rs | 63 +++++++++++++++++++ 6 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md create mode 100644 rust/tests/execution/keyless_condition_waits.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md new file mode 100644 index 000000000..b96d9b562 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md @@ -0,0 +1,21 @@ +# October 23, optional condition wait fields + +- **The gap.** Native refused an `open_condition_wait` command without a + `condition_key`, or without a `condition_definition_fingerprint` of the form + `sha256:` followed by 64 hex digits. Published PHP accepts both. +- **How PHP 2.11.1 does it.** `normalizeOpenConditionWaitCommand` treats the key + and the fingerprint as optional strings of any form. It trims them and leaves + them out when empty. `applyOpenConditionWait` records only the fields that + remain, on `ConditionWaitOpened` and on a condition timeout's + `TimerScheduled`. Run summaries name a keyless wait "Waiting for condition", + without a key. Observed live against the published image: a bare + `open_condition_wait`, a key of `" ready "` with the fingerprint + `"any form"`, and blank values all behave this way. +- **What native does now.** The same. The open wait projection stores no + target name for a keyless wait, and the run summary reads such a wait as + PHP does. A key or fingerprint that is not a string is still refused by the + completion transport rules. +- **How it was checked.** A new integration test covers a bare wait, a padded + key with an arbitrary fingerprint, blank values with a timeout, and a key + that is not a string. The 63 existing condition, timed-condition and + observation tests still pass. diff --git a/rust/src/runtime/external_wait_summary.rs b/rust/src/runtime/external_wait_summary.rs index 0f9a959a1..06abf03fa 100644 --- a/rust/src/runtime/external_wait_summary.rs +++ b/rust/src/runtime/external_wait_summary.rs @@ -19,6 +19,8 @@ pub(super) struct ExternalWaitHistory { struct ExpectedWait { kind: &'static str, + /// The signal name, or the condition key; PHP allows a condition + /// without one, which reads as empty. name: String, sequence: i64, timeout: Option, @@ -27,6 +29,26 @@ struct ExpectedWait { identity: Value, } +/// A wait's name: a signal always has one; a condition's key is optional. +fn wait_name<'a>(kind: &str, value: &'a Value) -> Option<&'a str> { + match value.as_str().filter(|name| !name.is_empty()) { + Some(name) => Some(name), + None if kind == "condition" && value.is_null() => Some(""), + None => None, + } +} + +impl ExpectedWait { + /// `conditionLabel`: the kind, then the name when there is one. + fn label(&self) -> String { + if self.name.is_empty() { + self.kind.to_owned() + } else { + format!("{} {}", self.kind, self.name) + } + } +} + impl ExternalWaitHistory { pub(super) fn apply(&mut self, kind: &str, payload: &Value) -> bool { if matches!(kind, "TimerScheduled" | "TimerFired" | "TimerCancelled") { @@ -70,7 +92,7 @@ impl ExternalWaitHistory { } let (Some(id), Some(name), Some(sequence)) = ( payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64().filter(|v| *v > 0), ) else { return false; @@ -102,7 +124,7 @@ impl ExternalWaitHistory { let (Some(timer_id), Some(wait_id), Some(name), Some(sequence), Some(delay)) = ( payload["timer_id"].as_str().filter(|v| !v.is_empty()), payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64(), payload["delay_seconds"].as_i64(), ) else { @@ -230,14 +252,11 @@ where kind: expected.kind, id: DB::string(row, "wait_id")?, reason: if expected.fired { - format!( - "Waiting to apply {} {} timeout", - expected.kind, expected.name - ) + format!("Waiting to apply {} timeout", expected.label()) } else if expected.timeout.is_some() { - format!("Waiting for {} {} or timeout", expected.kind, expected.name) + format!("Waiting for {} or timeout", expected.label()) } else { - format!("Waiting for {} {}", expected.kind, expected.name) + format!("Waiting for {}", expected.label()) }, started_at: DB::optional_instant(row, "opened_at")?, deadline_at: expected.timer.as_ref().map(|(_, at)| *at), @@ -272,7 +291,7 @@ where } fn external_wait_matches(row: &DB::Row, expected: &ExpectedWait) -> Result { if DB::string(row, "kind")? != expected.kind - || DB::optional_string(row, "target_name")?.as_deref() != Some(expected.name.as_str()) + || DB::optional_string(row, "target_name")?.unwrap_or_default() != expected.name || DB::optional_number(row, "sequence")? != Some(expected.sequence) || DB::optional_string(row, "history_authority")?.as_deref() != Some("typed_history") || DB::optional_string(row, "source_status")?.as_deref() diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 93c299d12..e86601249 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2199,17 +2199,9 @@ where { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_request")); } - text(command, "condition_key")?; + // `normalizeOpenConditionWaitCommand`: the key and the + // definition fingerprint are optional strings, of any form. super::wait_timeouts::timeout_for(command, "invalid_condition_timeout")?; - let fingerprint = text(command, "condition_definition_fingerprint")?; - if !fingerprint.strip_prefix("sha256:").is_some_and(|hash| { - hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_condition_fingerprint", - )); - } } "open_signal_wait" => { reject_fields(command, &["type", "signal_name", "timeout_seconds"])?; @@ -2555,6 +2547,18 @@ where .unwrap() .remove("condition_wait_occurrence_id"); } + // `normalizeOptionalString`: PHP trims the condition's key and + // definition fingerprint and drops them when empty. + if !direct { + for field in ["condition_key", "condition_definition_fingerprint"] { + match payload[field].as_str().map(str::trim).filter(|value| !value.is_empty()) { + Some(value) => payload[field] = json!(value), + None => { + payload.as_object_mut().unwrap().remove(field); + } + } + } + } let pending_wait_id = if direct { let pending = Self::query("SELECT signal_wait_id FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_name=$2 AND status='received' ORDER BY command_sequence,id LIMIT 1") .bind(&run_id).bind(text(command, "signal_name")?).fetch_optional(&mut *tx).await?; diff --git a/rust/src/runtime/wait_projection.rs b/rust/src/runtime/wait_projection.rs index 6467aa436..6dfdfc2ba 100644 --- a/rust/src/runtime/wait_projection.rs +++ b/rust/src/runtime/wait_projection.rs @@ -267,7 +267,8 @@ where ) -> Result<()> { if event_type == "ConditionWaitOpened" { let wait_id = super::text(payload, "condition_wait_id")?; - let key = super::text(payload, "condition_key")?; + // PHP records a condition wait without a key when none was given. + let key = payload["condition_key"].as_str(); let sequence = wait_sequence(payload)?; let projection_id = projection_id(run_id, wait_id); let summary = payload["timeout_seconds"].as_i64().map_or_else( diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index c3ce551e4..55a6b36ff 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod rejected_updates; mod continue_messages; #[path = "execution/update_wait.rs"] mod update_wait; +#[path = "execution/keyless_condition_waits.rs"] +mod keyless_condition_waits; #[path = "execution/payload_limits.rs"] mod payload_limits; #[path = "execution/start_definition.rs"] diff --git a/rust/tests/execution/keyless_condition_waits.rs b/rust/tests/execution/keyless_condition_waits.rs new file mode 100644 index 000000000..2d66bfa6a --- /dev/null +++ b/rust/tests/execution/keyless_condition_waits.rs @@ -0,0 +1,63 @@ +use super::*; + +// Workflow 2.9's `normalizeOpenConditionWaitCommand`: a condition wait's key +// and definition fingerprint are optional strings of any form. PHP trims them +// and leaves them out when empty. + +fn opened(history: &Value) -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "ConditionWaitOpened") + .map(|event| event["payload"].clone()).unwrap() +} + +#[tokio::test] +async fn condition_waits_need_neither_a_key_nor_a_fingerprint() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "keyless-condition").await; + register(&app, "keyless", json!(["echo"]), json!([])).await; + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_kind='condition' AND wait_reason='Waiting for condition'").await, 1); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_waits WHERE kind='condition' AND status='open' AND target_name IS NULL").await, 1); + + // A signal wakes the wait; the next one keeps a trimmed key and any + // fingerprint, and drops a blank one. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ready ", + "condition_definition_fingerprint":"any form"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let payload = opened(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + assert_eq!((payload["condition_key"].clone(), payload["condition_definition_fingerprint"].clone()), + (json!("ready"), json!("any form")), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition ready'").await, 1); + + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/again", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ", + "condition_definition_fingerprint":"","timeout_seconds":60}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + let scheduled = history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "TimerScheduled").unwrap(); + assert!(scheduled["payload"].get("condition_key").is_none(), "{scheduled}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition or timeout'").await, 1); + + // A key that is not a string is still refused. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/third", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":7}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); +} From 0ec63aae91f51015ef143b4630f9a08fb37e31ed Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 20:49:49 +0000 Subject: [PATCH 06/23] Snapshot the task on every event PHP records with it Published PHP's WorkflowHistoryEvent::record adds the task's snapshot to each event it records with the task model. Native now adds it centrally for those event types, uses the full snapshot on TimerScheduled and the activity events, and leases a timer task to itself while it fires, as RunTimerTask does, so TimerFired carries the leased timer task. --- .../2026-10-23-task-snapshots.md | 36 ++++++++++ rust/src/runtime/activity_failures.rs | 4 +- rust/src/runtime/store.rs | 66 +++++++++++++++++-- .../execution/worker_history_presentation.rs | 59 +++++++++++++++++ 4 files changed, 157 insertions(+), 8 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md new file mode 100644 index 000000000..10c5a1ce1 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -0,0 +1,36 @@ +# October 23, task snapshots on history events + +- **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` + snapshot to every event it records with the task model in hand. Native + added snapshots event by event and missed several. Driving one workflow + through a side effect, an activity, a timer, a child, a signal wait and a + condition wait against both servers showed that native had no snapshot on: + - `TimerFired`, `ChildWorkflowScheduled`, `ChildRunStarted`; + - `SignalWaitOpened`, `ConditionWaitOpened`, `ConditionWaitSatisfied`; + - `SignalApplied` and the instance `MessageCursorAdvanced`. + + Native's own snapshots on `TimerScheduled`, `ActivityStarted`, + `ActivityCompleted` and `ActivityFailed` also left out fields that PHP + keeps, such as `queue`, `connection`, `available_at` and `repair_count`. +- **What native does now.** + - When an event of a type PHP records with the task model is appended with + a task and no snapshot of its own, native adds the task's snapshot. The + list of those event types comes from the Workflow package's source. + - `TimerScheduled` uses the full snapshot. The activity events' snapshots + keep PHP's fields. + - As `RunTimerTask` does, a timer task is leased to itself while its timer + fires: status leased, `leased_at`, `lease_owner` set to the task id, a + workflow-task lease, and one more attempt. Its lease is cleared once the + timer has fired. `TimerFired` therefore carries the leased timer task. +- **How it was checked.** The same probe now finds the same events with + snapshots on both servers, with the same fields. A new integration test + checks the timer, child, signal-wait, signal and condition-wait events. +- **Left as they are.** + - PHP's command snapshots carry `request_route_name`, a name Laravel + generates at random for the deployment's cached routes. Native has no + such name. + - Native stamps a dispatch on every new task, so the snapshot of a task + created inside a completion keeps `last_dispatch_attempt_at` and + `last_dispatched_at`, which PHP's does not (Server #325 follow-up). + - A message stream's `SignalApplied` and the condition wait it satisfies + follow the message streams change (#709). diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 062f7720a..e24c73bb7 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -540,7 +540,9 @@ where } payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":count}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":count,"repair_count":DB::number(&task,"repair_count")?,"queue":DB::string(&task,"queue")?, + "connection":DB::PHP_CONNECTION}); let failed_event = payload.clone(); Self::append( &mut tx, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 03dfa699a..2995d2b9a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,6 +15,37 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// The events PHP's Workflow package records with the task model in hand, +/// which therefore carry the task's snapshot. +const TASK_SNAPSHOT_EVENTS: &[&str] = &[ + "ActivityCompleted", + "ActivityFailed", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ChildRunStarted", + "ChildWorkflowScheduled", + "ConditionWaitOpened", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "FailureHandled", + "MemoUpserted", + "MessageCursorAdvanced", + "SearchAttributesUpserted", + "SelectionOperationCancelled", + "SideEffectRecorded", + "SignalApplied", + "SignalWaitOpened", + "TimerFired", + "TimerScheduled", + "VersionMarkerRecorded", + "WorkflowCompleted", + "WorkflowContinuedAsNew", + "WorkflowFailed", + "WorkflowTimedOut", +]; /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; @@ -1709,7 +1740,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":leased_at, - "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue}); + "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue, + "connection":DB::PHP_CONNECTION}); Self::append_at( &mut tx, &run_id, @@ -2490,8 +2522,7 @@ where Self::query("INSERT INTO workflow_run_timers(id,workflow_run_id,sequence,status,delay_seconds,fire_at) VALUES ($1,$2,$3,'pending',$4,$5)") .bind(&timer_id).bind(&run_id).bind(sequence).bind(delay).bind(DB::bind_time(fire_at)).execute(&mut *tx).await?; let mut timer_payload = json!({"timer_id":timer_id,"sequence":sequence,"delay_seconds":delay,"fire_at":fire_at, - "task":{"id":task_id,"type":"workflow","status":"leased","lease_owner":DB::optional_string(&task,"lease_owner")?, - "lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?,"attempt_count":DB::number(&task,"attempt_count")?}}); + "task":Self::task_snapshot(&task)?}); super::selections::merge(&mut timer_payload, &metadata); Self::append_at( &mut tx, @@ -2889,7 +2920,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":DB::number(&task,"attempt_count")?}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":DB::number(&task,"attempt_count")?,"queue":DB::string(&task,"queue")?,"connection":DB::PHP_CONNECTION}); Self::append( &mut tx, &run_id, @@ -3015,6 +3047,15 @@ where json!({"timer_id":timer_id}) }; super::selections::merge(&mut payload, &metadata); + // `RunTimerTask`: the timer task is leased to itself while its + // timer fires, so `TimerFired` carries the leased task. + Self::query("UPDATE workflow_tasks SET status='leased',leased_at=$1,lease_owner=$2,lease_expires_at=$3,attempt_count=attempt_count+1 WHERE id=$4") + .bind(DB::bind_time(fired_at)) + .bind(&task_id) + .bind(DB::bind_time(fired_at + chrono::Duration::seconds(self.workflow_task_lease_seconds))) + .bind(&task_id) + .execute(&mut *tx) + .await?; Self::append( &mut tx, &run_id, @@ -3038,7 +3079,7 @@ where .bind(&timer_id) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_tasks SET status='completed' WHERE id=$1") + Self::query("UPDATE workflow_tasks SET status='completed',lease_expires_at=NULL WHERE id=$1") .bind(&task_id) .execute(&mut *tx) .await?; @@ -3367,9 +3408,20 @@ where .execute(&mut **tx) .await?; let recorded_at = recorded_at.unwrap_or_else(now); - // `WorkflowHistoryEvent::record`: an event recorded with a command - // carries that command's snapshot. let mut payload = payload; + // `WorkflowHistoryEvent::record`: an event PHP records with its task + // model carries that task's snapshot. + if let Some(task_id) = task_id + && TASK_SNAPSHOT_EVENTS.contains(&event_type) + && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + .bind(task_id) + .fetch_optional(&mut **tx) + .await? + { + payload["task"] = Self::task_snapshot(&task)?; + } + // An event recorded with a command carries that command's snapshot. if let Some(command_id) = command_id.filter(|_| command_snapshot) && payload.as_object().is_some_and(|fields| !fields.contains_key("command")) && let Some(snapshot) = Self::command_snapshot(tx, command_id).await? diff --git a/rust/tests/execution/worker_history_presentation.rs b/rust/tests/execution/worker_history_presentation.rs index db76a3d75..401ac593f 100644 --- a/rust/tests/execution/worker_history_presentation.rs +++ b/rust/tests/execution/worker_history_presentation.rs @@ -62,3 +62,62 @@ async fn a_worker_reads_the_sequence_that_consumed_a_signal() { } assert!(completed["payload"]["task"].get("last_claim_error").is_none(), "unset fields stay out: {completed}"); } + +// `WorkflowHistoryEvent::record` with the task model: each event PHP records +// while handling a task carries that task's snapshot, and a fired timer +// carries its timer task, leased to itself. +#[tokio::test] +async fn events_recorded_with_a_task_carry_its_snapshot() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "snapshot-events").await; + register(&app, "snapshotter", json!(["echo", "child"]), json!([])).await; + let events = || run_history(&app, &started["workflow_id"], &started["run_id"]); + let snapshot_of = |history: &Value, kind: &str| -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == kind) + .map(|event| event["payload"]["task"].clone()).unwrap_or(Value::Null) + }; + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await.0, StatusCode::OK); + let scheduled = snapshot_of(&events().await, "TimerScheduled"); + assert_eq!((scheduled["id"].clone(), scheduled["type"].clone()), (task["task_id"].clone(), json!("workflow")), "{scheduled}"); + for field in ["queue", "connection", "available_at", "leased_at", "repair_count"] { + assert!(scheduled.get(field).is_some(), "{field}: {scheduled}"); + } + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(15); + let fired = loop { + let fired = snapshot_of(&events().await, "TimerFired"); + if !fired.is_null() { + break fired; + } + assert!(tokio::time::Instant::now() < deadline, "the timer never fired"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + }; + assert_eq!((fired["type"].clone(), fired["status"].clone(), fired["attempt_count"].clone()), + (json!("timer"), json!("leased"), json!(1)), "{fired}"); + assert_eq!(fired["lease_owner"], fired["id"], "{fired}"); + assert!(fired["lease_expires_at"].is_string() && fired["leased_at"].is_string(), "{fired}"); + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([ + {"type":"start_child_workflow","workflow_type":"child","arguments":envelope(Payload::Array(vec![]))}, + {"type":"open_signal_wait","signal_name":"go"}])).await.0, StatusCode::OK); + let history = events().await; + for kind in ["ChildWorkflowScheduled", "ChildRunStarted", "SignalWaitOpened"] { + assert_eq!(snapshot_of(&history, kind)["id"], task["task_id"], "{kind}: {history}"); + } + let (status, body) = request(&app, "POST", "/api/workflows/snapshot-events/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let mut task = poll(&app, "snapshotter", "workflow").await; + // The child's first task may come first on the shared queue. + if task["workflow_type"] == "child" { + assert_eq!(finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await.0, StatusCode::OK); + task = poll(&app, "snapshotter", "workflow").await; + } + assert_eq!(finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":"k", + "condition_definition_fingerprint":format!("sha256:{}", "c".repeat(64))}])).await.0, StatusCode::OK); + let history = events().await; + assert_eq!(snapshot_of(&history, "SignalApplied")["id"], task["task_id"], "{history}"); + assert_eq!(snapshot_of(&history, "ConditionWaitOpened")["id"], task["task_id"], "{history}"); +} From 05a6110cb4122c62cf18e8b52c9437c7bf08116a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 21:04:16 +0000 Subject: [PATCH 07/23] Describe each command's workflow class and preview flag like PHP PHP's run description names the run's workflow class on every command and says whether the command's payload preview is omitted. Native left both out. Only a stored external reference is read to decide the flag. --- .../2026-10-23-task-snapshots.md | 10 +++++++++- rust/src/runtime/run_describe.rs | 2 +- rust/src/runtime/store.rs | 11 +++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 10c5a1ce1..87931cf15 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -1,4 +1,4 @@ -# October 23, task snapshots on history events +# October 23, task snapshots on history events and described commands - **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` snapshot to every event it records with the task model in hand. Native @@ -25,6 +25,14 @@ - **How it was checked.** The same probe now finds the same events with snapshots on both servers, with the same fields. A new integration test checks the timer, child, signal-wait, signal and condition-wait events. +- **Described commands.** A field-by-field diff of the same workflow's + description on both servers found two more gaps. Each command PHP + describes names the run's `workflow_class` (`WorkflowCommand::record` keeps + it), and says whether its payload preview is omitted + (`payload_preview_omitted`, true only for a stored external payload over the + size limit). Native now sends both. With fresh workers on both sides, the + descriptions then differ only in `request_route_name` and the + `request_fingerprint` hashed from it. - **Left as they are.** - PHP's command snapshots carry `request_route_name`, a name Laravel generates at random for the deployment's cached routes. Native has no diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 070e4e364..ae9e6ae3b 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,7 +31,7 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. -fn oversized(envelope: &Value) -> bool { +pub(super) fn oversized(envelope: &Value) -> bool { envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 2995d2b9a..268e891b8 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -553,7 +553,9 @@ where // refused signals which intentionally have no workflow history event. // Select metadata only: large customer payloads are never decoded or // copied merely to display the command's target and admission result. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + // Only a stored external reference is read, to tell whether its preview + // is omitted; inline payloads stay unread. + let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.payload LIKE 'dw-%' THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { @@ -564,7 +566,12 @@ where "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, - "workflow_type":DB::optional_string(&command,"workflow_type")?,"workflow_class":DB::optional_string(&command,"workflow_class")?, + // `WorkflowCommand::record` keeps the run's class on each command. + "workflow_type":DB::optional_string(&command,"workflow_type")?, + "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), + "payload_preview_omitted":DB::optional_string(&command,"stored_payload")?.as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)), "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); From 1cbe01ffdd0765c1a4969834f58c3c087697c804 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 22:56:07 +0000 Subject: [PATCH 08/23] Wait for the query acknowledgement before stopping Python and Rust clients worker.php already waits, inside the original 30 s budget, for the Worker's acknowledgement of its query result before it stops: the independent caller can see the committed result first. The Python and Rust clients stopped immediately, so a fast caller could end the Worker before its acknowledgement arrived (#708 attempt 1, MySQL 8.0, PHP target, timely). Both now wait the same bounded way. The submission and acknowledgement assertions are unchanged. --- .../server-parity/query-authority/python.py | 11 +++++++++++ .../server-parity/query-authority/rust/src/main.rs | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/scripts/conformance/server-parity/query-authority/python.py b/scripts/conformance/server-parity/query-authority/python.py index 023aa84cb..52e60c9fa 100644 --- a/scripts/conformance/server-parity/query-authority/python.py +++ b/scripts/conformance/server-parity/query-authority/python.py @@ -117,6 +117,17 @@ async def main(): if MODE == 'timely': result = await client.query_workflow(namespace, 'value', args=[]) assert result['result'] == EXPECTED and type(result['result']['long']) is int + # The independent caller can observe the commit before the + # Worker receives its acknowledgement. Wait inside the + # original budget before stopping it, as worker.php does. + PHASE = 'acknowledgement' + while not any(x['event'] == 'acknowledgement' for x in events()): + if runner.done(): + await runner + raise RuntimeError('Worker stopped before its acknowledgement') + if time.monotonic() - started >= 30: + raise AssertionError('No acknowledgement inside the original budget') + await asyncio.sleep(.02) else: try: await client.query_workflow(namespace, 'value', args=[]) diff --git a/scripts/conformance/server-parity/query-authority/rust/src/main.rs b/scripts/conformance/server-parity/query-authority/rust/src/main.rs index 45fae759d..d413a6dbc 100644 --- a/scripts/conformance/server-parity/query-authority/rust/src/main.rs +++ b/scripts/conformance/server-parity/query-authority/rust/src/main.rs @@ -169,6 +169,13 @@ async fn main() -> Result<()> { let query = client.query_workflow(&namespace, "value", json!([])).await; if mode == "timely" { assert_eq!(query?, expected()); + // The independent caller can observe the commit before the Worker + // receives its acknowledgement. Wait inside the original budget + // before stopping it, as worker.php does. + wait(until, || { + !named(&events(Path::new(journal)), "acknowledgement").is_empty() + }) + .await?; } else { match query { Err(Error::QueryFailed(failure)) => { From 911b89a67da880c974da0ed20ae54396f89502f7 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 00:23:44 +0000 Subject: [PATCH 09/23] Wait for the expiry evidence before stopping the Python query client In late-handler-sync mode the client stopped as soon as the handler returned, but the installed SDK logs the lost authority after that, so the expired-authority assertion could see none (#713 attempt 2, PHP target). worker.php waits for its Worker's refusal; every non-timely Python mode now also waits, inside the 30 s original budget, for the expired-authority evidence before stopping. Assertions are unchanged. --- .../server-parity/query-authority/python.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/conformance/server-parity/query-authority/python.py b/scripts/conformance/server-parity/query-authority/python.py index 52e60c9fa..e3f0d8b1c 100644 --- a/scripts/conformance/server-parity/query-authority/python.py +++ b/scripts/conformance/server-parity/query-authority/python.py @@ -136,12 +136,19 @@ async def main(): assert error.status == 504 and error.reason == 'query_worker_execution_timeout' # Delivery is recorded before the SDK receives its response. # Wait for its actual refusal before stopping the worker. + # A late handler's callback counter moves before the SDK + # logs the lost authority, so every mode also waits for + # that evidence, as worker.php waits for its refusal. PHASE = 'expiry-observation' - while (MODE == 'late-poll' and not any( - expired_losses(event['claim']['query_task_id']) - for event in events() if event['event'] == 'claim') + while (not any(expired_losses(event['claim']['query_task_id']) + for event in events() if event['event'] == 'claim') or MODE == 'late-handler-async' and COUNTERS['cancelled'] != 1 or MODE == 'late-handler-sync' and COUNTERS['returned'] != 1): + if runner.done(): + await runner + raise RuntimeError('Worker stopped before its expiry evidence') + if time.monotonic() - started >= 30: + raise AssertionError('No expiry evidence inside the original budget') await asyncio.sleep(.02) finally: PHASE = 'worker-stop' From 65772abac8ecb65f7bde7f85c56b80bb77054777 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 00:50:27 +0000 Subject: [PATCH 10/23] Rust #325: preview omission reads native stored references; exact prefix; run-bound task snapshots Security review of #718: - oversized() reads native's external_payload envelope as well as PHP's external_storage, so a stored reference over the payload limit omits the preview (and the target name, as PHP does). Run input and output previews use the same check. - The described-command query reads a payload only when its first 30 characters equal native's stored prefix, compared case-sensitively on each backend (COLLATE BINARY, binary cast, PostgreSQL text). - record() reads a task snapshot only from a task of the event's own run. Tests: a real stored reference at the 2 MiB limit and one byte above, upper-case/v2/malformed/large-inline counterfactuals, the query itself on SQLite, and a mismatched-run event with no snapshot. --- .../2026-10-23-task-snapshots.md | 27 ++++ rust/src/runtime/backend.rs | 8 ++ rust/src/runtime/run_describe.rs | 8 +- rust/src/runtime/store.rs | 37 ++++- rust/src/runtime/store_record_tests.rs | 128 ++++++++++++++++++ .../execution/external_payloads/commands.rs | 57 ++++++++ 6 files changed, 258 insertions(+), 7 deletions(-) create mode 100644 rust/src/runtime/store_record_tests.rs diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 87931cf15..5ea9a10aa 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -33,6 +33,33 @@ size limit). Native now sends both. With fresh workers on both sides, the descriptions then differ only in `request_route_name` and the `request_fingerprint` hashed from it. +- **Security review corrections.** + - The size check read only PHP's `external_storage` envelope. Native's + stored envelope names its reference `external_payload`, so the flag was + always false. It now reads both, which also corrects the run's input and + output previews. As PHP does, an omitted preview also omits the + command's `target_name`. + - The query read any payload starting with `dw-`, and SQLite's `LIKE` and + MySQL's case-insensitive collation ignore case. It now reads a payload + only when its first 30 characters equal native's stored prefix exactly: + `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, and + PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + 10.11 the binary cast refuses the upper-case prefix that the column's + collation would match. + - A task snapshot is read only from a task of the event's own run. + - Tests: + - a real uploaded reference on a signal, then the same reference stored + at the 2 MiB limit (not omitted) and one byte over (omitted, with no + target name); the described command never includes the reference; + - counterfactuals that are not native references: an upper-case or v2 + prefix, the prefix with no valid reference after it, and a large inline + payload starting with `dw-`; + - on SQLite, the query itself returns the payload only for the exact + prefix; + - an event recorded on one run with another run's task carries no + snapshot. + + Each test fails without its correction. - **Left as they are.** - PHP's command snapshots carry `request_route_name`, a name Laravel generates at random for the deployment's cached routes. Native has no diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 722cd8f75..22eb45864 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -35,6 +35,9 @@ pub(super) trait Backend: Database { const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; /// The history sequence of a signal's `SignalReceived` event. const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; + // PostgreSQL compares text case-sensitively. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("substr(c.payload,1,30)='dw-native-external-payload-v1:'"); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -185,6 +188,8 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'"); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -457,6 +462,9 @@ impl Backend for MySql { const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; + // The payload column's collation ignores case; compare bytes. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!("CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'"); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index ae9e6ae3b..c18bf74d1 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,8 +31,12 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. +/// PHP's stored envelope names its reference `external_storage`; native's +/// names it `external_payload`. pub(super) fn oversized(envelope: &Value) -> bool { - envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) + ["external_storage", "external_payload"] + .iter() + .any(|key| envelope[key]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes())) } /// Carbon's `toIso8601String()`, which PHP's run description uses. @@ -315,6 +319,8 @@ mod tests { assert!(oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024 + 1}}))); assert!(!oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024}}))); assert!(!oversized(&json!({"codec":"avro","blob":"AA=="}))); + assert!(oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024 + 1}}))); + assert!(!oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024}}))); assert_eq!(without_empty(json!({"a":null,"b":[],"c":{},"d":0})), json!({"c":{},"d":0})); let time = DateTime::parse_from_rfc3339("2026-10-21T07:38:51.123Z").unwrap().with_timezone(&Utc); assert_eq!(iso8601(Some(time)), json!("2026-10-21T07:38:51+00:00")); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index e09c281fe..8b970ff41 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -46,6 +46,20 @@ const TASK_SNAPSHOT_EVENTS: &[&str] = &[ "WorkflowFailed", "WorkflowTimedOut", ]; +/// Describe's command journal. `$stored` is each backend's exact, +/// case-sensitive test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`), so a command's payload is read only +/// when it is a stored reference. +macro_rules! run_commands_sql { + ($stored:literal) => { + concat!( + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN ", + $stored, + " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" + ) + }; +} +pub(super) use run_commands_sql; /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; @@ -555,23 +569,28 @@ where // copied merely to display the command's target and admission result. // Only a stored external reference is read, to tell whether its preview // is omitted; inline payloads stay unread. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.payload LIKE 'dw-%' THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + let command_rows = Self::query(DB::RUN_COMMANDS_SQL) .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { + // `oversizedPayloadPreview` over the stored reference's envelope; + // an omitted preview also omits the target name, as PHP does. + let omitted = DB::optional_string(&command, "stored_payload")? + .as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)); + let target_name = DB::optional_string(&command, "signal_name")?.filter(|_| !omitted); let mut public = json!({"id":DB::string(&command,"id")?,"sequence":DB::optional_number(&command,"command_sequence")?, "type":DB::string(&command,"command_type")?,"target_scope":DB::string(&command,"target_scope")?, "requested_run_id":DB::optional_string(&command,"requested_workflow_run_id")?, "resolved_run_id":DB::optional_string(&command,"resolved_workflow_run_id")?, - "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, + "target_name":target_name,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, // `WorkflowCommand::record` keeps the run's class on each command. "workflow_type":DB::optional_string(&command,"workflow_type")?, "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), - "payload_preview_omitted":DB::optional_string(&command,"stored_payload")?.as_deref() - .and_then(super::external_resolution::stored_envelope) - .is_some_and(|envelope| super::run_describe::oversized(&envelope)), + "payload_preview_omitted":omitted, "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); @@ -3425,11 +3444,13 @@ where if let Some(task_id) = task_id && TASK_SNAPSHOT_EVENTS.contains(&event_type) && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) - && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1") + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1 AND workflow_run_id=$2") .bind(task_id) + .bind(run_id) .fetch_optional(&mut **tx) .await? { + // Only the run's own task: another run's is never copied. payload["task"] = Self::task_snapshot(&task)?; } // An event recorded with a command carries that command's snapshot. @@ -4082,3 +4103,7 @@ pub(crate) fn capabilities() -> Value { } value } + +#[cfg(test)] +#[path = "store_record_tests.rs"] +mod record_tests; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs new file mode 100644 index 000000000..1f312ffeb --- /dev/null +++ b/rust/src/runtime/store_record_tests.rs @@ -0,0 +1,128 @@ +//! History recording and described-command boundaries that a public route +//! cannot reach with mismatched identifiers. +use super::Store; +use crate::runtime::backend::Backend; +use crate::runtime::external_resolution::STORED_PREFIX; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[test] +fn described_commands_read_only_the_exact_native_stored_prefix() { + // Each backend compares the first 30 characters, byte for byte, with + // the whole prefix. + assert_eq!(STORED_PREFIX.len(), 30); + let quoted = format!("'{STORED_PREFIX}'"); + for (backend, sql, comparison) in [ + ("sqlite", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + ] { + assert!(sql.contains(&format!("CASE WHEN {comparison}{quoted} THEN c.payload END")), "{backend}: {sql}"); + assert!(!sql.contains("LIKE"), "{backend}: {sql}"); + } +} + +async fn recorded_task(store: &Store, run_id: &str) -> Value { + let row = Store::::query( + "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SideEffectRecorded'", + ) + .bind(run_id) + .fetch_one(&store.pool) + .await + .unwrap(); + ::document_row(&row, "payload").unwrap()["task"].clone() +} + +#[tokio::test] +async fn a_task_snapshot_comes_only_from_the_events_own_run() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("record.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + store + .register( + json!({"worker_id":"record-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]}), + &crate::runtime::QueryTaskConfig::default(), + None, + ) + .await + .unwrap(); + let mut runs = Vec::new(); + for workflow_id in ["record-own", "record-other"] { + let started = store + .start( + json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":{"codec":"avro","blob":"AA=="}}), + BTreeMap::new(), + None, + None, + ) + .await + .unwrap(); + runs.push(started["run_id"].as_str().unwrap().to_owned()); + } + let task = Store::::query("SELECT id FROM workflow_tasks WHERE workflow_run_id=$1") + .bind(&runs[0]) + .fetch_one(&store.pool) + .await + .unwrap(); + let task_id = ::string(&task, "id").unwrap(); + + let mut tx = store.begin().await.unwrap(); + for run_id in &runs { + Store::::record(&mut tx, run_id, "SideEffectRecorded", json!({"sequence":1}), + Some(&task_id), None, None, false) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + assert_eq!(recorded_task(&store, &runs[0]).await["id"], json!(task_id), "the run's own task"); + // Recorded on the other run with the first run's task, the event carries + // no snapshot at all. + assert!(recorded_task(&store, &runs[1]).await.is_null()); +} + +// The query itself, run on rows whose payloads begin like a stored +// reference: only native's exact prefix is selected. +#[tokio::test] +async fn described_commands_select_no_inline_payload() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("commands.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + let exact = format!("{STORED_PREFIX}{{}}"); + let rows = [ + ("c1", exact.clone(), true), + ("c2", exact.to_uppercase(), false), + ("c3", format!("dw-native-external-payload-v1{}", "A".repeat(4096)), false), + ("c4", format!("dw-{}", "A".repeat(4096)), false), + ("c5", "dw-native".to_owned(), false), + ]; + for (sequence, (id, payload, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run','signal','accepted',$2,$3)") + .bind(*id) + .bind(payload.as_str()) + .bind(sequence as i64 + 1) + .execute(&store.pool) + .await + .unwrap(); + } + let selected = Store::::query(::RUN_COMMANDS_SQL) + .bind("run") + .bind("instance") + .fetch_all(&store.pool) + .await + .unwrap(); + assert_eq!(selected.len(), rows.len()); + for (row, (id, payload, read)) in selected.iter().zip(&rows) { + assert_eq!(::string(row, "id").unwrap(), *id); + let stored = ::optional_string(row, "stored_payload").unwrap(); + assert_eq!(stored.as_deref(), read.then_some(payload.as_str()), "{id}"); + } +} diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index 82237259c..bb962993b 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -151,3 +151,60 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { runtime.close().await; database.remove().await; } + +/// The described signal command, which must show only metadata. +async fn described_signal(app: &Router, workflow_id: &str) -> Value { + let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == "signal").unwrap().clone(); + for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { + assert!(command.get(field).is_none(), "{field}: {command}"); + } + assert!(!command.to_string().contains("ep_"), "no reference identity: {command}"); + command +} + +// `WorkflowController::oversizedPayloadPreview` over each command's stored +// envelope: only a stored external reference larger than the payload limit +// (2 MiB by default) omits the preview, and then the target name too. +#[tokio::test] +async fn described_commands_omit_only_an_oversized_stored_reference() { + let temp = tempfile::tempdir().unwrap(); + let (database, runtime, app) = setup(ExternalPayloadConfig::default(), &temp.path().join("objects")).await; + waiting_run(&app, "described-reference", json!({"worker_id":"describe-worker","task_queue":"described-reference", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + let reference = upload(&app, &argument("signal")).await; + let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", + json!({"input":{"codec":"avro","external_payload":reference}})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let command = described_signal(&app, "described-reference").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + let command_id = command["id"].as_str().unwrap().to_owned(); + + // The same stored reference, as native writes it, at sizes around the + // limit, and counterfactuals that are not native stored references. + let limit = 2 * 1024 * 1024_i64; + let stored = |size: i64| { + let mut reference = reference.clone(); + reference["size_bytes"] = json!(size); + format!("dw-native-external-payload-v1:{reference}") + }; + for (payload, omitted) in [ + (stored(limit), false), + (stored(limit + 1), true), + // Not native's exact prefix, so never read. + (stored(limit + 1).replace("dw-native-external-payload-v1:", "DW-NATIVE-EXTERNAL-PAYLOAD-V1:"), false), + (stored(limit + 1).replace("dw-native-external-payload-v1:", "dw-native-external-payload-v2:"), false), + // Native's prefix with no valid reference after it. + (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), + // A large inline payload. + (format!("dw-{}", "A".repeat(limit as usize + 1)), false), + ] { + database.execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_commands SET payload='{payload}' WHERE id='{command_id}'"))).await; + let command = described_signal(&app, "described-reference").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &payload[..payload.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), omitted, "{command}"); + } + runtime.close().await; + database.remove().await; +} From dc1b4e0ae26992d2ac45f450c2e4fd01e82e652b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 00:58:48 +0000 Subject: [PATCH 11/23] Server parity: run the SQLite published-SDK patch record without PHP's maintenance loop (#721) On SQLite the published PHP API server and its maintenance loop share one database file. A contended poll gets PHP's typed backend_lock_pressure refusal, and the published Python run_until helper ends its Worker on it (sdk-python #218). The cohort tests patch replacement, not writer contention: stop the loop for the PHP record, restart it right after, and record and assert its state. The observer still never retries a poll refusal; MySQL and PostgreSQL jobs are unchanged. --- .../published-sdk-patch-profile.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 5e012bd34..221adce99 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -25,13 +25,37 @@ shared=(-v "$PWD:/source:ro" -v "$PWD/published-sdk-adapters:/sdk-adapters:ro" -e DW_PARITY_RUST_PATCH_WORKER=/sdk-adapters/rust-worker -e PYTHONDONTWRITEBYTECODE=1 -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) +# Server #721. On SQLite the published PHP API server and its maintenance loop +# share one database file. SQLite refuses a deferred transaction's upgrade to a +# writer at once, without a busy wait, so PHP answers a contended poll with its +# typed backend_lock_pressure refusal. The published Python run_until helper +# then ends its Worker (sdk-python #218). This cohort tests patch replacement, +# not writer contention, so the PHP record runs with the maintenance loop +# stopped, and the loop restarts right after it. The fixtures use no timers or +# repair. The observer still never retries a poll refusal. +maintenance="$RESOURCE_SCOPE-php-maintenance" +schedule=parity-evidence/published-sdk-patch/maintenance-schedule.txt for target in php rust; do if test "$target" = php; then url="$php_url"; else url="$native_url"; fi + if test "$target" = php && test "$backend" = sqlite; then + echo "before: $(docker inspect -f '{{.State.Status}}' "$maintenance")" > "$schedule" + grep -qx 'before: running' "$schedule" + docker stop --time 2 "$maintenance" > /dev/null + trap 'docker start "$maintenance" > /dev/null' EXIT + echo "during php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + fi docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ --output "/evidence/published-sdk-patch/$target/record.json" + if test "$target" = php && test "$backend" = sqlite; then + docker start "$maintenance" > /dev/null + trap - EXIT + echo "after php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" + grep -qx 'after php record: running' "$schedule" + fi done case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref From 17f63ca8d63bdc2a541c13cd4475bd89dccb7a65 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 01:39:56 +0000 Subject: [PATCH 12/23] Server parity: arm the maintenance restore before the stop and keep it until a sustained restart (#721) Security review of #722: the restore trap was installed only after a successful stop and disarmed before the restart was checked. Move the window into published-sdk-patch/maintenance.sh: arm restoration before the stop, restart with up to three attempts that each require the loop to stay running across three checks, and disarm only after that. A failed restart records the observed state and fails with restoration still armed. tests/Unit/PatchCohortMaintenanceTest.mjs injects a stop that exits nonzero after taking effect, a failed PHP record, and a start that exits at once, with a fake docker. The previous ordering fails the first and the last. --- .github/workflows/phpunit-feature.yml | 3 + .../published-sdk-patch-profile.sh | 33 +++---- .../published-sdk-patch/maintenance.sh | 58 ++++++++++++ tests/Unit/PatchCohortMaintenanceTest.mjs | 90 +++++++++++++++++++ 4 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 scripts/conformance/server-parity/published-sdk-patch/maintenance.sh create mode 100644 tests/Unit/PatchCohortMaintenanceTest.mjs diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 9cecaf1bf..c4e233777 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -71,6 +71,9 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore the patch cohort's maintenance loop on every path + run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs + - name: Validate embedded Workflow package authority run: php scripts/ci/test-workflow-package-authority.php diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 221adce99..c24c58762 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -33,30 +33,21 @@ export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) # not writer contention, so the PHP record runs with the maintenance loop # stopped, and the loop restarts right after it. The fixtures use no timers or # repair. The observer still never retries a poll refusal. -maintenance="$RESOURCE_SCOPE-php-maintenance" -schedule=parity-evidence/published-sdk-patch/maintenance-schedule.txt -for target in php rust; do - if test "$target" = php; then url="$php_url"; else url="$native_url"; fi - if test "$target" = php && test "$backend" = sqlite; then - echo "before: $(docker inspect -f '{{.State.Status}}' "$maintenance")" > "$schedule" - grep -qx 'before: running' "$schedule" - docker stop --time 2 "$maintenance" > /dev/null - trap 'docker start "$maintenance" > /dev/null' EXIT - echo "during php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" - grep -qx 'during php record: exited' "$schedule" - fi +. scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +record() { docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ - scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ + scripts/conformance/server-parity.mjs record --mode http --url "$1" --target "$2" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ - --output "/evidence/published-sdk-patch/$target/record.json" - if test "$target" = php && test "$backend" = sqlite; then - docker start "$maintenance" > /dev/null - trap - EXIT - echo "after php record: $(docker inspect -f '{{.State.Status}}' "$maintenance")" >> "$schedule" - grep -qx 'after php record: running' "$schedule" - fi -done + --output "/evidence/published-sdk-patch/$2/record.json" +} +if test "$backend" = sqlite; then + run_without_maintenance "$RESOURCE_SCOPE-php-maintenance" \ + parity-evidence/published-sdk-patch/maintenance-schedule.txt -- record "$php_url" php +else + record "$php_url" php +fi +record "$native_url" rust case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password) ;; diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh new file mode 100644 index 000000000..71cdebacb --- /dev/null +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -0,0 +1,58 @@ +# Sourced by published-sdk-patch-profile.sh (Server #721). +# +# run_without_maintenance CONTAINER SCHEDULE -- COMMAND... +# +# Runs COMMAND while the maintenance container is stopped, then restarts it. +# Restoration is armed before the stop is attempted and stays armed until the +# container has stayed running for a sustained check, so a failed stop, a +# failed COMMAND or a restart that exits at once never leaves it stopped. +# SCHEDULE records the container's state before, during and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo unknown +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + test "$(maintenance_state "$1")" = running || return 1 + sleep "$maintenance_poll_seconds" + done + test "$(maintenance_state "$1")" = running +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$1" > /dev/null 2>&1 || true + if maintenance_stays_running "$1"; then + return 0 + fi + done + echo "maintenance container $1 did not stay running: $(maintenance_state "$1")" >&2 + return 1 +} + +run_without_maintenance() { + local container="$1" schedule="$2" + shift 2 + test "$1" = -- && shift + echo "before: $(maintenance_state "$container")" > "$schedule" + grep -qx 'before: running' "$schedule" + # shellcheck disable=SC2064 + trap "maintenance_restore '$container' || true" EXIT + docker stop --time 2 "$container" > /dev/null + echo "during php record: $(maintenance_state "$container")" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore "$container"; then + echo "after php record: $(maintenance_state "$container")" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs new file mode 100644 index 000000000..9283d780e --- /dev/null +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -0,0 +1,90 @@ +// Server #721: the SQLite published-SDK patch cohort stops PHP's maintenance +// loop for its PHP record. Every failure path must leave the loop restored or +// fail loudly, and never record a restart that did not stay running. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const helper = resolve('scripts/conformance/server-parity/published-sdk-patch/maintenance.sh'); + +// One container's state in a file; every call is logged. +// SCENARIO=stop-nonzero: the stop takes effect but exits 1. +// SCENARIO=immediate-exit: start exits 0, the container runs for one +// inspection and then exits. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +case "$1" in + inspect) + state=$(cat "$dir/state") + if test "$state" = running-once; then echo exited > "$dir/state"; state=running; fi + echo "$state" ;; + stop) echo exited > "$dir/state"; test "$SCENARIO" != stop-nonzero ;; + start) + if test "$SCENARIO" = immediate-exit; then echo running-once > "$dir/state"; else echo running > "$dir/state"; fi ;; + *) exit 2 ;; +esac +`; + +function run(scenario, command) { + const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', ['-c', + `set -euo pipefail; . "$HELPER"; run_without_maintenance maint "$SCHEDULE" -- ${command}`], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05'}, + }); + const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); + const calls = read('calls').trim().split('\n'); + return { + status: result.status, + stderr: result.stderr, + schedule: read('schedule.txt').trim().split('\n'), + starts: calls.filter(call => call.startsWith('start ')).length, + stopIndex: calls.findIndex(call => call.startsWith('stop ')), + lastStart: calls.findLastIndex(call => call.startsWith('start ')), + state: read('state').trim(), + }; +} + +test('a normal window stops, runs, and restores a loop that stays running', () => { + const result = run('normal', 'true'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.schedule, ['before: running', 'during php record: exited', 'after php record: running for 0.05s x3']); + assert.equal(result.starts, 1, 'restoration is disarmed after the sustained check'); + assert.equal(result.state, 'running'); +}); + +test('a stop that takes effect but exits nonzero still restores', () => { + const result = run('stop-nonzero', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.lastStart > result.stopIndex, 'a start follows the stop'); + assert.equal(result.state, 'running'); + assert.deepEqual(result.schedule, ['before: running']); +}); + +test('a failed PHP record restores the loop', () => { + const result = run('normal', 'false'); + assert.notEqual(result.status, 0); + assert.equal(result.state, 'running'); + assert.ok(result.starts >= 1); + assert.ok(!result.schedule.some(line => line.startsWith('after php record'))); +}); + +test('a restart that exits at once stays armed and fails without recording running', () => { + const result = run('immediate-exit', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.schedule.includes('after php record: exited'), result.schedule.join('\n')); + assert.ok(!result.schedule.some(line => line.startsWith('after php record: running'))); + // Three attempts after the record, then three more from the still-armed trap. + assert.equal(result.starts, 6); + assert.match(result.stderr, /did not stay running/); +}); From 38717d3372767b72cf17851b0bd16ca935515e81 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 01:43:15 +0000 Subject: [PATCH 13/23] Rust #325: read a signal's stored reference from its signal record Security review of #718 at 65772abac: native keeps a signal's stored input reference in workflow_signal_records.arguments; workflow_commands.payload holds the encoded command. Read the exact native sentinel from s.arguments for signals and from c.payload for starts only, case-sensitively per backend. The describe test now proves the real signal row is the source, mutates that row around the limit with the same counterfactuals, shows an oversized sentinel in a signal's command row does not count, and covers the start command's row. --- .../2026-10-23-task-snapshots.md | 30 +++++---- rust/src/runtime/backend.rs | 15 ++++- rust/src/runtime/store.rs | 18 +++--- rust/src/runtime/store_record_tests.rs | 62 +++++++++++++------ rust/tests/execution/external_payloads.rs | 2 +- .../execution/external_payloads/commands.rs | 50 +++++++++++---- 6 files changed, 124 insertions(+), 53 deletions(-) diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md index 5ea9a10aa..f139ee749 100644 --- a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -39,23 +39,31 @@ always false. It now reads both, which also corrects the run's input and output previews. As PHP does, an omitted preview also omits the command's `target_name`. - - The query read any payload starting with `dw-`, and SQLite's `LIKE` and - MySQL's case-insensitive collation ignore case. It now reads a payload - only when its first 30 characters equal native's stored prefix exactly: - `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, and - PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + - The query read any command payload starting with `dw-`, and SQLite's + `LIKE` and MySQL's case-insensitive collation ignore case. It also read the + wrong column for signals. Native keeps a signal's stored input reference + in its signal record (`workflow_signal_records.arguments`) and a start's + in the command row (`workflow_commands.payload`). Other commands keep + none. The query now reads only that column for each of the two command + types, and only when its first 30 characters equal native's stored prefix + exactly: `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, + and PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB 10.11 the binary cast refuses the upper-case prefix that the column's collation would match. - A task snapshot is read only from a task of the event's own run. - Tests: - - a real uploaded reference on a signal, then the same reference stored - at the 2 MiB limit (not omitted) and one byte over (omitted, with no - target name); the described command never includes the reference; + - a real uploaded reference on a signal, found in the signal record and + not in the command row; + - that signal record rewritten with the same reference at the 2 MiB limit + (not omitted) and one byte over (omitted, with no target name). The + described command never includes the reference; - counterfactuals that are not native references: an upper-case or v2 prefix, the prefix with no valid reference after it, and a large inline - payload starting with `dw-`; - - on SQLite, the query itself returns the payload only for the exact - prefix; + payload starting with `dw-`. An oversized reference in the signal's + command row is not counted either; + - the same sizes and counterfactuals in a start command's row; + - on SQLite, the query itself returns only the exact prefix in the + command type's own column; - an event recorded on one run with another run's task carries no snapshot. diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 22eb45864..67a823065 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -37,7 +37,10 @@ pub(super) trait Backend: Database { const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; // PostgreSQL compares text case-sensitively. const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("substr(c.payload,1,30)='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "substr(s.arguments,1,30)='dw-native-external-payload-v1:'", + "substr(c.payload,1,30)='dw-native-external-payload-v1:'" + ); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -189,7 +192,10 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "substr(s.arguments,1,30) COLLATE BINARY='dw-native-external-payload-v1:'", + "substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'" + ); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -464,7 +470,10 @@ impl Backend for MySql { const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; // The payload column's collation ignores case; compare bytes. const RUN_COMMANDS_SQL: &str = - super::store::run_commands_sql!("CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'"); + super::store::run_commands_sql!( + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)='dw-native-external-payload-v1:'", + "CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'" + ); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 8b970ff41..8c0133c61 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -46,15 +46,19 @@ const TASK_SNAPSHOT_EVENTS: &[&str] = &[ "WorkflowFailed", "WorkflowTimedOut", ]; -/// Describe's command journal. `$stored` is each backend's exact, -/// case-sensitive test for native's stored-reference prefix -/// (`external_resolution::STORED_PREFIX`), so a command's payload is read only -/// when it is a stored reference. +/// Describe's command journal. Each backend passes its exact, case-sensitive +/// test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`) over the column that holds a +/// command's input: a signal's in its signal record (`s.arguments`), a start's +/// in the command row (`c.payload`). Other commands keep no stored reference. +/// Only a stored reference is read; inline payloads are not. macro_rules! run_commands_sql { - ($stored:literal) => { + ($signal:literal, $start:literal) => { concat!( - "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN ", - $stored, + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.command_type='signal' AND ", + $signal, + " THEN s.arguments WHEN c.command_type='start' AND ", + $start, " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" ) }; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs index 1f312ffeb..9529c9473 100644 --- a/rust/src/runtime/store_record_tests.rs +++ b/rust/src/runtime/store_record_tests.rs @@ -8,16 +8,21 @@ use std::collections::BTreeMap; #[test] fn described_commands_read_only_the_exact_native_stored_prefix() { - // Each backend compares the first 30 characters, byte for byte, with - // the whole prefix. + // Each backend compares the first 30 characters, byte for byte, with the + // whole prefix: a signal's in its signal record, a start's in its command. assert_eq!(STORED_PREFIX.len(), 30); let quoted = format!("'{STORED_PREFIX}'"); - for (backend, sql, comparison) in [ - ("sqlite", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30) COLLATE BINARY="), - ("postgres", ::RUN_COMMANDS_SQL, "substr(c.payload,1,30)="), - ("mysql", ::RUN_COMMANDS_SQL, "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + for (backend, sql, signal, start) in [ + ("sqlite", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30) COLLATE BINARY=", "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30)=", "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)=", "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), ] { - assert!(sql.contains(&format!("CASE WHEN {comparison}{quoted} THEN c.payload END")), "{backend}: {sql}"); + let expected = format!("CASE WHEN c.command_type='signal' AND {signal}{quoted} THEN s.arguments \ + WHEN c.command_type='start' AND {start}{quoted} THEN c.payload END"); + assert!(sql.contains(&expected), "{backend}: {sql}"); assert!(!sql.contains("LIKE"), "{backend}: {sql}"); } } @@ -86,8 +91,8 @@ async fn a_task_snapshot_comes_only_from_the_events_own_run() { assert!(recorded_task(&store, &runs[1]).await.is_null()); } -// The query itself, run on rows whose payloads begin like a stored -// reference: only native's exact prefix is selected. +// The query itself, run on rows whose inputs begin like a stored reference: +// only native's exact prefix in the command type's own input column is read. #[tokio::test] async fn described_commands_select_no_inline_payload() { let directory = tempfile::tempdir().unwrap(); @@ -97,21 +102,40 @@ async fn described_commands_select_no_inline_payload() { .unwrap(), ); let exact = format!("{STORED_PREFIX}{{}}"); + let large = format!("dw-{}", "A".repeat(4096)); + let near = format!("dw-native-external-payload-v1{}", "A".repeat(4096)); + let upper = exact.to_uppercase(); + // (id, type, command payload, signal arguments, read) let rows = [ - ("c1", exact.clone(), true), - ("c2", exact.to_uppercase(), false), - ("c3", format!("dw-native-external-payload-v1{}", "A".repeat(4096)), false), - ("c4", format!("dw-{}", "A".repeat(4096)), false), - ("c5", "dw-native".to_owned(), false), + ("c1", "signal", "AA==", Some(exact.as_str()), Some(exact.as_str())), + ("c2", "signal", "AA==", Some(upper.as_str()), None), + ("c3", "signal", "AA==", Some(near.as_str()), None), + ("c4", "signal", "AA==", Some(large.as_str()), None), + // A signal's command row is not its input. + ("c5", "signal", exact.as_str(), Some("AA=="), None), + ("c6", "start", exact.as_str(), None, Some(exact.as_str())), + ("c7", "start", large.as_str(), None, None), + // Other commands keep no stored reference. + ("c8", "update", exact.as_str(), None, None), ]; - for (sequence, (id, payload, _)) in rows.iter().enumerate() { - Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run','signal','accepted',$2,$3)") + for (sequence, (id, kind, payload, arguments, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run',$2,'accepted',$3,$4)") .bind(*id) - .bind(payload.as_str()) + .bind(*kind) + .bind(*payload) .bind(sequence as i64 + 1) .execute(&store.pool) .await .unwrap(); + if let Some(arguments) = arguments { + Store::::query("INSERT INTO workflow_signal_records(id,workflow_command_id,signal_name,status,arguments) VALUES ($1,$2,'s','received',$3)") + .bind(format!("s-{id}")) + .bind(*id) + .bind(*arguments) + .execute(&store.pool) + .await + .unwrap(); + } } let selected = Store::::query(::RUN_COMMANDS_SQL) .bind("run") @@ -120,9 +144,9 @@ async fn described_commands_select_no_inline_payload() { .await .unwrap(); assert_eq!(selected.len(), rows.len()); - for (row, (id, payload, read)) in selected.iter().zip(&rows) { + for (row, (id, _, _, _, read)) in selected.iter().zip(&rows) { assert_eq!(::string(row, "id").unwrap(), *id); let stored = ::optional_string(row, "stored_payload").unwrap(); - assert_eq!(stored.as_deref(), read.then_some(payload.as_str()), "{id}"); + assert_eq!(stored.as_deref(), *read, "{id}"); } } diff --git a/rust/tests/execution/external_payloads.rs b/rust/tests/execution/external_payloads.rs index c3771c42e..5acb9e25a 100644 --- a/rust/tests/execution/external_payloads.rs +++ b/rust/tests/execution/external_payloads.rs @@ -133,7 +133,7 @@ async fn unclaimed(database: &TestDatabase) -> i64 { } /// One integer the database answers. -async fn count(database: &TestDatabase, query: &'static str) -> i64 { +async fn count(database: &TestDatabase, query: impl sqlx::SqlSafeStr) -> i64 { match database { TestDatabase::Sqlite(dir) => { let pool = sqlx::SqlitePool::connect(&format!( diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index bb962993b..e93b63183 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -152,11 +152,11 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { database.remove().await; } -/// The described signal command, which must show only metadata. -async fn described_signal(app: &Router, workflow_id: &str) -> Value { +/// One described command of `kind`, which must show only metadata. +async fn described(app: &Router, workflow_id: &str, kind: &str) -> Value { let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; assert_eq!(status, StatusCode::OK, "{body}"); - let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == "signal").unwrap().clone(); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == kind).unwrap().clone(); for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { assert!(command.get(field).is_none(), "{field}: {command}"); } @@ -166,7 +166,9 @@ async fn described_signal(app: &Router, workflow_id: &str) -> Value { // `WorkflowController::oversizedPayloadPreview` over each command's stored // envelope: only a stored external reference larger than the payload limit -// (2 MiB by default) omits the preview, and then the target name too. +// (2 MiB by default) omits the preview, and then the target name too. Native +// keeps a signal's stored reference in its signal record and a start's in the +// command row. #[tokio::test] async fn described_commands_omit_only_an_oversized_stored_reference() { let temp = tempfile::tempdir().unwrap(); @@ -177,9 +179,15 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", json!({"input":{"codec":"avro","external_payload":reference}})).await; assert_eq!(status, StatusCode::ACCEPTED, "{body}"); - let command = described_signal(&app, "described-reference").await; + let command = described(&app, "described-reference", "signal").await; assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); - let command_id = command["id"].as_str().unwrap().to_owned(); + let signal = command["id"].as_str().unwrap().to_owned(); + // The real stored reference is in the signal record, not the command row. + let identity = reference["reference_id"].as_str().unwrap(); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{signal}' AND arguments LIKE 'dw-native-external-payload-v1:%{identity}%'"))).await, 1); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_commands WHERE id='{signal}' AND payload LIKE 'dw-native%'"))).await, 0); // The same stored reference, as native writes it, at sizes around the // limit, and counterfactuals that are not native stored references. @@ -189,7 +197,7 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { reference["size_bytes"] = json!(size); format!("dw-native-external-payload-v1:{reference}") }; - for (payload, omitted) in [ + let cases = [ (stored(limit), false), (stored(limit + 1), true), // Not native's exact prefix, so never read. @@ -199,11 +207,29 @@ async fn described_commands_omit_only_an_oversized_stored_reference() { (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), // A large inline payload. (format!("dw-{}", "A".repeat(limit as usize + 1)), false), - ] { - database.execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_commands SET payload='{payload}' WHERE id='{command_id}'"))).await; - let command = described_signal(&app, "described-reference").await; - assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &payload[..payload.len().min(48)]); - assert_eq!(command.get("target_name").is_none(), omitted, "{command}"); + ]; + for (arguments, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{arguments}' WHERE workflow_command_id='{signal}'"))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &arguments[..arguments.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), *omitted, "{command}"); + } + // An oversized reference in the signal's command row is not its input. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{}' WHERE workflow_command_id='{signal}'", stored(limit)))).await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{}' WHERE id='{signal}'", stored(limit + 1)))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + + // A start keeps its stored input reference in the command row. + let start = described(&app, "described-reference", "start").await["id"].as_str().unwrap().to_owned(); + for (payload, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{payload}' WHERE id='{start}'"))).await; + let command = described(&app, "described-reference", "start").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "start {}: {command}", &payload[..payload.len().min(48)]); } runtime.close().await; database.remove().await; From 17029019b8b38850134c7d1a4503a7a730fb3f71 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Sat, 24 Oct 2026 02:19:19 +0000 Subject: [PATCH 14/23] Server parity: name the maintenance container from the job identity (#721) The Docker workflow isolation policy requires a stopped container's name to resolve to github.run_id, run_attempt and job in the same source. Name the SQLite job's maintenance container from GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT and GITHUB_JOB, refuse to act unless it equals the job's $RESOURCE_SCOPE-php-maintenance, and add a control proving a container outside the job scope is never touched. --- .../published-sdk-patch-profile.sh | 4 +- .../published-sdk-patch/maintenance.sh | 46 ++++++++++--------- tests/Unit/PatchCohortMaintenanceTest.mjs | 16 +++++-- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index c24c58762..738c21d1d 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -42,8 +42,8 @@ record() { --output "/evidence/published-sdk-patch/$2/record.json" } if test "$backend" = sqlite; then - run_without_maintenance "$RESOURCE_SCOPE-php-maintenance" \ - parity-evidence/published-sdk-patch/maintenance-schedule.txt -- record "$php_url" php + run_without_maintenance parity-evidence/published-sdk-patch/maintenance-schedule.txt -- \ + record "$php_url" php else record "$php_url" php fi diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh index 71cdebacb..33528001f 100644 --- a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -1,56 +1,60 @@ # Sourced by published-sdk-patch-profile.sh (Server #721). # -# run_without_maintenance CONTAINER SCHEDULE -- COMMAND... +# run_without_maintenance SCHEDULE -- COMMAND... # -# Runs COMMAND while the maintenance container is stopped, then restarts it. -# Restoration is armed before the stop is attempted and stays armed until the -# container has stayed running for a sustained check, so a failed stop, a -# failed COMMAND or a restart that exits at once never leaves it stopped. -# SCHEDULE records the container's state before, during and after. +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" maintenance_state() { - docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo unknown + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown } # Running at each of three checks, one poll interval apart. maintenance_stays_running() { local check for check in 1 2 3; do - test "$(maintenance_state "$1")" = running || return 1 + test "$(maintenance_state)" = running || return 1 sleep "$maintenance_poll_seconds" done - test "$(maintenance_state "$1")" = running + test "$(maintenance_state)" = running } # Up to three starts, each followed by the sustained check. maintenance_restore() { local attempt for attempt in 1 2 3; do - docker start "$1" > /dev/null 2>&1 || true - if maintenance_stays_running "$1"; then + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then return 0 fi done - echo "maintenance container $1 did not stay running: $(maintenance_state "$1")" >&2 + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 return 1 } run_without_maintenance() { - local container="$1" schedule="$2" - shift 2 + local schedule="$1" + shift test "$1" = -- && shift - echo "before: $(maintenance_state "$container")" > "$schedule" + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + test "$maintenance_container" = "$RESOURCE_SCOPE-php-maintenance" + echo "before: $(maintenance_state)" > "$schedule" grep -qx 'before: running' "$schedule" - # shellcheck disable=SC2064 - trap "maintenance_restore '$container' || true" EXIT - docker stop --time 2 "$container" > /dev/null - echo "during php record: $(maintenance_state "$container")" >> "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" grep -qx 'during php record: exited' "$schedule" "$@" - if ! maintenance_restore "$container"; then - echo "after php record: $(maintenance_state "$container")" >> "$schedule" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" return 1 fi trap - EXIT diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs index 9283d780e..24398e2cc 100644 --- a/tests/Unit/PatchCohortMaintenanceTest.mjs +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -18,6 +18,7 @@ const fakeDocker = `#!/usr/bin/env bash set -u dir=$(dirname "$0") echo "$*" >> "$dir/calls" +test "\${!#}" = dw-parity-7-2-sqlite-php-maintenance || exit 3 case "$1" in inspect) state=$(cat "$dir/state") @@ -30,17 +31,18 @@ case "$1" in esac `; -function run(scenario, command) { +function run(scenario, command, scope = 'dw-parity-7-2-sqlite') { const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); writeFileSync(join(dir, 'docker'), fakeDocker); chmodSync(join(dir, 'docker'), 0o755); writeFileSync(join(dir, 'state'), 'running\n'); writeFileSync(join(dir, 'calls'), ''); const result = spawnSync('bash', ['-c', - `set -euo pipefail; . "$HELPER"; run_without_maintenance maint "$SCHEDULE" -- ${command}`], { + `set -euo pipefail; . "$HELPER"; run_without_maintenance "$SCHEDULE" -- ${command}`], { encoding: 'utf8', env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, - SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05'}, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05', + GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', RESOURCE_SCOPE: scope}, }); const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); const calls = read('calls').trim().split('\n'); @@ -88,3 +90,11 @@ test('a restart that exits at once stays armed and fails without recording runni assert.equal(result.starts, 6); assert.match(result.stderr, /did not stay running/); }); + +test('a container outside the job scope is never touched', () => { + const result = run('normal', 'true', 'dw-parity-other-job'); + assert.notEqual(result.status, 0); + assert.equal(result.starts, 0); + assert.equal(result.stopIndex, -1); + assert.equal(result.state, 'running'); +}); From 42cfa99a79c9b4b0e48115bd1cc81b6ee09e189d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:15:20 +0000 Subject: [PATCH 15/23] Check native's unserved published routes against a recorded list docs/rust-server-port.md lists the published Server 2.11.1 routes native does not serve yet, and that prose drifted from the code (it said 125 of 174). An execution test now sends every method of every published API route, from the embedded PHP route table, through the native router, and requires the routes that answer 501 rust_capability_not_implemented to equal rust/tests/fixtures/unserved-php-routes.v2.11.1.txt. A port that serves a route removes it from the list in the same change. At this tree native serves 126 of 175 route methods; 49 remain. The test fails, naming the route, when one is removed from the list. --- rust/tests/execution.rs | 3 + rust/tests/execution/route_coverage.rs | 76 +++++++++++++++++++ .../fixtures/unserved-php-routes.v2.11.1.txt | 52 +++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 rust/tests/execution/route_coverage.rs create mode 100644 rust/tests/fixtures/unserved-php-routes.v2.11.1.txt diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 96f2db7e4..2915050d2 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -287,6 +287,9 @@ mod external_payloads; #[path = "execution/trace_carriers.rs"] mod trace_carriers; +#[path = "execution/route_coverage.rs"] +mod route_coverage; + enum TestDatabase { Sqlite(tempfile::TempDir), Postgres { diff --git a/rust/tests/execution/route_coverage.rs b/rust/tests/execution/route_coverage.rs new file mode 100644 index 000000000..e49e1ff40 --- /dev/null +++ b/rust/tests/execution/route_coverage.rs @@ -0,0 +1,76 @@ +use super::*; + +// The published Server 2.11.1 API routes native does not serve yet. A port +// that serves a route removes it from this list in the same change, so the +// list in docs/rust-server-port.md cannot drift from the code. +const UNSERVED: &str = include_str!("../fixtures/unserved-php-routes.v2.11.1.txt"); + +/// Sends every method of every published PHP API route through the native +/// router, with each path parameter filled in, and lists those that answer +/// 501 rust_capability_not_implemented. +#[tokio::test] +async fn unserved_published_routes_match_the_recorded_list() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let table: Value = serde_json::from_str(include_str!("../../src/runtime/php-routes.v2.11.1.json")).unwrap(); + let mut unserved = Vec::new(); + let mut checked = 0; + for route in table["routes"].as_array().unwrap() { + let (methods, uri) = (route[0].as_str().unwrap(), route[1].as_str().unwrap()); + if !uri.starts_with("api/") { + continue; + } + for method in methods.split('|').filter(|method| *method != "HEAD") { + checked += 1; + let path = format!("/{}", fill_parameters(uri)); + let worker = path.starts_with("/api/worker/"); + let body = if matches!(method, "GET" | "DELETE") { Body::empty() } else { Body::from("{}") }; + let request = Request::builder() + .method(method) + .uri(&path) + .header("authorization", "Bearer test-token") + .header("accept", "application/json") + .header("content-type", "application/json") + .header( + if worker { "x-durable-workflow-protocol-version" } else { "x-durable-workflow-control-plane-version" }, + if worker { "1.20" } else { "2" }, + ) + .body(body) + .unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 8 * 1024 * 1024).await.unwrap(); + if status == StatusCode::NOT_IMPLEMENTED + && String::from_utf8_lossy(&bytes).contains("rust_capability_not_implemented") + { + unserved.push(format!("{method} /{uri}")); + } + } + } + assert!(checked > 170, "the published route table lists {checked} API route methods"); + let recorded: Vec<&str> = UNSERVED.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect(); + let now_served: Vec<&&str> = recorded.iter().filter(|route| !unserved.iter().any(|found| found == **route)).collect(); + let now_unserved: Vec<&String> = unserved.iter().filter(|found| !recorded.contains(&found.as_str())).collect(); + assert!( + now_served.is_empty() && now_unserved.is_empty(), + "update rust/tests/fixtures/unserved-php-routes.v2.11.1.txt and the port log. Now served: {now_served:?}. Now unserved: {now_unserved:?}" + ); + assert_eq!(unserved, recorded, "keep the recorded list in route-table order"); + runtime.close().await; + database.remove().await; +} + +/// Replaces each `{parameter}` in a route with a placeholder value. +fn fill_parameters(uri: &str) -> String { + let mut path = String::new(); + let mut rest = uri; + while let Some(open) = rest.find('{') { + let close = open + rest[open..].find('}').expect("closed route parameter"); + path.push_str(&rest[..open]); + path.push_str("probe-x"); + rest = &rest[close + 1..]; + } + path.push_str(rest); + path +} diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt new file mode 100644 index 000000000..886219a91 --- /dev/null +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -0,0 +1,52 @@ +# Published Server 2.11.1 API routes that native answers with 501 +# rust_capability_not_implemented, in route-table order. Checked by +# rust/tests/execution/route_coverage.rs. +POST /api/bridge-adapters/webhook/{adapter} +GET /api/service-endpoints +POST /api/service-endpoints +GET /api/service-endpoints/{endpointName} +PUT /api/service-endpoints/{endpointName} +DELETE /api/service-endpoints/{endpointName} +GET /api/service-endpoints/{endpointName}/services +POST /api/service-endpoints/{endpointName}/services +GET /api/service-endpoints/{endpointName}/services/{serviceName} +PUT /api/service-endpoints/{endpointName}/services/{serviceName} +DELETE /api/service-endpoints/{endpointName}/services/{serviceName} +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +PUT /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +DELETE /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName} +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/execute +GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId} +POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}/cancel +GET /api/worker-sessions +GET /api/worker-sessions/{sessionId} +POST /api/worker/activity-tasks/{taskId}/acknowledge-cancellation +POST /api/worker/sessions +DELETE /api/worker/sessions/{sessionId} +POST /api/worker/sessions/{sessionId}/heartbeat +POST /api/worker/update-validation-tasks/poll +POST /api/worker/update-validation-tasks/{taskId}/approve +POST /api/worker/update-validation-tasks/{taskId}/reject +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/deliver +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/open +POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/prepare +POST /api/worker/workflow-tasks/{taskId}/local-activities/checkpoint +POST /api/worker/workflow-tasks/{taskId}/local-activities/checkpoint-group +POST /api/worker/workflow-tasks/{taskId}/local-activities/prepare +POST /api/worker/workflow-tasks/{taskId}/local-activities/recover +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat +POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome +POST /api/workflows/import/embedded-v2 +POST /api/workflows/import/waterline-v1 +GET /api/workflows/{workflowId}/message-streams +GET /api/workflows/{workflowId}/message-streams/{streamName} +POST /api/workflows/{workflowId}/message-streams/{streamName}/messages +GET /api/workflows/{workflowId}/nexus-operations +GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations +POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close +POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items From 22271b6082c88b5e749691b2e218ca42256790f1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:26:12 +0000 Subject: [PATCH 16/23] Server parity: activity cancellation acknowledgement fixture, recorded from published 2.11.1 Fixture first for POST /api/worker/activity-tasks/{task}/acknowledge- cancellation, the last activity-task route native does not serve. - The probe follows the cooperative path over raw worker HTTP: request cancellation, deliver it to the activity from the workflow task, then acknowledge from the activity worker. It records every ownership, protocol and validation refusal, the receipt on the status route, the duplicate, a later outcome, a claim polled on 1.19, an immediate cancel, a retried attempt and a run terminated after delivery. - The contract pins the published 2.11.1 bodies and event. Identities and times come from the observation; their relations are exact. - One documented reference boundary: published 2.11.1 delivers a retried activity's cancellation to the execution's first task (Workflow #923), so that attempt's acknowledgement is refused as not fenced. The contract accepts exactly that defect's signature or the correct acknowledgement, nothing else. - The unit test runs the contract on the recorded 2.11.1 observation, on the corrected retry, and on 16 mutations it must refuse. Published 2.11.1 passes 3/3 locally with its maintenance loop running. Native still answers 501 here; this branch adds the native route next. --- .../activity-cancellation-ack-contract.mjs | 245 ++++++++++++++++++ .../activity-cancellation-ack-probe.php | 193 ++++++++++++++ .../conformance/server-parity/contract.mjs | 4 + scripts/conformance/server-parity/probe.php | 7 + ...activity-cancellation-acknowledgement.json | 129 +++++++++ ...cellation-acknowledgement-observation.json | 1 + ...ityActivityCancellationAckContractTest.mjs | 95 +++++++ 7 files changed, 674 insertions(+) create mode 100644 scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs create mode 100644 scripts/conformance/server-parity/activity-cancellation-ack-probe.php create mode 100644 tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json create mode 100644 tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json create mode 100644 tests/Unit/ServerParityActivityCancellationAckContractTest.mjs diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs new file mode 100644 index 000000000..eb15e8d94 --- /dev/null +++ b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs @@ -0,0 +1,245 @@ +import assert from 'node:assert/strict'; + +const MICROS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; +const ULID = /^[0-9a-z]{26}$/; +const REQUEST = /^[0-9A-Z]{26}$/; +const ACKS = ['before_request', 'before_delivery', 'foreign_owner', 'unknown_attempt', 'unknown_task', 'task_mismatch', + 'other_namespace', 'protocol_1_19', 'protocol_missing', 'protocol_1_21', 'missing_fields', 'non_string_fields', + 'long_request_id', 'wrong_request_id', 'acknowledged', 'duplicate', 'legacy_ack', 'immediate_ack', + 'stale_first_attempt_own_task', 'stale_first_attempt_new_task', 'stale_second_attempt_old_task', 'stale_second_attempt', + 'terminated_ack']; +const MAIN_EVENTS = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', + 'CooperativeCancellationRequested', 'ActivityCancelled', 'CooperativeCancellationDelivered']; + +// Milliseconds since the epoch of a UTC timestamp with any fraction length. +function at(value, label) { + assert.ok(typeof value === 'string', `${label}: a timestamp, got ${JSON.stringify(value)}`); + const millis = Date.parse(value.replace(/\.(\d{3})\d*Z$/, '.$1Z')); + assert.ok(!Number.isNaN(millis), `${label}: ${value}`); + return millis; +} + +// Reviewed activity cancellation acknowledgement +// (ActivityTaskController::acknowledgeCancellation over Workflow's +// ActivityCancellationAcknowledgement::recordStopped), recorded against +// published Server 2.11.1. Identities and times come from the observation; +// their relations and every other field are exact. +// +// One documented reference boundary: published 2.11.1 delivers a cooperative +// cancellation of a retried activity to the execution's first task, not the +// live attempt's task (Workflow #923), so that attempt's acknowledgement is +// refused. The retried-attempt case accepts either the correct outcome or +// exactly that defect's signature, and nothing else; native's own execution +// tests require the correct outcome. +export function checkActivityCancellationAcknowledgement(fixture, observation, workflowId) { + const definition = fixture.activity_cancellation_acknowledgement; + const state = observation.activity_cancellation_acknowledgement; + const projection = {scope: 'activity_cancellation_acknowledgement', cleanup_timeout_seconds: definition.cleanup_timeout_seconds}; + assert.ok(state && typeof state === 'object', 'actual cancellation acknowledgement observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded cancellation acknowledgement explicitly inapplicable'); + return projection; + } + const equal = (actual, expected, label) => assert.deepStrictEqual(actual, expected, label); + const main = state.main; + equal([state.applicable, state.namespace, main.workflow_id], [true, `${workflowId}-cack`.toLowerCase(), `${workflowId}-main`], + 'acknowledgement namespace and run'); + equal(state.discovery, {activity_cancellation_acknowledgement: true, cooperative_cancellation: true}, + 'cluster info advertises cooperative cancellation and its acknowledgement'); + for (const key of ACKS) { + equal([state[key]?.response?.protocol_version, state[key]?.acknowledgement_capability], ['1.20', true], + `${key} answers in worker protocol 1.20 and advertises acknowledgement`); + } + + // A cancellation request on the main run, read back from its workflow task. + equal(state.request.status, 202, 'cancellation requested'); + const {cancellation_request: request, ...requested} = state.request.response; + equal(requested, {workflow_id: main.workflow_id, run_id: main.run_id, accepted: true, duplicate: false, run_status: 'waiting', + reason: null}, 'cancellation request receipt'); + assert.match(request.request_id, REQUEST, 'request id'); + assert.match(request.requested_at, MICROS, 'request time in UTC microseconds'); + equal(at(request.cleanup_deadline_at, 'cleanup deadline') - at(request.requested_at, 'request time'), + definition.cleanup_timeout_seconds * 1000, 'the cleanup budget runs from the request'); + equal([request.delivery_sequence, request.delivered_at, request.history_refresh_page_token], [null, null, 'MA=='], + 'request not yet delivered'); + equal(state.poll_cancellation_request, request, 'the workflow task carries the request'); + const requestId = request.request_id; + + const ack = (claim, requestValue, fields) => ({task_id: claim.task_id, activity_attempt_id: claim.activity_attempt_id, + lease_owner: claim.lease_owner, request_id: requestValue, acknowledged: false, duplicate: false, reason: null, + history_event_id: null, heartbeat_recorded: false, ...fields, protocol_version: '1.20'}); + const refused = (key, claim, requestValue, reason) => equal(state[key], + {status: 409, response: ack(claim, requestValue, {reason}), acknowledgement_capability: true}, key); + refused('before_request', main, 'not-yet-requested', 'cancellation_request_mismatch'); + refused('before_delivery', main, requestId, 'activity_cancellation_not_recorded'); + + // Workflow's attempt status for a claim, before any field is overridden. + const status = (claim, fields = {}) => ({task_id: claim.task_id, activity_attempt_id: claim.activity_attempt_id, + lease_owner: claim.lease_owner, can_continue: true, cancel_requested: false, reason: null, heartbeat_recorded: false, + last_heartbeat_at: null, run_status: 'waiting', run_closed_reason: null, run_closed_at: null, activity_status: 'running', + attempt_status: 'running', task_status: 'leased', worker_session: null, ...fields, protocol_version: '1.20'}); + const withoutTimes = (key, recorded) => { + equal(recorded.status, 200, `${key} answers`); + const {lease_expires_at: lease, deadlines, ...body} = recorded.response; + equal(Object.keys(deadlines), ['start_to_close'], `${key} deadlines`); + assert.match(deadlines.start_to_close, MICROS, `${key} start-to-close deadline`); + return {lease, deadline: deadlines.start_to_close, body}; + }; + const pending = withoutTimes('status_before_delivery', state.status_before_delivery); + assert.match(pending.lease, MICROS, 'a pending request leaves the lease'); + equal(pending.body, status(main), 'a pending request does not stop the attempt'); + + // Delivery to the activity cancels its attempt; the worker learns it from + // its heartbeat and finds the request on its status. + const deliveryBody = (claim, requestValue) => ({delivered: true, task_id: state.delivery.response.task_id, + workflow_run_id: claim.run_id, request_id: requestValue, sequence: 1, call_kind: 'activity', sequence_span: 1, + operation_sequence: null, operation_sequence_span: 1, reason: null, protocol_version: '1.20'}); + equal(state.delivery, {status: 200, response: deliveryBody(main, requestId), acknowledgement_capability: true}, 'delivered'); + assert.match(state.delivery.response.task_id, ULID, 'delivered from the cancellation workflow task'); + equal(state.heartbeat_after_delivery, {status: 200, acknowledgement_capability: true, response: {task_id: main.task_id, + activity_attempt_id: main.activity_attempt_id, lease_owner: main.lease_owner, cancel_requested: true, can_continue: false, + reason: 'attempt_closed', run_closed_reason: null, run_closed_at: null, heartbeat_recorded: false, lease_expires_at: null, + last_heartbeat_at: null, worker_session: null, protocol_version: '1.20'}}, 'heartbeat reports the stop'); + const stopped = (claim) => status(claim, {can_continue: false, cancel_requested: true, reason: 'attempt_closed', + activity_status: 'cancelled', attempt_status: 'cancelled', task_status: 'cancelled'}); + const receipt = (cancellationEvent, fields = {}) => ({request_id: requestId, root_request_id: requestId, + cleanup_deadline_at: request.cleanup_deadline_at, cancellation_history_event_id: cancellationEvent, callback_state: 'unknown', + history_event_id: null, acknowledged_at: null, received_after_deadline: null, ...fields}); + const unacknowledged = withoutTimes('status_before_ack', state.status_before_ack); + equal([unacknowledged.lease, unacknowledged.deadline], [null, pending.deadline], 'delivery ends the lease, not the deadline'); + const {cancellation_acknowledgement: before, ...beforeBody} = unacknowledged.body; + const cancellationEvent = before?.cancellation_history_event_id; + assert.match(cancellationEvent ?? '', ULID, 'the receipt names the ActivityCancelled event'); + equal(beforeBody, stopped(main), 'the delivered attempt is stopped'); + equal(before, receipt(cancellationEvent), 'the receipt before acknowledgement'); + equal(state.history_before_ack.map(event => event.event_type), MAIN_EVENTS, 'request and delivery history'); + + // Ownership in PHP's order and shape, protocol, then Laravel's validation. + // None records anything. + const fault = (key, code, task, attempt, error, reason, extra = {}) => equal(state[key], {status: code, + acknowledgement_capability: true, response: {task_id: task, activity_attempt_id: attempt, error, reason, ...extra, + protocol_version: '1.20'}}, key); + fault('foreign_owner', 409, main.task_id, main.activity_attempt_id, 'Activity attempt lease is owned by another worker.', + 'lease_owner_mismatch', {lease_owner: main.lease_owner}); + fault('unknown_attempt', 404, main.task_id, 'no-such-attempt', 'Activity attempt not found.', 'attempt_not_found'); + fault('unknown_task', 404, 'no-such-task', main.activity_attempt_id, 'Activity task not found.', 'task_not_found'); + fault('task_mismatch', 409, main.workflow_task_id, main.activity_attempt_id, 'Activity attempt is not leased for this task.', + 'task_mismatch'); + fault('other_namespace', 404, main.task_id, main.activity_attempt_id, 'Activity task not found.', 'task_not_found'); + equal(state.protocol_1_19, {status: 422, acknowledgement_capability: true, response: {acknowledged: false, + reason: 'activity_cancellation_acknowledgement_not_supported', minimum_protocol_version: '1.20', + unavailable: ['worker_protocol'], protocol_version: '1.20'}}, 'a 1.19 request'); + for (const key of ['protocol_missing', 'protocol_1_21', 'missing_fields', 'non_string_fields', 'long_request_id']) { + const {status: code, ...body} = definition.refusals[key]; + equal(state[key], {status: code, acknowledgement_capability: true, response: {...body, protocol_version: '1.20'}}, key); + } + refused('wrong_request_id', main, 'other-request', 'cancellation_request_mismatch'); + equal(state.history_after_refusals, state.history_before_ack, 'refusals record nothing'); + + // The acknowledgement: one event, then the receipt reads stopped and a + // repeat returns the same event. + const acknowledgedEvent = state.acknowledged?.response?.history_event_id; + assert.match(acknowledgedEvent ?? '', ULID, 'acknowledgement event id'); + equal(state.acknowledged, {status: 200, acknowledgement_capability: true, + response: ack(main, requestId, {acknowledged: true, history_event_id: acknowledgedEvent})}, 'acknowledged'); + const acknowledgedStatus = withoutTimes('status_after_ack', state.status_after_ack); + const {cancellation_acknowledgement: after, ...afterBody} = acknowledgedStatus.body; + equal([acknowledgedStatus.lease, acknowledgedStatus.deadline, afterBody], [null, pending.deadline, stopped(main)], + 'acknowledgement changes no attempt state'); + assert.match(after?.acknowledged_at ?? '', MICROS, 'acknowledgement time in UTC microseconds'); + equal(after, receipt(cancellationEvent, {callback_state: 'stopped', history_event_id: acknowledgedEvent, + acknowledged_at: after.acknowledged_at, received_after_deadline: false}), 'the receipt after acknowledgement'); + assert.ok(at(after.acknowledged_at, 'acknowledged') < at(request.cleanup_deadline_at, 'deadline'), 'acknowledged within the budget'); + equal(state.duplicate, {status: 200, acknowledgement_capability: true, + response: ack(main, requestId, {acknowledged: true, duplicate: true, history_event_id: acknowledgedEvent})}, 'duplicate'); + equal(state.completed_after_ack, {status: 409, acknowledgement_capability: true, response: {task_id: main.task_id, + activity_attempt_id: main.activity_attempt_id, outcome: 'completed', recorded: false, reason: 'stale_attempt', + next_task_id: null, lease_owner: main.lease_owner, activity_status: 'cancelled', attempt_status: 'cancelled', + task_status: 'cancelled', protocol_version: '1.20'}}, 'a later outcome is refused'); + const history = state.history_after_ack; + equal(history.slice(0, -1), state.history_before_ack, 'acknowledgement only appends'); + const event = history.at(-1); + const delivered = history.at(-2); + const {task, ...payload} = event.payload; + equal([event.event_type, event.sequence, event.principal], ['ActivityCancellationAcknowledged', delivered.sequence + 1, null], + 'one acknowledgement event'); + equal(payload, {sequence: 1, activity_execution_id: main.activity_execution_id, activity_attempt_id: main.activity_attempt_id, + lease_owner: main.lease_owner, cancellation_history_event_id: cancellationEvent, request_id: requestId, + root_request_id: requestId, cleanup_deadline_at: request.cleanup_deadline_at, callback_state: 'stopped', + evidence_source: 'activity_worker', acknowledged_at: after.acknowledged_at, received_after_deadline: false}, + 'acknowledgement payload'); + const {available_at: available, leased_at: leased, connection, ...snapshot} = task; + assert.match(available, MICROS, 'snapshot availability'); + assert.match(leased, MICROS, 'snapshot lease time'); + const cancelledTask = history.find(item => item.event_type === 'ActivityCancelled')?.payload?.task; + equal([cancelledTask?.id, cancelledTask?.status], [main.task_id, 'cancelled'], 'delivery snapshots the cancelled activity task'); + assert.ok(typeof connection === 'string' && connection !== '', 'snapshot connection'); + equal(connection, cancelledTask.connection, 'both activity task snapshots name one storage connection'); + equal(snapshot, {id: main.task_id, type: 'activity', status: 'cancelled', lease_owner: main.lease_owner, attempt_count: 1, + repair_count: 0, queue: main.queue}, 'acknowledgement snapshots the cancelled activity task, without a lease'); + + // A claim polled on protocol 1.19 cannot acknowledge. + const legacy = state.legacy_claim; + equal([state.legacy_request, state.legacy_delivery.status, state.legacy_delivery.response.delivered], [202, 200, true], + 'legacy claim cancelled cooperatively'); + equal(state.legacy_ack, {status: 409, acknowledgement_capability: true, response: {acknowledged: false, + reason: 'activity_cancellation_acknowledgement_not_supported', required_capability: 'cooperative_cancellation', + minimum_protocol_version: '1.20', worker_id: legacy.lease_owner, unavailable: ['worker_claim_capability'], + remediation: 'Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.', + protocol_version: '1.20'}}, 'a 1.19 claim cannot acknowledge'); + + // An immediate cancel records no cooperative request and no receipt. + const immediate = state.immediate_claim; + equal(state.immediate_cancel.status, 200, 'immediate cancel accepted'); + const commandId = state.immediate_cancel.response.command_id; + assert.match(commandId, ULID, 'immediate cancel command'); + refused('immediate_ack', immediate, commandId, 'cancellation_request_mismatch'); + const cancelled = withoutTimes('immediate_status', state.immediate_status); + assert.match(cancelled.body.run_closed_at ?? '', MICROS, 'immediate cancel closes the run'); + equal([cancelled.lease, cancelled.body], [null, status(immediate, {can_continue: false, cancel_requested: true, + reason: 'run_cancelled', run_status: 'cancelled', run_closed_reason: 'cancelled', run_closed_at: cancelled.body.run_closed_at, + activity_status: 'cancelled', attempt_status: 'cancelled', task_status: 'cancelled'})], 'immediate cancel status, no receipt'); + + // A retried attempt: the first attempt is fenced, and each attempt belongs + // to its own task. + const [first, second] = [state.stale_first, state.stale_second]; + equal([state.stale_fail.status, state.stale_fail.response.recorded, state.stale_request, state.stale_delivery.status, + state.stale_delivery.response.delivered], [200, true, 202, 200, true], 'retried attempt cancelled cooperatively'); + assert.notEqual(first.task_id, second.task_id, 'the retry has its own task'); + const staleRequest = state.stale_delivery.response.request_id; + refused('stale_first_attempt_own_task', first, staleRequest, 'activity_cancellation_acknowledgement_fence_mismatch'); + fault('stale_first_attempt_new_task', 409, second.task_id, first.activity_attempt_id, + 'Activity attempt is not leased for this task.', 'task_mismatch'); + fault('stale_second_attempt_old_task', 409, first.task_id, second.activity_attempt_id, + 'Activity attempt is not leased for this task.', 'task_mismatch'); + const staleCancelled = state.stale_history.find(item => item.event_type === 'ActivityCancelled'); + equal(staleCancelled?.payload?.activity_attempt?.id, second.activity_attempt_id, 'the live attempt is cancelled'); + const correct = staleCancelled.payload.task?.id === second.task_id; + if (correct) { + equal(state.stale_second_attempt.status, 200, 'the live attempt acknowledges'); + equal(state.stale_second_attempt.response.acknowledged, true, 'the live attempt acknowledges'); + } else { + // Workflow #923 in published 2.11.1: the delivery cancels the first, + // already completed task, so the live task stays leased and the + // acknowledgement is refused as not fenced. + equal(staleCancelled.payload.task?.id, first.task_id, 'reference defect: delivery snapshots the first task'); + equal([state.stale_second_status.response.attempt_status, state.stale_second_status.response.task_status], + ['cancelled', 'leased'], 'reference defect: the live task stays leased'); + refused('stale_second_attempt', second, staleRequest, 'activity_cancellation_not_fenced'); + } + + // A run terminated after delivery still records the late stop, after its + // terminal event. + const terminated = state.terminated_claim; + equal([state.terminated_request, state.terminated_delivery.status, state.terminate], [202, 200, 200], + 'terminated after delivery'); + const terminatedEvent = state.terminated_ack?.response?.history_event_id; + assert.match(terminatedEvent ?? '', ULID, 'terminated acknowledgement event id'); + equal(state.terminated_ack, {status: 200, acknowledgement_capability: true, response: ack(terminated, + state.terminated_delivery.response.request_id, {acknowledged: true, history_event_id: terminatedEvent})}, + 'a terminated run still acknowledges'); + equal(state.terminated_history, [...MAIN_EVENTS, 'TerminateRequested', 'WorkflowTerminated', 'ActivityCancellationAcknowledged'], + 'the late acknowledgement follows the terminal event'); + return projection; +} diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-probe.php b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php new file mode 100644 index 000000000..46af4b3c3 --- /dev/null +++ b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php @@ -0,0 +1,193 @@ +createNamespace($namespace, 'activity cancellation acknowledgement λ', 30); + $admin->createNamespace($other, 'activity cancellation acknowledgement other λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new CancellationAckHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = [], ?string $protocol = '1.20', ?string $space = null) use ($http, $token, $namespace): array { + $options = ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $space ?? $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => $protocol, + ])]; + if ($body !== null) { + $options['json'] = $body === [] ? (object) [] : $body; + } + $response = $http->request($method, $path, $options); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + $capability = is_array($decoded) && array_key_exists('server_capabilities', $decoded) + ? ($decoded['server_capabilities']['activity_cancellation_acknowledgement'] ?? null) : 'absent'; + unset($decoded['server_capabilities'], $decoded['control_plane']); + + return ['status' => $response->getStatusCode(), 'response' => $decoded, 'acknowledgement_capability' => $capability]; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Activity cancellation acknowledgement '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + // Avro payloads precomputed by the published serializer: [] and 'done'. + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $done = ['codec' => 'avro', 'blob' => $definition['result_blob']]; + // Starts a run on its own queue with its own cooperative worker and claims + // its single activity, on the given poll protocol. + $claim = static function (string $suffix, array $retry = ['max_attempts' => 1, 'backoff_seconds' => []], string $activityProtocol = '1.20') use ($raw, $ok, $empty, $definition, $workflowId): array { + $id = $workflowId.'-'.$suffix; + $queue = $id.'-queue'; + $worker = $id.'-worker'; + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => ['cooperative_cancellation'], + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])]), 201, 'registration'); + $poll = ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 0]; + $started = $ok($raw('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => $definition['workflow_type'], + 'task_queue' => $queue, 'input' => $empty]), 201, 'start'); + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $poll), 200, 'workflow claim')['task']; + $ok($raw('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $empty, 'retry_policy' => $retry, 'start_to_close_timeout' => 300]]]), 200, 'schedule activity'); + $activity = $ok($raw('POST', '/api/worker/activity-tasks/poll', $poll, $activityProtocol), 200, 'activity claim')['task']; + + return ['workflow_id' => $id, 'run_id' => $started['run_id'], 'queue' => $queue, 'worker' => $worker, + 'workflow_task_id' => $task['task_id'], 'task_id' => $activity['task_id'], + 'activity_attempt_id' => $activity['activity_attempt_id'], + 'activity_execution_id' => $activity['activity_execution_id'] ?? null, 'lease_owner' => $activity['lease_owner']]; + }; + $own = static fn (array $claimed): array => ['activity_attempt_id' => $claimed['activity_attempt_id'], 'lease_owner' => $claimed['lease_owner']]; + $ack = static fn (array $claimed, ?array $body, ?string $protocol = '1.20', ?string $space = null, ?string $task = null): array => $raw('POST', + '/api/worker/activity-tasks/'.($task ?? $claimed['task_id']).'/acknowledge-cancellation', $body, $protocol, $space); + $status = static fn (array $claimed): array => $raw('POST', '/api/worker/activity-tasks/'.$claimed['task_id'].'/status', $own($claimed)); + $history = static fn (array $claimed): array => $ok($raw('GET', + '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/history', null), 200, 'history')['events'] ?? []; + // Requests cooperative cancellation of the selected run. + $request = static fn (array $claimed): array => $raw('POST', + '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/request-cancellation', + ['reason' => 'acknowledgement fixture λ', 'cleanup_timeout_seconds' => $definition['cleanup_timeout_seconds']]); + // Polls the run's cancellation workflow task and delivers the request to + // the activity, the first command of the run. + $deliver = static function (array $claimed) use ($raw, $ok): array { + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', + ['worker_id' => $claimed['worker'], 'task_queue' => $claimed['queue'], 'timeout_seconds' => 0]), 200, 'cancellation claim')['task']; + $requestId = $task['cancellation_request']['request_id'] ?? null; + + return ['request_id' => $requestId, 'poll_cancellation_request' => $task['cancellation_request'] ?? null, + 'delivery' => $raw('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/deliver-cancellation', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'request_id' => $requestId, 'sequence' => 1, 'call_kind' => 'activity'])]; + }; + + $state = ['applicable' => true, 'namespace' => $namespace]; + $cluster = $ok($raw('GET', '/api/cluster/info', null), 200, 'cluster info'); + $capabilities = $cluster['worker_protocol']['server_capabilities'] ?? []; + $state['discovery'] = ['activity_cancellation_acknowledgement' => $capabilities['activity_cancellation_acknowledgement'] ?? 'absent', + 'cooperative_cancellation' => $capabilities['cooperative_cancellation'] ?? 'absent']; + + $main = $state['main'] = $claim('main'); + $state['before_request'] = $ack($main, $own($main) + ['request_id' => 'not-yet-requested']); + $state['request'] = $request($main); + $requestId = $state['request']['response']['cancellation_request']['request_id'] ?? null; + $owned = $own($main) + ['request_id' => $requestId]; + $state['before_delivery'] = $ack($main, $owned); + $state['status_before_delivery'] = $status($main); + $delivered = $deliver($main); + $state['poll_cancellation_request'] = $delivered['poll_cancellation_request']; + $state['delivery'] = $delivered['delivery']; + $state['heartbeat_after_delivery'] = $raw('POST', '/api/worker/activity-tasks/'.$main['task_id'].'/heartbeat', $own($main)); + $state['status_before_ack'] = $status($main); + $state['history_before_ack'] = $history($main); + $state['foreign_owner'] = $ack($main, ['lease_owner' => 'someone-else'] + $owned); + $state['unknown_attempt'] = $ack($main, ['activity_attempt_id' => 'no-such-attempt'] + $owned); + $state['unknown_task'] = $ack($main, $owned, '1.20', null, 'no-such-task'); + $state['task_mismatch'] = $ack($main, $owned, '1.20', null, $main['workflow_task_id']); + $state['other_namespace'] = $ack($main, $owned, '1.20', $other); + $state['protocol_1_19'] = $ack($main, $owned, '1.19'); + $state['protocol_missing'] = $ack($main, $owned, null); + $state['protocol_1_21'] = $ack($main, $owned, '1.21'); + $state['missing_fields'] = $ack($main, []); + $state['non_string_fields'] = $ack($main, ['activity_attempt_id' => 5, 'lease_owner' => ['x'], 'request_id' => true]); + $state['long_request_id'] = $ack($main, $own($main) + ['request_id' => str_repeat('r', 256)]); + $state['wrong_request_id'] = $ack($main, $own($main) + ['request_id' => 'other-request']); + $state['history_after_refusals'] = $history($main); + $state['acknowledged'] = $ack($main, $owned + ['message' => 'stopped']); + $state['status_after_ack'] = $status($main); + $state['duplicate'] = $ack($main, $owned); + $state['completed_after_ack'] = $raw('POST', '/api/worker/activity-tasks/'.$main['task_id'].'/complete', $own($main) + ['result' => $done]); + $state['history_after_ack'] = $history($main); + + // The activity was claimed on protocol 1.19, so its claim cannot + // acknowledge even on a 1.20 request. + $legacy = $state['legacy_claim'] = $claim('legacy', activityProtocol: '1.19'); + $state['legacy_request'] = $request($legacy)['status']; + $legacyDelivered = $deliver($legacy); + $state['legacy_delivery'] = $legacyDelivered['delivery']; + $state['legacy_ack'] = $ack($legacy, $own($legacy) + ['request_id' => $legacyDelivered['request_id']]); + + // An immediate cancel records no cooperative request. + $immediate = $state['immediate_claim'] = $claim('immediate'); + $state['immediate_cancel'] = $raw('POST', '/api/workflows/'.$immediate['workflow_id'].'/cancel', ['reason' => 'acknowledgement fixture']); + $state['immediate_ack'] = $ack($immediate, $own($immediate) + + ['request_id' => $state['immediate_cancel']['response']['command_id'] ?? 'no-command-id']); + $state['immediate_status'] = $status($immediate); + + // A retried attempt: attempt 1 fails retryably and attempt 2 is claimed on + // a new task before the request is delivered. + $first = $state['stale_first'] = $claim('stale', ['max_attempts' => 2, 'backoff_seconds' => [0]]); + $state['stale_fail'] = $raw('POST', '/api/worker/activity-tasks/'.$first['task_id'].'/fail', $own($first) + + ['failure' => ['message' => 'retry me', 'type' => 'parity.retryable', 'non_retryable' => false]]); + $retried = $ok($raw('POST', '/api/worker/activity-tasks/poll', + ['worker_id' => $first['worker'], 'task_queue' => $first['queue'], 'timeout_seconds' => 0]), 200, 'retry claim')['task']; + $second = $state['stale_second'] = ['task_id' => $retried['task_id'], 'activity_attempt_id' => $retried['activity_attempt_id'], + 'lease_owner' => $retried['lease_owner']] + $first; + $state['stale_request'] = $request($first)['status']; + $staleDelivered = $deliver($first); + $state['stale_delivery'] = $staleDelivered['delivery']; + $staleRequest = ['request_id' => $staleDelivered['request_id']]; + $state['stale_first_attempt_own_task'] = $ack($first, $own($first) + $staleRequest); + $state['stale_first_attempt_new_task'] = $ack($first, $own($first) + $staleRequest, '1.20', null, $second['task_id']); + $state['stale_second_attempt_old_task'] = $ack($second, $own($second) + $staleRequest, '1.20', null, $first['task_id']); + $state['stale_second_attempt'] = $ack($second, $own($second) + $staleRequest); + $state['stale_second_status'] = $status($second); + $state['stale_history'] = $history($first); + + // Terminated after delivery, then acknowledged. + $terminated = $state['terminated_claim'] = $claim('terminated'); + $state['terminated_request'] = $request($terminated)['status']; + $terminatedDelivered = $deliver($terminated); + $state['terminated_delivery'] = $terminatedDelivered['delivery']; + $state['terminate'] = $raw('POST', '/api/workflows/'.$terminated['workflow_id'].'/terminate', ['reason' => 'acknowledgement fixture'])['status']; + $state['terminated_ack'] = $ack($terminated, $own($terminated) + ['request_id' => $terminatedDelivered['request_id']]); + $state['terminated_history'] = array_column($history($terminated), 'event_type'); + + return $state; +} diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 69b947698..2394ae4bf 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -13,6 +13,7 @@ import {checkLateWorkflowCompletion} from './late-workflow-completion-contract.m import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; +import {checkActivityCancellationAcknowledgement} from './activity-cancellation-ack-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; import {checkOperatorDashboardTypes} from './operator-dashboard-types-contract.mjs'; @@ -151,6 +152,8 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const searchAttributes = fixture.search_attributes ? checkSearchAttributes(fixture, observation, workflowId) : null; const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; + const activityCancellationAcknowledgement = fixture.activity_cancellation_acknowledgement + ? checkActivityCancellationAcknowledgement(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; const operatorDashboardTypes = fixture.operator_dashboard_types ? checkOperatorDashboardTypes(fixture, observation, workflowId) : null; @@ -706,6 +709,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(searchAttributes ? {search_attributes: searchAttributes} : {}), ...(storageTest ? {storage_test: storageTest} : {}), ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), + ...(activityCancellationAcknowledgement ? {activity_cancellation_acknowledgement: activityCancellationAcknowledgement} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), ...(operatorDashboardTypes ? {operator_dashboard_types: operatorDashboardTypes} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 4df73a65f..0aa235b35 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -431,6 +431,8 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $searchAttributes = isset($fixture['search_attributes']) ? finishHttpSearchAttributes($client, $fixture, $workflowId, $url) : null; $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; + $activityCancellationAcknowledgement = isset($fixture['activity_cancellation_acknowledgement']) + ? finishHttpActivityCancellationAcknowledgement($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; $operatorDashboardTypes = isset($fixture['operator_dashboard_types']) ? finishHttpOperatorDashboardTypes($client, $fixture, $workflowId, $url) : null; @@ -539,6 +541,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'search_attributes' => $searchAttributes, 'storage_test' => $storageTest, 'activity_lease_renewal' => $activityLeaseRenewal, + 'activity_cancellation_acknowledgement' => $activityCancellationAcknowledgement, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, 'operator_dashboard_types' => $operatorDashboardTypes, @@ -851,6 +854,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/search-attribute-probe.php'; require __DIR__.'/storage-test-probe.php'; require __DIR__.'/activity-lease-renewal-probe.php'; + require __DIR__.'/activity-cancellation-ack-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; require __DIR__.'/operator-dashboard-types-probe.php'; @@ -1020,6 +1024,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['activity_lease_renewal'])) { $observation['activity_lease_renewal'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['activity_cancellation_acknowledgement'])) { + $observation['activity_cancellation_acknowledgement'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } diff --git a/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json b/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json new file mode 100644 index 000000000..3acc7f6d7 --- /dev/null +++ b/tests/Fixtures/ServerParity/activity-cancellation-acknowledgement.json @@ -0,0 +1,129 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "activity-cancellation-acknowledgement", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "cancellation acknowledgement λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "cancellation acknowledgement λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "activity_cancellation_acknowledgement": { + "workflow_type": "parity.v1.cancel_ack", + "activity_type": "parity.v1.cancel_ack_activity", + "cleanup_timeout_seconds": 60, + "empty_blob": "wwHioz3/VYAiNwwA", + "result_blob": "wwHioz3/VYAiNwoIZG9uZQ==", + "refusals": { + "protocol_missing": { + "status": 400, + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests." + }, + "protocol_1_21": { + "status": 400, + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches." + }, + "missing_fields": { + "status": 422, + "message": "The activity attempt id field is required. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "activity_attempt_id": [ + "The activity attempt id field is required." + ], + "lease_owner": [ + "The lease owner field is required." + ], + "request_id": [ + "The request id field is required." + ] + }, + "validation_errors": { + "activity_attempt_id": [ + "The activity attempt id field is required." + ], + "lease_owner": [ + "The lease owner field is required." + ], + "request_id": [ + "The request id field is required." + ] + } + }, + "non_string_fields": { + "status": 422, + "message": "The activity attempt id field must be a string. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "activity_attempt_id": [ + "The activity attempt id field must be a string." + ], + "lease_owner": [ + "The lease owner field must be a string." + ], + "request_id": [ + "The request id field must be a string." + ] + }, + "validation_errors": { + "activity_attempt_id": [ + "The activity attempt id field must be a string." + ], + "lease_owner": [ + "The lease owner field must be a string." + ], + "request_id": [ + "The request id field must be a string." + ] + } + }, + "long_request_id": { + "status": 422, + "message": "The request id field must not be greater than 255 characters.", + "reason": "validation_failed", + "errors": { + "request_id": [ + "The request id field must not be greater than 255 characters." + ] + }, + "validation_errors": { + "request_id": [ + "The request id field must not be greater than 255 characters." + ] + } + } + } + }, + "contract": "POST /api/worker/activity-tasks/{task}/acknowledge-cancellation (published Server 2.11, advertised as server_capabilities.activity_cancellation_acknowledgement, also in cluster info) records an activity worker's stop after a cooperative cancellation reached its attempt. The path is request-cancellation, then deliver-cancellation with call_kind activity from the workflow task, which cancels the attempt and records ActivityCancelled; the activity heartbeat then reports attempt_closed and the status route carries the cancellation_acknowledgement receipt with the request id. The acknowledgement answers 200 and records one ActivityCancellationAcknowledged event with the stop's evidence and the cancelled task's snapshot; the receipt then reads stopped and a repeat answers duplicate with the same event, while later outcomes stay refused. Before the request, with another request id or after an immediate cancel it answers cancellation_request_mismatch; before delivery activity_cancellation_not_recorded; for an older attempt of the execution activity_cancellation_acknowledgement_fence_mismatch. Ownership is checked in PHP's order with its bodies, then the claim: an attempt polled on protocol 1.19 cannot acknowledge. A request below protocol 1.20 is refused before validation; missing, non-string or over-long fields fail Laravel's validation. A run terminated after delivery still records the late stop, after its terminal event. Published 2.11.1 delivers a retried activity's cancellation to the execution's first task (Workflow #923), so that attempt's acknowledgement is refused as not fenced; the contract accepts exactly that reference defect or the correct acknowledgement. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json new file mode 100644 index 000000000..bb922669c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json @@ -0,0 +1 @@ +{"applicable": true, "namespace": "pr1-activity-cancellation-acknowledgement-cack", "discovery": {"activity_cancellation_acknowledgement": true, "cooperative_cancellation": true}, "main": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5pce4j95qsm4nh4kj68q5v9", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "worker": "pr1-activity-cancellation-acknowledgement-main-worker", "workflow_task_id": "01m5pce4jm5yjq52g1gh13zq17", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker"}, "before_request": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "not-yet-requested", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "request": {"status": 202, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5pce4j95qsm4nh4kj68q5v9", "accepted": true, "duplicate": false, "run_status": "waiting", "cancellation_request": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "reason": null}, "acknowledgement_capability": "absent"}, "before_delivery": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_recorded", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_delivery": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": true, "cancel_requested": false, "reason": null, "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T07:24:08.525226Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "running", "attempt_status": "running", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "poll_cancellation_request": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce56p4sjtyxzrv4e86j4d", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "heartbeat_after_delivery": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancel_requested": true, "can_continue": false, "reason": "attempt_closed", "run_closed_reason": null, "run_closed_at": null, "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_ack": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_before_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}], "foreign_owner": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity attempt lease is owned by another worker.", "reason": "lease_owner_mismatch", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_attempt": {"status": 404, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "no-such-attempt", "error": "Activity attempt not found.", "reason": "attempt_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_task": {"status": 404, "response": {"task_id": "no-such-task", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "task_mismatch": {"status": 409, "response": {"task_id": "01m5pce4jm5yjq52g1gh13zq17", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "other_namespace": {"status": 404, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_19": {"status": 422, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "minimum_protocol_version": "1.20", "unavailable": ["worker_protocol"], "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_missing": {"status": 400, "response": {"error": "Missing worker protocol version header.", "reason": "missing_protocol_version", "supported_version": "1.20", "requested_version": null, "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_21": {"status": 400, "response": {"error": "Unsupported worker protocol version.", "reason": "unsupported_protocol_version", "supported_version": "1.20", "requested_version": "1.21", "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "missing_fields": {"status": 422, "response": {"message": "The activity attempt id field is required. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "non_string_fields": {"status": 422, "response": {"message": "The activity attempt id field must be a string. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "long_request_id": {"status": 422, "response": {"message": "The request id field must not be greater than 255 characters.", "reason": "validation_failed", "errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "validation_errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "wrong_request_id": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "other-request", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}], "acknowledged": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_after_ack": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "callback_state": "stopped", "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "acknowledged_at": "2026-10-24T07:19:10.136717Z", "received_after_deadline": false}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "duplicate": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": true, "duplicate": true, "reason": null, "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "completed_after_ack": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "outcome": "completed", "recorded": false, "reason": "stale_attempt", "next_task_id": null, "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}, {"sequence": 8, "event_type": "ActivityCancellationAcknowledged", "timestamp": "2026-10-24T07:19:10.137480Z", "principal": null, "payload": {"sequence": 1, "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "callback_state": "stopped", "evidence_source": "activity_worker", "acknowledged_at": "2026-10-24T07:19:10.136717Z", "received_after_deadline": false, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}], "legacy_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-legacy", "run_id": "01m5pce71geceqgfrvb8mgzghc", "queue": "pr1-activity-cancellation-acknowledgement-legacy-queue", "worker": "pr1-activity-cancellation-acknowledgement-legacy-worker", "workflow_task_id": "01m5pce71tq57627pyxztvv08b", "task_id": "01m5pce79fbyqm23p77xmhqz4n", "activity_attempt_id": "01M5PCE7F9QG7DMWQKX1NQA5T6", "activity_execution_id": "01m5pce79b8nyyk1n89129btqt", "lease_owner": "pr1-activity-cancellation-acknowledgement-legacy-worker"}, "legacy_request": 202, "legacy_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce7mp7x45m0zy0mvynpxe", "workflow_run_id": "01m5pce71geceqgfrvb8mgzghc", "request_id": "01M5PCE7MFYSZ2T8SGGPZ5EEGB", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "legacy_ack": {"status": 409, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "required_capability": "cooperative_cancellation", "minimum_protocol_version": "1.20", "worker_id": "pr1-activity-cancellation-acknowledgement-legacy-worker", "unavailable": ["worker_claim_capability"], "remediation": "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5pce84zt7623a5j72s1g4p3", "queue": "pr1-activity-cancellation-acknowledgement-immediate-queue", "worker": "pr1-activity-cancellation-acknowledgement-immediate-worker", "workflow_task_id": "01m5pce8594pxswss94983p374", "task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "activity_execution_id": "01m5pce8cgkm09vw8y9grnzhgk", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker"}, "immediate_cancel": {"status": 200, "response": {"outcome": "cancelled", "workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5pce84zt7623a5j72s1g4p3", "requested_run_id": null, "resolved_run_id": "01m5pce84zt7623a5j72s1g4p3", "command_id": "01m5pce8men6x37hxx01ke5gc7", "command_sequence": 2, "target_scope": "instance", "workflow_type": "parity.v1.cancel_ack", "command_status": "accepted", "command_source": "control_plane", "reason": "acknowledgement fixture", "rejection_reason": null, "validation_errors": []}, "acknowledgement_capability": "absent"}, "immediate_ack": {"status": 409, "response": {"task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "request_id": "01m5pce8men6x37hxx01ke5gc7", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_status": {"status": 200, "response": {"task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "can_continue": false, "cancel_requested": true, "reason": "run_cancelled", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "cancelled", "run_closed_reason": "cancelled", "run_closed_at": "2026-10-24T07:19:12.280883Z", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:12.145804Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first": {"workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5pce91n33d9v1cfb6ncvh2z", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5pce91z14aca5gjm9p1dj6k", "task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker"}, "stale_fail": {"status": 200, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "outcome": "failed", "recorded": true, "reason": null, "next_task_id": "01m5pce9m78fmfh88ce3jqee3e", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5pce91n33d9v1cfb6ncvh2z", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5pce91z14aca5gjm9p1dj6k", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g"}, "stale_request": 202, "stale_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce9wk5q46atc83g1pd2sr", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_own_task": {"status": 409, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_acknowledgement_fence_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_new_task": {"status": 409, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt_old_task": {"status": 409, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt": {"status": 409, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_fenced", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_status": {"status": 200, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T07:24:13.443669Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T07:24:13.443566Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "cancellation_history_event_id": "01m5pcea5t4kt69sf5ns6661jy", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_history": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:12.696936Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce91q61dexcb48296z3nx", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce91q61dexcb48296z3nx", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:12.693941Z", "applied_at": "2026-10-24T07:19:12.693946Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:12.700510Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "workflow_command_id": "01m5pce91q61dexcb48296z3nx", "command": {"id": "01m5pce91q61dexcb48296z3nx", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:12.693941Z", "applied_at": "2026-10-24T07:19:12.693946Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:12.971905Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce91z14aca5gjm9p1dj6k", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:12.703025Z", "leased_at": "2026-10-24T07:19:12.817249Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:20:12.817202Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:12.718128Z", "last_dispatched_at": "2026-10-24T07:19:12.718128Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:13.160244Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.155017Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:24:13.155152Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 5, "event_type": "ActivityRetryScheduled", "timestamp": "2026-10-24T07:19:13.290467Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "retry_task_id": "01m5pce9m78fmfh88ce3jqee3e", "retry_of_task_id": "01m5pce9ae7cyss9tzhnbch75b", "retry_available_at": "2026-10-24T07:19:13.284568Z", "retry_backoff_seconds": 0, "retry_after_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "retry_after_attempt": 1, "max_attempts": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "exception_type": "parity.retryable", "activity_attempt": {"id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "task_id": "01m5pce9ae7cyss9tzhnbch75b", "status": "failed", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "closed_at": "2026-10-24T07:19:13.284791Z"}, "exception_class": "RuntimeException", "message": "retry me", "code": 0, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "status": "pending", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.155017Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "completed", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 6, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:13.448417Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "status": "running", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.443566Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "task": {"id": "01m5pce9m78fmfh88ce3jqee3e", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:13.284568Z", "leased_at": "2026-10-24T07:19:13.443566Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:24:13.443669Z", "attempt_count": 2, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:13.322057Z", "last_dispatched_at": "2026-10-24T07:19:13.322057Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:13.551795Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:13.548489Z", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "lineage": [{"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z"}]}, "command": {"id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDoxMy41NDg0ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFOVdDWjJHUEgwUVNNMjZEUUJGSx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U5V0NaMkdQSDBRU00yNkRRQkZLMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXnByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LXN0YWxlKHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlOTFuMzNkOXYxY2ZiNm5jdmgyeiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDA3OjE5OjEzLjU0ODQ4OVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDoxMy41NDg0ODlaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVQQ0U5V0NaMkdQSDBRU00yNkRRQkZLKHdvcmtmbG93X2luc3RhbmNlX2lkCl5wcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1zdGFsZR53b3JrZmxvd19ydW5faWQKNDAxbTVwY2U5MW4zM2Q5djFjZmI2bmN2aDJ6AAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-stale/runs/01m5pce91n33d9v1cfb6ncvh2z/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:209709fdff9285a483ef965eae0aa5b23a07df94ec273ea1fe3ad52822cd4701", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:13.548489Z", "applied_at": "2026-10-24T07:19:13.548489Z"}}}, {"sequence": 8, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:13.849540Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "cancelled_at": "2026-10-24T07:19:13.843676Z", "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "status": "cancelled", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.443566Z", "closed_at": "2026-10-24T07:19:13.843676Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "activity_attempt": {"id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "task_id": "01m5pce9m78fmfh88ce3jqee3e", "attempt_number": 2, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "started_at": "2026-10-24T07:19:13.443566Z", "closed_at": "2026-10-24T07:19:13.843676Z"}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:13.853476Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:13.548489Z", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "lineage": [{"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z"}]}, "task": {"id": "01m5pce9wk5q46atc83g1pd2sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:13.548489Z", "leased_at": "2026-10-24T07:19:13.723070Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:19:23.723013Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:13.577280Z", "last_dispatched_at": "2026-10-24T07:19:13.577280Z"}}}], "terminated_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-terminated", "run_id": "01m5pceask5cer356gz7m630am", "queue": "pr1-activity-cancellation-acknowledgement-terminated-queue", "worker": "pr1-activity-cancellation-acknowledgement-terminated-worker", "workflow_task_id": "01m5pceaswp3wq87dt3xy1ftkq", "task_id": "01m5pceb2yhq2n7mhkg1kmwryh", "activity_attempt_id": "01M5PCEB8D36PD1YJ91T2PEK4B", "activity_execution_id": "01m5pceb2tga6mgbnp7k8yfca8", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker"}, "terminated_request": 202, "terminated_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pcebcnbv13ep26mtw7y73c", "workflow_run_id": "01m5pceask5cer356gz7m630am", "request_id": "01M5PCEBCD5HM18A6YAE8QFA5E", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminate": 200, "terminated_ack": {"status": 200, "response": {"task_id": "01m5pceb2yhq2n7mhkg1kmwryh", "activity_attempt_id": "01M5PCEB8D36PD1YJ91T2PEK4B", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker", "request_id": "01M5PCEBCD5HM18A6YAE8QFA5E", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5pcebxrbwvbd8n7t8ezc0ks", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminated_history": ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityStarted", "CooperativeCancellationRequested", "ActivityCancelled", "CooperativeCancellationDelivered", "TerminateRequested", "WorkflowTerminated", "ActivityCancellationAcknowledged"]} \ No newline at end of file diff --git a/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs new file mode 100644 index 000000000..bd249b3ef --- /dev/null +++ b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkActivityCancellationAcknowledgement} from '../../scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/activity-cancellation-acknowledgement.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json', import.meta.url))); +const id = 'pr1-activity-cancellation-acknowledgement'; +const projection = {scope: 'activity_cancellation_acknowledgement', cleanup_timeout_seconds: 60}; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', activity_cancellation_acknowledgement: state}; +}; +const check = state => checkActivityCancellationAcknowledgement(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), projection); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', activity_cancellation_acknowledgement: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), projection); +}); + +// The correct retried-attempt outcome (Workflow #923 fixed): the delivery +// cancels the live task and its attempt acknowledges. +const fixedRetry = state => { + const cancelled = state.stale_history.find(event => event.event_type === 'ActivityCancelled'); + cancelled.payload.task.id = state.stale_second.task_id; + state.stale_second_attempt.status = 200; + Object.assign(state.stale_second_attempt.response, {acknowledged: true, reason: null, history_event_id: '01m5pcebxrbwvbd8n7t8ezc0kz'}); +}; + +test('the correct retried-attempt acknowledgement is accepted', () => { + assert.deepStrictEqual(check(observation(fixedRetry)), projection); +}); + +for (const [label, change, message] of [ + ['a refused live attempt whose delivery cancelled its own task', state => { + state.stale_history.find(event => event.event_type === 'ActivityCancelled').payload.task.id = state.stale_second.task_id; + }, /the live attempt acknowledges/], + ['the reference defect without its leased live task', state => { + state.stale_second_status.response.task_status = 'cancelled'; + }, /reference defect: the live task stays leased/], + ['an acknowledgement that does not advertise the capability', state => { + state.acknowledged.acknowledgement_capability = false; + }, /acknowledged answers in worker protocol 1.20 and advertises acknowledgement/], + ['cluster info without the capability', state => { + state.discovery.activity_cancellation_acknowledgement = 'absent'; + }, /cluster info advertises/], + ['a refusal that records history', state => { + state.history_after_refusals.push(structuredClone(state.history_before_ack.at(-1))); + }, /refusals record nothing/], + ['a status receipt before delivery', state => { + state.status_before_delivery.response.cancellation_acknowledgement = {}; + }, /a pending request does not stop the attempt/], + ['a receipt that already reads stopped', state => { + state.status_before_ack.response.cancellation_acknowledgement.callback_state = 'stopped'; + }, /the receipt before acknowledgement/], + ['a duplicate that records a second event', state => { + state.duplicate.response.history_event_id = '01m5pce6htfpz4bjzf62naz6cz'; + }, /duplicate/], + ['an acknowledgement event without its evidence source', state => { + delete state.history_after_ack.at(-1).payload.evidence_source; + }, /acknowledgement payload/], + ['an acknowledgement snapshot that keeps a lease', state => { + state.history_after_ack.at(-1).payload.task.lease_expires_at = '2026-10-24T07:24:08.525107Z'; + }, /without a lease/], + ['a 1.19 claim that acknowledges', state => { + state.legacy_ack = structuredClone(state.acknowledged); + }, /a 1\.19 claim cannot acknowledge/], + ['a 1.19 request refused by validation instead', state => { + state.protocol_1_19 = structuredClone(state.missing_fields); + }, /a 1\.19 request/], + ['an immediate cancel with a receipt', state => { + state.immediate_status.response.cancellation_acknowledgement = {}; + }, /immediate cancel status, no receipt/], + ['a stale first attempt that acknowledges', state => { + state.stale_first_attempt_own_task.status = 200; + }, /stale_first_attempt_own_task/], + ['a terminated run whose acknowledgement precedes its terminal event', state => { + state.terminated_history = [...state.terminated_history.slice(0, -3), 'ActivityCancellationAcknowledged', + 'TerminateRequested', 'WorkflowTerminated']; + }, /follows the terminal event/], + ['a cleanup budget other than the requested one', state => { + state.request.response.cancellation_request.cleanup_deadline_at = '2026-10-24T07:21:08.747589Z'; + }, /the cleanup budget runs from the request/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} From ee55d34e1922ce12963f70543172a8c7f8d1735a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:57:10 +0000 Subject: [PATCH 17/23] Rust #325: activity cancellation acknowledgement, with cooperative delivery to activities Native now serves POST /api/worker/activity-tasks/{task}/acknowledge- cancellation like published Server 2.11.1, and the cooperative path that leads to it: - deliver-cancellation accepts call_kind activity with the default try_cancel policy. It cancels the open execution, its current attempt and that attempt's own task, and records ActivityCancelled under the request with PHP's attempt and task snapshots. Published 2.11.1 picks the execution's first task instead, which breaks a retried activity (Workflow #923); native cancels the live task. - request-cancellation no longer refuses a run with open activities. - Activity polls bind the immutable claim, as workflow polls do, so a claim polled on protocol 1.19 cannot acknowledge. - The heartbeat reports a delivered stop as PHP does (200, cancel_requested, attempt_closed), and the status route carries the cancellation_acknowledgement receipt. - The acknowledgement follows PHP's order: protocol, validation, ownership guard, claim, then the fence, request, recorded cancellation, snapshot, context and duplicate checks. It records one ActivityCancellationAcknowledged event with the stop's evidence and the cancelled task's snapshot, and never changes the attempt. - server_capabilities advertises activity_cancellation_acknowledgement, and named namespaces admit deliver-cancellation and the new route. Smaller parity fixes found by the fixture: the status route and task snapshots format times with microseconds, and stale activity outcome receipts carry the worker protocol fields. Explicit wait_cancellation_completed and abandon activity policies stay unported: native does not record an activity's policy, so every delivery is try_cancel. The activity-cancellation-acknowledgement fixture passes 3/3 against native on SQLite locally, taking the correct retried-attempt branch. --- rust/src/runtime/activity_cancellation_ack.rs | 250 ++++++++++++++++++ rust/src/runtime/activity_failures.rs | 3 +- rust/src/runtime/activity_heartbeats.rs | 24 +- rust/src/runtime/activity_lease_renewal.rs | 18 +- rust/src/runtime/cooperative.rs | 73 ++++- rust/src/runtime/execution.rs | 1 + rust/src/runtime/http.rs | 20 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 10 +- rust/src/runtime/worker_incarnations.rs | 3 +- .../activity-cancellation-ack-contract.mjs | 6 +- 11 files changed, 381 insertions(+), 28 deletions(-) create mode 100644 rust/src/runtime/activity_cancellation_ack.rs diff --git a/rust/src/runtime/activity_cancellation_ack.rs b/rust/src/runtime/activity_cancellation_ack.rs new file mode 100644 index 000000000..f89e65662 --- /dev/null +++ b/rust/src/runtime/activity_cancellation_ack.rs @@ -0,0 +1,250 @@ +//! An activity worker's stop acknowledgement after a cooperative +//! cancellation reached its attempt, as published Server 2.11 serves it +//! (`ActivityTaskController::acknowledgeCancellation` over Workflow 2.10's +//! `ActivityCancellationAcknowledgement::recordStopped`), and the receipt +//! the status route shows for it. An acknowledgement records one event with +//! the stop's evidence; it never changes the attempt, its lease or the +//! cleanup deadline, and a repeat returns the original event. +use super::activity_lease_renewal::{micros, worker}; +use super::laravel_rules::{self, Rule}; +use super::transaction::Transaction; +use super::{ + Result, RuntimeError, + store::{capabilities, now}, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; + +/// Published PHP's validation rules for the request, in their order. +const RULES: &laravel_rules::Rules = &[ + ("activity_attempt_id", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("lease_owner", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("request_id", &[Rule::Required, Rule::String, Rule::Max(255)]), +]; + +const REMEDIATION: &str = "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer."; + +/// Whether a request's worker protocol is 1.20 or newer. +fn protocol_supported(protocol: &str) -> bool { + super::cooperative::supports(&json!(["cooperative_cancellation"]), protocol) +} + +store_methods! { + pub(crate) async fn acknowledge_activity_cancellation(&self, task_id: &str, body: Value, protocol: &str) -> Result { + // The protocol gate comes before validation, as in PHP. + if !protocol_supported(protocol) { + return Err(worker(StatusCode::UNPROCESSABLE_ENTITY, json!({"acknowledged":false, + "reason":"activity_cancellation_acknowledgement_not_supported","minimum_protocol_version":"1.20", + "unavailable":["worker_protocol"]}))); + } + let body = laravel_rules::normalize(&body); + if let Err(RuntimeError::Protocol { status, response }) = laravel_rules::validate(&body, RULES) { + return Err(worker(status, response)); + } + let attempt_id = body["activity_attempt_id"].as_str().unwrap_or_default(); + let owner = body["lease_owner"].as_str().unwrap_or_default(); + let request_id = body["request_id"].as_str().unwrap_or_default(); + let refusal = |status, error: &str, reason: &str| { + worker(status, json!({"task_id":task_id,"activity_attempt_id":attempt_id,"error":error,"reason":reason})) + }; + let outcome = |acknowledged: bool, duplicate: bool, reason: Option<&str>, event: Option| { + json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner,"request_id":request_id, + "acknowledged":acknowledged,"duplicate":duplicate,"reason":reason,"history_event_id":event, + "heartbeat_recorded":false}) + }; + // PHP answers a refusal of the recorder with 404 only for a missing + // attempt, otherwise 409. + let refused = |reason: &'static str| { + let status = if reason == "activity_attempt_not_found" { StatusCode::NOT_FOUND } else { StatusCode::CONFLICT }; + worker(status, outcome(false, false, Some(reason), None)) + }; + let mut tx = self.begin().await?; + // PHP's ownership guard, in its order: the task in this namespace, + // then the attempt by its identity, its task and its owner. + let task = Self::query("SELECT id FROM workflow_tasks WHERE id=$1 AND namespace=$2") + .bind(task_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| refusal(StatusCode::NOT_FOUND, "Activity task not found.", "task_not_found"))?; + let Some(rows) = Self::attempt_rows(&mut tx, attempt_id).await? else { + return Err(refusal(StatusCode::NOT_FOUND, "Activity attempt not found.", "attempt_not_found")); + }; + let observed = Self::attempt_status(attempt_id, &rows, None)?; + if observed["workflow_task_id"].as_str() != Some(DB::string(&task, "id")?.as_str()) { + return Err(refusal(StatusCode::CONFLICT, "Activity attempt is not leased for this task.", "task_mismatch")); + } + if observed["lease_owner"].as_str() != Some(owner) { + let mut error = json!({"task_id":task_id,"activity_attempt_id":attempt_id, + "error":"Activity attempt lease is owned by another worker.","reason":"lease_owner_mismatch"}); + error["lease_owner"] = observed["lease_owner"].clone(); + return Err(worker(StatusCode::CONFLICT, error)); + } + let (Some(activity), Some(run), Some(task)) = (&rows.activity, &rows.run, &rows.task) else { + return Err(refused("activity_attempt_not_found")); + }; + // The original claim must come from a cooperative worker on 1.20, + // checked before the request or its delivery. + if !Self::claim_supports_cancellation(task)? { + return Err(worker(StatusCode::CONFLICT, json!({"acknowledged":false, + "reason":"activity_cancellation_acknowledgement_not_supported","required_capability":"cooperative_cancellation", + "minimum_protocol_version":"1.20","worker_id":owner,"unavailable":["worker_claim_capability"], + "remediation":REMEDIATION}))); + } + let run_id = DB::string(run, "id")?; + let execution_id = DB::string(activity, "id")?; + // `recordStopped`'s fence: the execution's current attempt, on this + // run, leased to this owner on both the attempt and its task. + if DB::optional_string(activity, "current_attempt_id")?.as_deref() != Some(attempt_id) + || DB::string(activity, "workflow_run_id")? != run_id + || DB::string(task, "workflow_run_id")? != run_id + || DB::optional_string(&rows.attempt, "lease_owner")?.as_deref() != Some(owner) + || DB::optional_string(task, "lease_owner")?.as_deref() != Some(owner) + { + return Err(refused("activity_cancellation_acknowledgement_fence_mismatch")); + } + if DB::optional_string(run, "cancellation_request_command_id")?.as_deref() != Some(request_id) { + return Err(refused("cancellation_request_mismatch")); + } + let Some((cancellation_event, cancellation)) = + Self::activity_cancellation_event(&mut tx, &run_id, request_id, &execution_id, attempt_id).await? + else { + return Err(refused("activity_cancellation_not_recorded")); + }; + let snapshot = &cancellation["activity_attempt"]; + if snapshot["id"] != attempt_id || snapshot["status"] != "cancelled" || snapshot["lease_owner"] != owner + || snapshot["task_id"] != task_id || snapshot["activity_execution_id"] != execution_id.as_str() + { + return Err(refused("activity_cancellation_snapshot_mismatch")); + } + let Some(context) = Self::cooperative_context(&mut tx, &run_id, request_id).await? else { + return Err(refused("cancellation_context_not_recorded")); + }; + if let Some((event, _)) = Self::acknowledgement_event(&mut tx, &run_id, &cancellation_event, attempt_id).await? { + tx.commit().await?; + return Ok(with_protocol(outcome(true, true, None, Some(event)))); + } + if DB::string(&rows.attempt, "status")? != "cancelled" || DB::string(activity, "status")? != "cancelled" + || DB::string(task, "status")? != "cancelled" + { + return Err(refused("activity_cancellation_not_fenced")); + } + let at = now(); + let acknowledged_at = micros(at); + let mut payload = json!({"sequence":DB::number(activity,"sequence")?,"activity_execution_id":execution_id, + "activity_attempt_id":attempt_id,"lease_owner":owner,"cancellation_history_event_id":cancellation_event, + "request_id":request_id,"root_request_id":context.root_request_id,"cleanup_deadline_at":micros(context.deadline), + "callback_state":"stopped","evidence_source":"activity_worker","acknowledged_at":acknowledged_at, + "received_after_deadline":at >= context.deadline}); + payload["task"] = Self::task_snapshot(task)?; + Self::append_by_command_id(&mut tx, &run_id, "ActivityCancellationAcknowledged", payload, Some(task_id), request_id).await?; + Self::query("UPDATE workflow_runs SET last_progress_at=$1 WHERE id=$2") + .bind(DB::bind_time(at)).bind(&run_id).execute(&mut *tx).await?; + let event = Self::query("SELECT h.id FROM workflow_history_events h JOIN workflow_runs r ON r.id=h.workflow_run_id AND r.last_history_sequence=h.sequence WHERE h.workflow_run_id=$1") + .bind(&run_id).fetch_one(&mut *tx).await?; + let event = DB::string(&event, "id")?; + tx.summary_changed(&run_id); + tx.commit().await?; + Ok(with_protocol(outcome(true, false, None, Some(event)))) + } + + /// The status route's `cancellation_acknowledgement` receipt: present once + /// the run's cooperative request cancelled this attempt. + pub(super) async fn activity_cancellation_receipt( + tx: &mut Transaction<'_, DB>, + run: &DB::Row, + execution_id: &str, + attempt_id: &str, + ) -> Result> { + let Some(request_id) = DB::optional_string(run, "cancellation_request_command_id")? else { + return Ok(None); + }; + let run_id = DB::string(run, "id")?; + let Some((cancellation_event, _)) = + Self::activity_cancellation_event(tx, &run_id, &request_id, execution_id, attempt_id).await? + else { + return Ok(None); + }; + let Some(context) = Self::cooperative_context(tx, &run_id, &request_id).await? else { + return Ok(None); + }; + let mut receipt = json!({"request_id":request_id,"root_request_id":context.root_request_id, + "cleanup_deadline_at":micros(context.deadline),"cancellation_history_event_id":cancellation_event, + "callback_state":"unknown","history_event_id":null,"acknowledged_at":null,"received_after_deadline":null}); + if let Some((event, payload)) = Self::acknowledgement_event(tx, &run_id, &cancellation_event, attempt_id).await? { + receipt["callback_state"] = payload["callback_state"].clone(); + receipt["history_event_id"] = json!(event); + receipt["acknowledged_at"] = payload["acknowledged_at"].clone(); + receipt["received_after_deadline"] = payload["received_after_deadline"].clone(); + } + Ok(Some(receipt)) + } + + /// The `ActivityCancelled` event the request recorded for this attempt. + async fn activity_cancellation_event( + tx: &mut Transaction<'_, DB>, + run_id: &str, + request_id: &str, + execution_id: &str, + attempt_id: &str, + ) -> Result> { + let events = Self::query("SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='ActivityCancelled' ORDER BY sequence") + .bind(run_id).bind(request_id).fetch_all(&mut **tx).await?; + for event in events { + let payload = DB::document_row(&event, "payload")?; + if payload["activity_execution_id"] == execution_id && payload["activity_attempt_id"] == attempt_id { + return Ok(Some((DB::string(&event, "id")?, payload))); + } + } + Ok(None) + } + + /// An earlier acknowledgement of the same cancellation by this attempt. + async fn acknowledgement_event( + tx: &mut Transaction<'_, DB>, + run_id: &str, + cancellation_event: &str, + attempt_id: &str, + ) -> Result> { + let events = Self::query("SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='ActivityCancellationAcknowledged' ORDER BY sequence") + .bind(run_id).fetch_all(&mut **tx).await?; + for event in events { + let payload = DB::document_row(&event, "payload")?; + if payload["cancellation_history_event_id"] == cancellation_event && payload["activity_attempt_id"] == attempt_id { + return Ok(Some((DB::string(&event, "id")?, payload))); + } + } + Ok(None) + } + + /// The request's root and original cleanup deadline, from its + /// `CooperativeCancellationRequested` event. + async fn cooperative_context(tx: &mut Transaction<'_, DB>, run_id: &str, request_id: &str) -> Result> { + let Some(requested) = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='CooperativeCancellationRequested' ORDER BY sequence LIMIT 1") + .bind(run_id).bind(request_id).fetch_optional(&mut **tx).await? + else { + return Ok(None); + }; + let cancellation = &DB::document_row(&requested, "payload")?["cancellation"]; + let Some(deadline) = cancellation["cleanup_deadline_at"].as_str() + .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok()) + .map(|value| value.with_timezone(&chrono::Utc)) + else { + return Ok(None); + }; + let root_request_id = cancellation["lineage"].as_array().and_then(|lineage| lineage.first()) + .and_then(|root| root["request_id"].as_str()).unwrap_or(request_id).to_owned(); + Ok(Some(CooperativeContext { root_request_id, deadline })) + } +} + +struct CooperativeContext { + root_request_id: String, + deadline: chrono::DateTime, +} + +fn with_protocol(mut response: Value) -> Value { + response["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + response["server_capabilities"] = capabilities(); + response +} diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 062f7720a..f4d8ab03c 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -430,7 +430,8 @@ where return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":reported, - "recorded":false,"reason":"stale_attempt","next_task_id":null}), + "recorded":false,"reason":"stale_attempt","next_task_id":null, + "protocol_version":super::WORKER_PROTOCOL_VERSION,"server_capabilities":super::store::capabilities()}), }); } Self::closed_run_outcome(&mut tx, &task, &body, reported).await?; diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 51c16efc6..1460fa8db 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -128,12 +128,20 @@ where (true, false, None) }; // Observation preserves the existing lease and never records progress. - let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, + let mut response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, "can_continue":can_continue,"cancel_requested":cancel_requested,"reason":reason,"heartbeat_recorded":false, - "lease_expires_at":expires,"last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?), - "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?, + // PHP formats each time in UTC with microseconds. + "lease_expires_at":expires.map(super::cooperative::timestamp), + "last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?).map(super::cooperative::timestamp), + "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?.map(super::cooperative::timestamp), "activity_status":activity_status,"attempt_status":attempt_status,"task_status":task_status, "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); + // A cooperative request that reached this attempt shows its receipt. + if cancel_requested + && let Some(receipt) = Self::activity_cancellation_receipt(tx, &run, &DB::string(&activity, "id")?, attempt_id).await? + { + response["cancellation_acknowledgement"] = receipt; + } Ok((response, Self::cancellation_request(&run)?.is_none())) } @@ -193,11 +201,13 @@ where other => other, })?; // Published PHP answers with this structured refusal for ordinary - // expired authority, and for any closed run, whose activity is told - // it was cancelled. This read never records progress or renews a - // lease; a concurrently closed attempt remains fenced. + // expired authority, for any closed run, whose activity is told it + // was cancelled, and for an attempt a cooperative cancellation + // reached. This read never records progress or renews a lease; a + // concurrently closed attempt remains fenced. let closed = !observed["run_closed_reason"].is_null(); - if observed["can_continue"] == false && (closed || ordinary && observed["cancel_requested"] == false) { + let delivered = observed["cancel_requested"] == true && observed["reason"] == "attempt_closed"; + if observed["can_continue"] == false && (closed || delivered || ordinary && observed["cancel_requested"] == false) { let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], "cancel_requested":observed["cancel_requested"],"can_continue":false,"reason":observed["reason"], "run_closed_reason":observed["run_closed_reason"],"run_closed_at":observed["run_closed_at"], diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs index ac1a84f9e..f5d6d3cb6 100644 --- a/rust/src/runtime/activity_lease_renewal.rs +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -20,7 +20,7 @@ const RULES: &laravel_rules::Rules = &[ ]; /// A worker-protocol response with PHP's protocol fields. -fn worker(status: StatusCode, mut response: Value) -> RuntimeError { +pub(super) fn worker(status: StatusCode, mut response: Value) -> RuntimeError { response["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); response["server_capabilities"] = capabilities(); RuntimeError::Protocol { status, response } @@ -35,16 +35,16 @@ fn outcome_status(reason: Option<&str>) -> StatusCode { } } -fn micros(at: DateTime) -> String { +pub(super) fn micros(at: DateTime) -> String { at.to_rfc3339_opts(SecondsFormat::Micros, true) } /// The rows an attempt's authority is read from. -struct Rows { - attempt: R, - activity: Option, - run: Option, - task: Option, +pub(super) struct Rows { + pub(super) attempt: R, + pub(super) activity: Option, + pub(super) run: Option, + pub(super) task: Option, } store_methods! { @@ -157,7 +157,7 @@ store_methods! { Ok(status["can_continue"] == true && status["workflow_task_id"] == task_id && status["lease_owner"] == owner) } - async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { + pub(super) async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { let Some(attempt) = Self::query("SELECT * FROM activity_attempts WHERE id=$1") .bind(attempt_id) .fetch_optional(&mut **tx) @@ -188,7 +188,7 @@ store_methods! { /// Workflow's `attemptStatus` over its continuation state. `refused` /// replaces the state with a refusal that cannot continue. - fn attempt_status(attempt_id: &str, rows: &Rows, refused: Option<&'static str>) -> Result { + pub(super) fn attempt_status(attempt_id: &str, rows: &Rows, refused: Option<&'static str>) -> Result { let (can_continue, cancel_requested, reason) = match refused { Some(reason) => (false, false, Some(reason)), None => Self::continuation(rows, now())?, diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index df633552d..366dc0cdb 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -202,7 +202,7 @@ where .filter(|span| (1..=PARALLEL_SPAN_LIMIT).contains(span) && sequence.checked_add(*span).is_some()) .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_cancellation_delivery"))?, }; - if !matches!(call_kind, "timer" | "child" | "parallel" | "condition" | "signal") + if !matches!(call_kind, "timer" | "child" | "parallel" | "condition" | "signal" | "activity") || (call_kind != "parallel" && span != 1) || body.get("operation_sequence").is_some_and(|v| !v.is_null()) || body @@ -336,6 +336,14 @@ where "operation_sequence_span":null,"reason":"cancellation_waiting_for_child","claim_released":true})); } } + } else if call_kind == "activity" { + // `cancelOpenWaitAtSequence` with the default `try_cancel` + // policy: an open activity at the sequence is cancelled under the + // request. Native records no explicit activity cancellation + // policy, so it has no waiting or abandoning delivery. + if sequence < next { + Self::deliver_activity_cancellation(&mut tx, &run_id, request_id, sequence, now()).await?; + } } else if matches!(call_kind, "condition" | "signal") { if sequence < next { Self::validate_wait_cancellation_boundary(&mut tx, &run_id, sequence, call_kind, request_sequence).await?; @@ -411,6 +419,63 @@ where /// at its index, and a group whose members all resolved before the /// request is not eligible. Native cancels timer members only; a group /// with another kind of member keeps its earlier refusal. + /// `ActivityCancellation::record` for a cooperative request: the open + /// execution at `sequence`, its current attempt and that attempt's own + /// task are cancelled, and `ActivityCancelled` names the request. PHP + /// 2.11.1 picks the execution's first task here, which for a retried + /// activity is an earlier, closed one (Workflow #923); native cancels the + /// live attempt's task, so its worker can acknowledge the stop. + async fn deliver_activity_cancellation( + tx: &mut Transaction<'_, DB>, + run_id: &str, + request_id: &str, + sequence: i64, + at: DateTime, + ) -> Result<()> { + let Some(activity) = Self::query("SELECT * FROM activity_executions WHERE workflow_run_id=$1 AND sequence=$2 AND status IN ('pending','running')") + .bind(run_id).bind(sequence).fetch_optional(&mut **tx).await? + else { + return Ok(()); + }; + let activity_id = DB::string(&activity, "id")?; + let attempt_id = DB::optional_string(&activity, "current_attempt_id")?; + Self::query("UPDATE activity_executions SET status='cancelled',closed_at=COALESCE(closed_at,$1) WHERE id=$2") + .bind(DB::bind_time(at)).bind(&activity_id).execute(&mut **tx).await?; + let mut attempt_snapshot = Value::Null; + let mut task_id = None; + if let Some(attempt_id) = &attempt_id { + Self::query("UPDATE activity_attempts SET status='cancelled',lease_expires_at=NULL,closed_at=COALESCE(closed_at,$1) WHERE id=$2 AND activity_execution_id=$3 AND workflow_run_id=$4") + .bind(DB::bind_time(at)).bind(attempt_id).bind(&activity_id).bind(run_id).execute(&mut **tx).await?; + let attempt = Self::query("SELECT * FROM activity_attempts WHERE id=$1 AND activity_execution_id=$2 AND workflow_run_id=$3") + .bind(attempt_id).bind(&activity_id).bind(run_id).fetch_one(&mut **tx).await?; + let live = DB::string(&attempt, "workflow_task_id")?; + // The task keeps its owner and attempt count, as PHP's does. + Self::query("UPDATE workflow_tasks SET status='cancelled',lease_expires_at=NULL,last_error=NULL WHERE id=$1 AND workflow_run_id=$2 AND status IN ('ready','leased')") + .bind(&live).bind(run_id).execute(&mut **tx).await?; + attempt_snapshot = json!({"id":attempt_id,"activity_execution_id":activity_id,"task_id":live, + "attempt_number":DB::number(&attempt,"attempt_number")?,"status":"cancelled", + "lease_owner":DB::optional_string(&attempt,"lease_owner")?, + "started_at":DB::optional_instant(&attempt,"started_at")?,"closed_at":DB::optional_instant(&attempt,"closed_at")?}); + task_id = Some(live); + } + let cancelled = Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(&activity_id).fetch_one(&mut **tx).await?; + let mut payload = Self::activity_payload(&cancelled)?; + payload["workflow_command_id"] = json!(request_id); + payload["activity_attempt_id"] = json!(attempt_id); + payload["attempt_number"] = json!(DB::number(&cancelled, "attempt_count")?); + payload["cancelled_at"] = json!(DB::optional_instant(&cancelled, "closed_at")?); + payload["activity_attempt"] = attempt_snapshot; + if let Some(task_id) = &task_id { + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + payload["task"] = Self::task_snapshot(&task)?; + } + // Recorded with the request's id, as the delivery itself is. + Self::append_by_command_id(tx, run_id, "ActivityCancelled", payload, task_id.as_deref(), request_id).await?; + tx.summary_changed(run_id); + Ok(()) + } + async fn parallel_timer_members( tx: &mut Transaction<'_, DB>, run_id: &str, @@ -869,9 +934,9 @@ where break; } } - let activities = Self::scalar("SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$1 AND status IN ('pending','running')") - .bind(run_id).fetch_one(&mut *tx).await?; - if unsupported_child || activities != 0 { + // Open activities no longer block a request: the workflow delivers it + // to each activity it awaits (`call_kind: activity`), as PHP's does. + if unsupported_child { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, "cooperative_cancellation_propagation_not_available", diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index a75eadc46..8c0b755d7 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -1401,6 +1401,7 @@ impl Runtime { delegate!(heartbeat_activity(task_id: &str, body: Value, protocol: &str) -> Result); delegate!(activity_status(task_id: &str, body: Value, protocol: &str) -> Result); delegate!(renew_activity_lease(task_id: &str, body: Value) -> Result); + delegate!(acknowledge_activity_cancellation(task_id: &str, body: Value, protocol: &str) -> Result); pub(crate) async fn fail_activity(&self, task_id: &str, body: Value) -> Result { let (failure, blob) = super::activity_failures::prepare_failure(&body, self.signal_codec.clone()).await?; diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 3cc195f41..9b78390aa 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -270,6 +270,10 @@ pub fn router(runtime: Runtime) -> Router { "/api/worker/activity-tasks/{task_id}/renew-lease", post(renew_activity_lease), ) + .route( + "/api/worker/activity-tasks/{task_id}/acknowledge-cancellation", + post(acknowledge_activity_cancellation), + ) .route( "/api/worker/activity-tasks/{task_id}/fail", post(fail_activity), @@ -751,8 +755,8 @@ fn worker_route(rest: &str) -> Option { | ["workflow-tasks" | "activity-tasks", "poll"] | ["query-tasks", "poll"] | ["query-tasks", _, "complete" | "fail" | "status"] - | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat"] - | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease"] + | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat" | "deliver-cancellation"] + | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease" | "acknowledge-cancellation"] ) .then_some(Allowed::Any) } @@ -2032,6 +2036,18 @@ async fn renew_activity_lease( runtime.renew_activity_lease(&task_id, body).await.map(Json) } +async fn acknowledge_activity_cancellation( + Extension(runtime): Extension, + Path(task_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result> { + runtime + .acknowledge_activity_cancellation(&task_id, body, request_protocol(&headers)?) + .await + .map(Json) +} + async fn activity_status( Extension(runtime): Extension, Path(task_id): Path, diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index a38f42362..b88db1da6 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -24,6 +24,7 @@ macro_rules! store_methods { } mod active_summary; +mod activity_cancellation_ack; mod activity_completion_uploads; mod activity_deadlines; mod activity_failures; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 0b36704f1..a0d92f5f2 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1663,7 +1663,10 @@ where self.cancellation_lease_expiry(&run)? }; let leased_at = now(); - if kind == "workflow" { + // PHP binds the immutable claim on every workflow and activity + // claim (`PollRequestLeaseBinding::bindClaimedTask`); cooperative + // delivery and the activity's stop acknowledgement read it back. + if matches!(kind, "workflow" | "activity") { task_payload[super::cooperative::CLAIM_KEY] = json!({"registration_id":DB::number(&worker,"id")?, "worker_id":worker_id,"namespace":self.namespace.as_ref(),"protocol_version":protocol_version, "capabilities":DB::optional_document_row(&worker,"capabilities")?.unwrap_or(json!([])), @@ -2848,6 +2851,9 @@ where receipt["attempt_status"] = json!(DB::string(&attempt, "status")?); receipt["task_status"] = json!(DB::string(&task, "status")?); } + // Every worker answer carries PHP's protocol fields. + receipt["protocol_version"] = json!(super::WORKER_PROTOCOL_VERSION); + receipt["server_capabilities"] = capabilities(); return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response: receipt, @@ -3995,7 +4001,7 @@ pub(crate) fn capabilities() -> Value { "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, - "activity_lease_renewal": true}); + "activity_lease_renewal": true, "activity_cancellation_acknowledgement": true}); value["runtime_external_payload_transport"] = super::payload_discovery::transport_manifest(); value["supported_workflow_task_commands"] .as_array_mut() diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 5685a92be..e3079316a 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -349,8 +349,9 @@ where "last_claim_failed_at", "repair_available_at", ] { + // PHP's snapshot times are UTC with microseconds. if let Some(value) = DB::optional_instant(task, field)? { - snapshot[field] = json!(value); + snapshot[field] = json!(super::query_authority::wire(value)); } } for field in ["lease_owner", "compatibility", "last_dispatch_error", "last_claim_error"] { diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs index eb15e8d94..5568db624 100644 --- a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs +++ b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs @@ -1,8 +1,10 @@ import assert from 'node:assert/strict'; const MICROS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; -const ULID = /^[0-9a-z]{26}$/; -const REQUEST = /^[0-9A-Z]{26}$/; +// Identities are opaque ULIDs; their case differs between servers, as the +// other contracts allow. +const ULID = /^[0-9A-Za-z]{26}$/; +const REQUEST = ULID; const ACKS = ['before_request', 'before_delivery', 'foreign_owner', 'unknown_attempt', 'unknown_task', 'task_mismatch', 'other_namespace', 'protocol_1_19', 'protocol_missing', 'protocol_1_21', 'missing_fields', 'non_string_fields', 'long_request_id', 'wrong_request_id', 'acknowledged', 'duplicate', 'legacy_ack', 'immediate_ack', From 252ffa49aada4640950b29fb59cde8274bf877c0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:21:42 +0000 Subject: [PATCH 18/23] Native tests and polls follow PHP's worker answers; retried-attempt acknowledgement test - The workflow and activity polls report lease_expires_at with microseconds, as PHP does; the status route already does. - Native's stale outcome receipts now carry the worker protocol fields, so the execution tests that pinned the older receipt expect PHP's. - A native execution test requires that a retried activity's cooperative cancellation cancels the live attempt's task, keeps the earlier task completed, fences the earlier attempt and acknowledges the live one once. The shared fixture accepts published 2.11.1's defect (Workflow #923) here; native is held to the correct outcome. Full native SQLite suite: 257 library and 457 execution tests pass. The acknowledgement, lease renewal, one-activity and cancel-leased- activity fixtures pass against native locally. --- .github/workflows/server-parity.yml | 4 +- rust/src/runtime/store.rs | 2 +- rust/tests/execution.rs | 16 +++- .../execution/activity_cancellation_ack.rs | 75 +++++++++++++++++++ rust/tests/execution/activity_deadlines.rs | 13 +++- rust/tests/execution/worker_listing.rs | 14 +++- 6 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 rust/tests/execution/activity_cancellation_ack.rs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 9960470e7..22fadc1ee 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -35,6 +35,7 @@ on: - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityActivityCancellationAckContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs - tests/Unit/ServerParitySystemMetricsContractTest.mjs - tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs @@ -91,6 +92,7 @@ on: - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityActivityCancellationAckContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs - tests/Unit/ServerParitySystemMetricsContractTest.mjs - tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs @@ -1529,7 +1531,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index a0d92f5f2..537ec3b9a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1687,7 +1687,7 @@ where let mut claim = json!({"task_id": task_id, "workflow_id": DB::string(&run,"workflow_instance_id")?, "run_id": run_id, "workflow_instance_id": DB::string(&run,"workflow_instance_id")?, "workflow_run_id": run_id, "workflow_type": DB::string(&run,"workflow_type")?, "namespace": self.namespace.as_ref(), "payload_codec": "avro", - "queue": queue, "task_queue": queue, "lease_owner": worker_id, "lease_expires_at": expires}); + "queue": queue, "task_queue": queue, "lease_owner": worker_id, "lease_expires_at": super::cooperative::timestamp(expires)}); if let Some(activity) = activity { let activity_id = DB::string(&activity, "id")?; let policy = diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 96f2db7e4..3458260a6 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -287,6 +287,9 @@ mod external_payloads; #[path = "execution/trace_carriers.rs"] mod trace_carriers; +#[path = "execution/activity_cancellation_ack.rs"] +mod activity_cancellation_ack; + enum TestDatabase { Sqlite(tempfile::TempDir), Postgres { @@ -4533,10 +4536,12 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o conflicting["failure"]["message"] = json!("different"); let refused = request(&app_b, "POST", &fail_path, conflicting).await; assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + let mut receipt = refused.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( - refused.1, + receipt, json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null,"protocol_version":"1.20"}) ); let old_complete_path = format!( "/api/worker/activity-tasks/{}/complete", @@ -4582,11 +4587,14 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o // stale receipt PHP returns to that attempt's owner. let refused = request(&app_b, "POST", &old_complete_path, old_complete.clone()).await; assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + let mut receipt = refused.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( - refused.1, + receipt, json!({"task_id":first["task_id"],"activity_attempt_id":first["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"worker","activity_status":"running","attempt_status":"failed","task_status":"completed"}) + "lease_owner":"worker","activity_status":"running","attempt_status":"failed","task_status":"completed", + "protocol_version":"1.20"}) ); assert_eq!( request(&app_b, "POST", &old_complete_path, old_complete) diff --git a/rust/tests/execution/activity_cancellation_ack.rs b/rust/tests/execution/activity_cancellation_ack.rs new file mode 100644 index 000000000..a03d4ae7b --- /dev/null +++ b/rust/tests/execution/activity_cancellation_ack.rs @@ -0,0 +1,75 @@ +//! An activity worker's stop acknowledgement after cooperative delivery. +//! Native cancels a retried activity's live attempt and task, so its worker +//! can acknowledge the stop; published Server 2.11.1 delivers to the +//! execution's first task instead (Workflow #923), which the shared fixture +//! records as a documented reference boundary. +use super::*; + +async fn acknowledge(app: &Router, task: &Value, attempt: &Value, owner: &Value, request_id: &Value) -> (StatusCode, Value) { + request_protocol(app, "POST", + &format!("/api/worker/activity-tasks/{}/acknowledge-cancellation", task.as_str().unwrap()), + json!({"activity_attempt_id":attempt,"lease_owner":owner,"request_id":request_id}), "1.20").await +} + +/// Whether a task row has this status. +async fn task_has_status(database: &TestDatabase, task: &Value, status: &str) -> bool { + let sql = format!("SELECT COUNT(*) FROM workflow_tasks WHERE id='{}' AND status='{status}'", task.as_str().unwrap()); + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await == 1 +} + +#[tokio::test] +async fn a_retried_activity_acknowledges_on_its_live_task() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let registered = request_protocol(&app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["stop-activity"], + "capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, "retried-ack").await; + let task = cooperative_poll(&app, "workflow").await; + let scheduled = finish_task(&app, &task, json!([{"type":"schedule_activity","activity_type":"stop-activity", + "arguments":envelope(Payload::Array(vec![])),"retry_policy":{"max_attempts":2,"backoff_seconds":[0]}, + "start_to_close_timeout":300}])).await; + assert_eq!(scheduled.0, StatusCode::OK, "{}", scheduled.1); + let first = cooperative_poll(&app, "activity").await; + let failed = request_protocol(&app, "POST", + &format!("/api/worker/activity-tasks/{}/fail", first["task_id"].as_str().unwrap()), + json!({"activity_attempt_id":first["activity_attempt_id"],"lease_owner":first["lease_owner"], + "failure":{"message":"retry me","type":"parity.retryable","non_retryable":false}}), "1.20").await; + assert_eq!(failed.0, StatusCode::OK, "{}", failed.1); + let second = cooperative_poll(&app, "activity").await; + assert_ne!(second["task_id"], first["task_id"], "the retry has its own task"); + let requested = request(&app, "POST", "/api/workflows/retried-ack/request-cancellation", + json!({"reason":"stop the retry","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + let request_id = requested.1["cancellation_request"]["request_id"].clone(); + let resumed = cooperative_poll(&app, "workflow").await; + let delivered = request_protocol(&app, "POST", + &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", resumed["task_id"].as_str().unwrap()), + json!({"lease_owner":resumed["lease_owner"],"workflow_task_attempt":resumed["workflow_task_attempt"], + "request_id":request_id,"sequence":1,"call_kind":"activity"}), "1.20").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone()), (StatusCode::OK, json!(true)), "{}", delivered.1); + + // The live task is cancelled and the earlier one keeps its outcome. + assert!(task_has_status(&database, &second["task_id"], "cancelled").await, "live task cancelled"); + assert!(task_has_status(&database, &first["task_id"], "completed").await, "earlier task keeps its outcome"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let cancelled = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ActivityCancelled").unwrap(); + assert_eq!(cancelled["payload"]["task"]["id"], second["task_id"], "{cancelled}"); + assert_eq!(cancelled["payload"]["activity_attempt"]["id"], second["activity_attempt_id"], "{cancelled}"); + + // The earlier attempt is fenced; the live attempt acknowledges once. + let stale = acknowledge(&app, &first["task_id"], &first["activity_attempt_id"], &first["lease_owner"], &request_id).await; + assert_eq!((stale.0, stale.1["reason"].clone()), + (StatusCode::CONFLICT, json!("activity_cancellation_acknowledgement_fence_mismatch")), "{}", stale.1); + let acknowledged = acknowledge(&app, &second["task_id"], &second["activity_attempt_id"], &second["lease_owner"], &request_id).await; + assert_eq!((acknowledged.0, acknowledged.1["acknowledged"].clone(), acknowledged.1["duplicate"].clone()), + (StatusCode::OK, json!(true), json!(false)), "{}", acknowledged.1); + let repeated = acknowledge(&app, &second["task_id"], &second["activity_attempt_id"], &second["lease_owner"], &request_id).await; + assert_eq!((repeated.0, repeated.1["duplicate"].clone(), repeated.1["history_event_id"].clone()), + (StatusCode::OK, json!(true), acknowledged.1["history_event_id"].clone()), "{}", repeated.1); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index ea2d4d3a9..148694cce 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -234,22 +234,27 @@ async fn every_authored_deadline_fences_outcomes_and_resolves_once() { json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], "result":envelope(Payload::Long(-1))})).await; assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + let mut receipt = late.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "{field}: worker protocol fields"); assert_eq!( - late.1, + receipt, json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, "lease_owner":activity["lease_owner"],"activity_status":"failed","attempt_status":"failed", - "task_status":"cancelled"}), + "task_status":"cancelled","protocol_version":"1.20"}), "{field}" ); let late = request(&app, "POST", &path("fail"), json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"], "failure":{"type":"RuntimeException","message":"late","code":0}})).await; assert_eq!(late.0, StatusCode::CONFLICT, "{field}: {}", late.1); + let mut receipt = late.1.clone(); + assert!(receipt.as_object_mut().unwrap().remove("server_capabilities").is_some(), "{field}: worker protocol fields"); assert_eq!( - late.1, + receipt, json!({"task_id":activity["task_id"],"activity_attempt_id":activity["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}), + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null, + "protocol_version":"1.20"}), "{field}" ); } diff --git a/rust/tests/execution/worker_listing.rs b/rust/tests/execution/worker_listing.rs index 034b47201..36aae24c2 100644 --- a/rust/tests/execution/worker_listing.rs +++ b/rust/tests/execution/worker_listing.rs @@ -332,11 +332,14 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { ); let (status, stale) = request(&app, "POST", &path, complete(&original)).await; assert_eq!(status, StatusCode::CONFLICT, "{stale}"); + let mut stale = stale; + assert!(stale.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( stale, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"replaced","activity_status":"running","attempt_status":"expired","task_status":"leased"}) + "lease_owner":"replaced","activity_status":"running","attempt_status":"expired","task_status":"leased", + "protocol_version":"1.20"}) ); let (status, done) = request(&app, "POST", &path, complete(&replacement)).await; assert_eq!(status, StatusCode::OK, "{done}"); @@ -345,11 +348,14 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { // result is still stale, as in PHP, not a conflicting duplicate. let (status, late) = request(&app, "POST", &path, complete(&original)).await; assert_eq!(status, StatusCode::CONFLICT, "{late}"); + let mut late = late; + assert!(late.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( late, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], "outcome":"completed","recorded":false,"reason":"stale_attempt","next_task_id":null, - "lease_owner":"replaced","activity_status":"completed","attempt_status":"expired","task_status":"completed"}) + "lease_owner":"replaced","activity_status":"completed","attempt_status":"expired","task_status":"completed", + "protocol_version":"1.20"}) ); let fail_path = format!( "/api/worker/activity-tasks/{}/fail", @@ -359,10 +365,12 @@ async fn a_replacement_registration_releases_the_activity_lease_at_once() { json!({"activity_attempt_id":original["activity_attempt_id"],"lease_owner":"replaced", "failure":{"message":"late","type":"RuntimeException"}})).await; assert_eq!(status, StatusCode::CONFLICT, "{late_failure}"); + let mut late_failure = late_failure; + assert!(late_failure.as_object_mut().unwrap().remove("server_capabilities").is_some(), "worker protocol fields"); assert_eq!( late_failure, json!({"task_id":original["task_id"],"activity_attempt_id":original["activity_attempt_id"], - "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null}) + "outcome":"failed","recorded":false,"reason":"stale_attempt","next_task_id":null,"protocol_version":"1.20"}) ); // The replacement's own repeated result keeps its idempotent receipt. let (status, again) = request(&app, "POST", &path, complete(&replacement)).await; From 6bde9459cf0a2d03d5e4c3be75dd97ca1d287deb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:51:33 +0000 Subject: [PATCH 19/23] Message streams are served: drop them from the unserved route list #709 serves the three message-stream routes, which the route coverage test reported as now served. Native serves 129 of 175 route methods at this tree; 46 remain. --- rust/tests/fixtures/unserved-php-routes.v2.11.1.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index 886219a91..5bd1bdd32 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -43,9 +43,6 @@ POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome POST /api/workflows/import/embedded-v2 POST /api/workflows/import/waterline-v1 -GET /api/workflows/{workflowId}/message-streams -GET /api/workflows/{workflowId}/message-streams/{streamName} -POST /api/workflows/{workflowId}/message-streams/{streamName}/messages GET /api/workflows/{workflowId}/nexus-operations GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close From bf914f97cd7bcaed6780f475ccde2b18144def7d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:26:25 +0000 Subject: [PATCH 20/23] Compare activity lease renewals as instants, not RFC3339 text heartbeat_renews_only_progress_deadline_and_cannot_resurrect_elapsed_authority compared two lease_expires_at strings. Native times are whole milliseconds and serialize without a fraction on a whole second, so a claim lease of ...:26Z sorted after a renewal of ...:26.045Z and the assertion failed about one run in a thousand (train #729, PostgreSQL 17 job 120852521096). The test now parses both and reports the values. --- rust/tests/execution/activity_deadlines.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index ea2d4d3a9..c67b5f724 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -483,9 +483,14 @@ async fn heartbeat_renews_only_progress_deadline_and_cannot_resurrect_elapsed_au assert_eq!(after.0, initial.0, "Start-to-close deadline remains fixed"); assert!(after.1 > initial.1, "Heartbeat deadline advances"); assert_eq!(after.2, initial.2, "Lifecycle deadline remains fixed"); + // Compare instants, not text: a lease on a whole second is written + // without a fraction ("...:26Z"), which sorts after "...:26.045Z". + let instant = |value: &Value| chrono::DateTime::parse_from_rfc3339(value.as_str().unwrap()).unwrap(); assert!( - renewed.1["lease_expires_at"].as_str().unwrap() - > activity["lease_expires_at"].as_str().unwrap() + instant(&renewed.1["lease_expires_at"]) > instant(&activity["lease_expires_at"]), + "{} renews {}", + renewed.1["lease_expires_at"], + activity["lease_expires_at"] ); let prefix = run_history(&app, &started["workflow_id"], &started["run_id"]).await; database From 4f34e89a3f6c587ed36fed49e38b600969f68e03 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:31:19 +0000 Subject: [PATCH 21/23] Bring the Rust port log's current state up to October 24 --- docs/rust-server-port.md | 166 ++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 90 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 48e0adc97..582784510 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -14,97 +14,83 @@ qualify a later tree. ## Current state -Updated October 23, 2026. - -- **Merged since October 20.** - - **Backup-first takeover** on SQLite, PostgreSQL and MySQL/MariaDB: - [#523](https://github.com/durable-workflow/server/pull/523), - [#529](https://github.com/durable-workflow/server/pull/529), - [#532](https://github.com/durable-workflow/server/pull/532), - [#533](https://github.com/durable-workflow/server/pull/533), - [#536](https://github.com/durable-workflow/server/pull/536), - [#538](https://github.com/durable-workflow/server/pull/538) and - [#539](https://github.com/durable-workflow/server/pull/539). This covers the - fingerprint, the layouts and WAL preparation. It also covers a PHP-accepted - signal and an expired PHP lease resumed under native, and the refusal of - unsupported features. - - **Multi-node and API parity.** - - [#549](https://github.com/durable-workflow/server/pull/549): the - multi-node kill and restart qualification. - - [#548](https://github.com/durable-workflow/server/pull/548): ownership - guard refusals. - - [#550](https://github.com/durable-workflow/server/pull/550) and - [#551](https://github.com/durable-workflow/server/pull/551): the search - attribute registry and namespace update. - - [#594](https://github.com/durable-workflow/server/pull/594): history - export. - - [#595](https://github.com/durable-workflow/server/pull/595) and - [#596](https://github.com/durable-workflow/server/pull/596): retention and - activity timeouts. - - **The October 21-22 trains.** - [#629](https://github.com/durable-workflow/server/pull/629), - [#638](https://github.com/durable-workflow/server/pull/638), - [#641](https://github.com/durable-workflow/server/pull/641), - [#649](https://github.com/durable-workflow/server/pull/649), - [#659](https://github.com/durable-workflow/server/pull/659) and - [#665](https://github.com/durable-workflow/server/pull/665) covered: - - input envelopes, run repair, legacy settings, activity deadlines, - runtime credentials and principals; - - query and update admission, signals rejected after a run closes, start - fingerprints and all-of barriers; - - cooperative waits, child cancellation and parent-close policies, - continue-as-new messages, history pages, declared signals and build IDs. - - **The reference moved to published Server 2.11.1** - ([#679](https://github.com/durable-workflow/server/pull/679)), with - activity lease renewal ([#680](https://github.com/durable-workflow/server/pull/680)) - and the service catalog reads ([#681](https://github.com/durable-workflow/server/pull/681)). - - **The October 23 trains.** - [#675](https://github.com/durable-workflow/server/pull/675) covered - definition routing, worker admission, poll waits and the schedule probe; - [#678](https://github.com/durable-workflow/server/pull/678), namespace - delete; [#676](https://github.com/durable-workflow/server/pull/676), - PHP's health read during its own repair. - [#696](https://github.com/durable-workflow/server/pull/696) covered - command batches, named-namespace commands, completion checks, external - inputs, the operator surface (system metrics, Prometheus, repair and - payload cleanup passes), activity leases, deployments, priority and - fairness, standalone activities, archive and redrive. - - **GHSA-6q48-6c7w-9gw2 trust remediation.** - [#671](https://github.com/durable-workflow/server/pull/671) moves the Rust - locks to the fixed webpki roots. - [#672](https://github.com/durable-workflow/server/pull/672) pins the image's - TLS toolchains and checks every trust store. -- **Open.** - - [#707](https://github.com/durable-workflow/server/pull/707): cluster info - payload discovery, takeover storage refusal and the operator dashboard by - workflow type. - - [#708](https://github.com/durable-workflow/server/pull/708): accept - published PHP's operator metrics read across its redispatch steps. - - [#673](https://github.com/durable-workflow/server/pull/673): pip, Composer - and Cargo trust in CI and conformance, in Security review. - [#677](https://github.com/durable-workflow/server/issues/677) owns the - remaining clients, with an explicit non-completion boundary. +Updated October 24, 2026. + +- **Merged October 24.** Train + [#729](https://github.com/durable-workflow/server/pull/729) (merge + `61219609b`): + - MySQL statement closes are sent with the next command + ([#717](https://github.com/durable-workflow/server/pull/717)), ending the + MySQL-family stalls of [#714](https://github.com/durable-workflow/server/issues/714); + - policy maps that are not JSON objects are refused, like PHP #705, and a + namespace quota map that is not UTF-8 is refused under either name + ([#719](https://github.com/durable-workflow/server/pull/719)); + - worker history shows consumed signal sequences, hydrating only each page's + own signals, plus full task snapshots + ([#710](https://github.com/durable-workflow/server/pull/710)); + - a newer push cancels a superseded Shared Server fixtures run + ([#724](https://github.com/durable-workflow/server/pull/724)). +- **Merged October 20-23.** These are summarized in the checkpoint history: + - backup-first takeover on every database; + - multi-node kill and restart; + - ownership guards, search attributes, namespace update, history export, + retention and activity timeouts; + - the October 21-23 trains; + - the move to the published 2.11.1 reference; + - the GHSA-6q48-6c7w-9gw2 fixes. + + Each train's PR holds its evidence. +- **In review or qualifying, in merge order.** + 1. CI speed and the merge queue: + - [#741](https://github.com/durable-workflow/server/pull/741) runs the + SDK direction profiles in parallel lanes, about 14 minutes off every + database job, and fixes a timestamp-comparison test flake; + - [#742](https://github.com/durable-workflow/server/pull/742) adds the + merge-queue prerequisites (`merge_group` triggers, one fixtures + result, and fixture cells skipped when a change cannot affect them). + 2. This train: operator metrics read across PHP's redispatch steps + ([#708](https://github.com/durable-workflow/server/pull/708)), + query-authority clients that wait for the acknowledgement + ([#720](https://github.com/durable-workflow/server/pull/720)), and the + machine-checked route list below. + 3. Condition waits without a key, task snapshots, the SQLite patch cohort and + message streams (#713, #718, #722, #709). + 4. Update validation with PHP's poll request ID contract + ([#712](https://github.com/durable-workflow/server/pull/712)), then the + held-poll registration fence + ([#739](https://github.com/durable-workflow/server/pull/739), #734) and + worker sessions + ([#740](https://github.com/durable-workflow/server/pull/740), #736), + stacked in that order. + 5. Activity cancellation acknowledgement, and pip, Composer and Cargo trust + in CI ([#673](https://github.com/durable-workflow/server/pull/673)). - **Shared Server fixtures.** Native, embedded and published PHP 2.11.1 are - compared on SQLite, MySQL 8.0, MariaDB 10.11 and PostgreSQL 17. The known - failures come from timing races inside the published PHP reference, not - from native: - - `activity-deadline-receipts` (fixed by #696); - - `operator-metrics` at `waiting_after_redispatch` (#708). - - Each was a recorded contract decision; no expectation was edited to make - native pass. -- **Parity surface.** Native serves 125 of published PHP 2.11.1's 174 API - routes. That count compares PHP's `route:list` table - (`rust/src/runtime/php-routes.v2.11.1.json`) with the native router. The - 49 missing routes are listed below, by family. -- **Next action.** - 1. Land #707 and #708. - 2. Port message streams, which all three published SDKs declare from - worker protocol 1.15. Native refuses a worker that asks for them, - with `message_streams_unavailable`. - 3. Port the remaining worker protocol families below. - - Packaging the native binary into the server image remains an open gate. + compared on SQLite, MySQL 8.0, MariaDB 10.11 and PostgreSQL 17. Where the + published reference itself misbehaves, an explicit boundary separates it + from native, and the PHP owner has an issue for each case: + - `operator-metrics` across PHP's redispatch window (#708); + - the published-SDK patch cohort's SQLite poll refusal + ([#721](https://github.com/durable-workflow/server/issues/721), #722); + - update validation under SQLite write locks + ([#726](https://github.com/durable-workflow/server/issues/726), fixed in + PHP by #730); + - `worker-sdk-persistent-pressure`'s shutdown window + ([#728](https://github.com/durable-workflow/server/issues/728)); + - a retried activity's cooperative cancellation + ([Workflow #923](https://github.com/durable-workflow/workflow/issues/923)); + - a reacquired worker session keeping its old failure reason + ([#737](https://github.com/durable-workflow/server/issues/737)). + + No expectation is edited to make native pass. +- **Parity surface.** Native serves 126 of the 175 API route methods in + published 2.11.1's route table. An execution test sends each one through + the native router, and the ones still answering + `501 rust_capability_not_implemented` must match + [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). + The work in review serves 12 more: message streams (3), update validation + (3), worker sessions (5) and activity cancellation acknowledgement (1). +- **Next action.** Land the items above in order, keep porting the unserved + routes fixture first, and package the native binary into the server image. ## API surface still missing From efc35b461ac6f15aab2372f69ab62802c0d68bec Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:33:26 +0000 Subject: [PATCH 22/23] Rust #325: activity heartbeats answer times with microseconds, as status reads do --- rust/src/runtime/activity_heartbeats.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 1460fa8db..24412be8b 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -311,7 +311,9 @@ where Ok( json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], "cancel_requested":false,"can_continue":true,"reason":null,"run_closed_reason":null,"run_closed_at":null, - "heartbeat_recorded":true,"lease_expires_at":expires,"last_heartbeat_at":at,"worker_session":null, + // PHP formats each time in UTC with microseconds, as status reads do. + "heartbeat_recorded":true,"lease_expires_at":super::cooperative::timestamp(expires), + "last_heartbeat_at":super::cooperative::timestamp(at),"worker_session":null, "protocol_version":protocol,"server_capabilities":super::store::capabilities()}), ) } From 9c450ca5621b1e551d06c1396e41f02a25d9c737 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:32:55 +0000 Subject: [PATCH 23/23] Rust #325: refuse cancellation deliveries at the wrong or a resolved activity or timer Security's R2 finding on #745: native recorded a cooperative cancellation delivery for call_kind activity even when no open activity was at the sequence. That pinned the wrong delivery sequence, and the duplicate path then refused the correct delivery. Native now runs published Workflow 2.11.1's validateCallBoundary for activity and timer deliveries before cancelling anything: - Every typed step at the sequence must have the call's shape, and an authored sequence must have typed history. Either fault is cancellation_delivery_shape_mismatch. A wait's timeout timer belongs to the wait, not to a timer call. - A call that resolved before the request is cancellation_delivery_not_eligible, including a member of a cancelled selection operation. One that resolved after the request still takes the delivery as it is. - The next sequence may be a call not yet authored. - Each validation refusal answers 409 with the bridge's undelivered body in the worker envelope, as Server 2.11.1 does, and records nothing. The activity acknowledgement probe adds a run whose activity completed before the request, with a timer at sequence 2. The contract pins both refusals, the unchanged history, the still-undelivered request and the timer's delivery. Published 2.11.1 passes it 4/4 and the pr1 recording replaces the expected observation. Native passes 3/3. The previous head fails it with a 200 delivery at the completed activity. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/cooperative.rs | 197 ++++++++++++++---- rust/src/runtime/http.rs | 4 +- .../execution/activity_cancellation_ack.rs | 121 +++++++++++ rust/tests/execution/cooperative_waits.rs | 84 ++++++++ .../activity-cancellation-ack-contract.mjs | 34 +++ .../activity-cancellation-ack-probe.php | 31 ++- ...cellation-acknowledgement-observation.json | 2 +- ...ityActivityCancellationAckContractTest.mjs | 22 ++ 8 files changed, 453 insertions(+), 42 deletions(-) diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index 366dc0cdb..b3aa73d38 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -105,6 +105,75 @@ pub(super) async fn encode_command(value: Value, semaphore: Arc) -> R /// PHP's default `command_batch_size` structural limit. const PARALLEL_SPAN_LIMIT: i64 = 1000; +/// The reasons `CooperativeCancellationDelivery::validate` returns. +/// `DefaultWorkflowTaskBridge::deliverCancellation` answers each with its +/// full delivery body under 409: the task and run, no delivery. +const VALIDATION_REASONS: [&str; 8] = [ + "cancellation_request_mismatch", + "cancellation_request_history_missing", + "cancellation_delivery_mismatch", + "cancellation_delivery_history_missing", + "cancellation_delivery_sequence_mismatch", + "cancellation_delivery_shape_mismatch", + "cancellation_delivery_not_eligible", + "cancellation_delivery_requires_scope", +]; + +fn validation_refusal(task_id: &str, run_id: &str, error: RuntimeError) -> RuntimeError { + match error { + RuntimeError::Refused { status: StatusCode::CONFLICT, reason } if VALIDATION_REASONS.contains(&reason) => { + RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"delivered":false,"task_id":task_id,"workflow_run_id":run_id,"request_id":null, + "sequence":null,"call_kind":null,"sequence_span":null,"operation_sequence":null, + "operation_sequence_span":null,"reason":reason}), + } + } + other => other, + } +} + +/// `WorkflowStepHistory::WORKFLOW_STEP_EVENT_TYPES` without activity starts +/// and heartbeats: each follows its own `ActivityScheduled` at the same +/// sequence, so neither changes a boundary's shape, and heartbeats are +/// unbounded. +const STEP_EVENTS: [&str; 30] = [ + "CancellationScopeOpened", "ActivityScheduled", "ActivityRetryScheduled", "ActivityCompleted", "ActivityFailed", + "ActivityCancelled", "ActivityTimedOut", "ChildWorkflowScheduled", "ChildRunStarted", "ChildRunCompleted", + "ChildRunFailed", "ChildRunCancelled", "ChildRunTerminated", "ServiceCallStarted", "ServiceCallCompleted", + "ServiceCallFailed", "ServiceCallCancelled", "WorkflowContinuedAsNew", "ConditionWaitOpened", + "ConditionWaitSatisfied", "ConditionWaitTimedOut", "SignalWaitOpened", "SignalApplied", "MemoUpserted", + "SearchAttributesUpserted", "SideEffectRecorded", "VersionMarkerRecorded", "TimerScheduled", "TimerFired", + "TimerCancelled", +]; + +/// `CooperativeCancellationDelivery::RESOLUTION_TYPES`. +const RESOLUTION_EVENTS: [&str; 18] = [ + "ActivityCompleted", "ActivityFailed", "ActivityCancelled", "ActivityTimedOut", "TimerFired", "TimerCancelled", + "ConditionWaitSatisfied", "ConditionWaitTimedOut", "SignalApplied", "ChildRunCompleted", "ChildRunFailed", + "ChildRunCancelled", "ChildRunTerminated", "ServiceCallCompleted", "ServiceCallFailed", "ServiceCallCancelled", + "SelectionResolved", "SelectionOperationCancelled", +]; + +/// `WorkflowStepHistory::eventMatchesShape` for an activity or timer call. +fn matches_call_shape(call_kind: &str, event_type: &str, payload: &Value) -> bool { + match call_kind { + "activity" => { + matches!(event_type, "ActivityScheduled" | "ActivityRetryScheduled" | "ActivityCompleted" + | "ActivityFailed" | "ActivityCancelled" | "ActivityTimedOut") + // `isLocalActivityEvent` + && payload["execution_mode"] != "local" + && payload["local_activity"] != true + } + "timer" => { + matches!(event_type, "TimerScheduled" | "TimerFired" | "TimerCancelled") + // `isPureTimerEvent`: a wait's timeout timer belongs to the wait. + && !matches!(payload["timer_kind"].as_str(), Some("condition_timeout" | "signal_timeout")) + } + _ => false, + } +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -237,13 +306,16 @@ where } let run_id = DB::string(&task, "workflow_run_id")?; let run = Self::active_run(&mut tx, &run_id).await?; + // `CooperativeCancellationDelivery::validate` refusals carry the + // bridge's delivery body. + let refusal = |error: RuntimeError| validation_refusal(task_id, &run_id, error); if DB::optional_string(&run, "cancellation_request_command_id")?.as_deref() != Some(request_id) { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_request_mismatch", - )); + ))); } if DB::optional_instant(&run, "cancellation_deadline_at")? .is_none_or(|deadline| deadline <= now()) @@ -258,16 +330,16 @@ where "operation_sequence_span":1,"reason":null}); let requested=Self::query("SELECT sequence,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND workflow_command_id=$2 AND event_type='CooperativeCancellationRequested' ORDER BY sequence LIMIT 1") .bind(&run_id).bind(request_id).fetch_optional(&mut *tx).await? - .ok_or_else(|| refuse(StatusCode::CONFLICT,"cancellation_request_history_missing"))?; + .ok_or_else(|| refusal(refuse(StatusCode::CONFLICT,"cancellation_request_history_missing")))?; let request_sequence = DB::number(&requested, "sequence")?; let requested = DB::document_row(&requested, "payload")?; if requested["workflow_command_id"] != request_id || requested["cancellation"]["request_id"] != request_id { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_request_history_missing", - )); + ))); } if let Some(delivery)=Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='CooperativeCancellationDelivered' ORDER BY sequence LIMIT 1") .bind(&run_id).fetch_optional(&mut *tx).await? { @@ -275,7 +347,7 @@ where if payload["workflow_command_id"]!=request_id || payload["sequence"]!=sequence || payload["call_kind"]!=call_kind || payload["sequence_span"].as_i64().unwrap_or(1)!=span || payload["cancellation"]!=requested["cancellation"] || DB::optional_number(&run,"cancellation_delivery_sequence")?!=Some(sequence) { - return Err(refuse(StatusCode::CONFLICT,"cancellation_delivery_mismatch")); + return Err(refusal(refuse(StatusCode::CONFLICT,"cancellation_delivery_mismatch"))); } Self::query("UPDATE workflow_tasks SET lease_expires_at=$1 WHERE id=$2") .bind(DB::bind_time(self.cancellation_lease_expiry(&run)?)) @@ -284,26 +356,26 @@ where return Ok(response); } if DB::optional_number(&run, "cancellation_delivery_sequence")?.is_some() { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_delivery_history_missing", - )); + ))); } let next = Self::authored_sequence(&mut tx, &run_id) .await? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; if sequence > next { - return Err(refuse( + return Err(refusal(refuse( StatusCode::CONFLICT, "cancellation_delivery_sequence_mismatch", - )); + ))); } if call_kind == "parallel" { // `parallelEligibility`; a boundary at the next sequence has no // authored member yet. if sequence < next { - for timer in Self::parallel_timer_members(&mut tx, &run_id, sequence, span, request_sequence).await? { + for timer in Self::parallel_timer_members(&mut tx, &run_id, sequence, span, request_sequence).await.map_err(refusal)? { if DB::string(&timer, "status")? == "pending" { Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } @@ -317,7 +389,8 @@ where sequence, request_sequence, ) - .await?; + .await + .map_err(refusal)?; // `ChildCancellation::prepare`: a waiting policy holds the // delivery and releases the claim until the child closes. if boundary.policy != "abandon" @@ -337,16 +410,23 @@ where } } } else if call_kind == "activity" { - // `cancelOpenWaitAtSequence` with the default `try_cancel` - // policy: an open activity at the sequence is cancelled under the - // request. Native records no explicit activity cancellation - // policy, so it has no waiting or abandoning delivery. + // `validateCallBoundary`, then `cancelOpenWaitAtSequence` with + // the default `try_cancel` policy: an open activity at the + // sequence is cancelled under the request, and one that resolved + // after the request takes the delivery as it is. Native records + // no explicit activity cancellation policy, so it has no waiting + // or abandoning delivery. + Self::validate_call_boundary(&mut tx, &run_id, sequence, call_kind, sequence < next, request_sequence) + .await + .map_err(refusal)?; if sequence < next { Self::deliver_activity_cancellation(&mut tx, &run_id, request_id, sequence, now()).await?; } } else if matches!(call_kind, "condition" | "signal") { if sequence < next { - Self::validate_wait_cancellation_boundary(&mut tx, &run_id, sequence, call_kind, request_sequence).await?; + Self::validate_wait_cancellation_boundary(&mut tx, &run_id, sequence, call_kind, request_sequence) + .await + .map_err(refusal)?; } // `cancelOpenWaitAtSequence`: a timed wait's pending timeout timer is // cancelled, as at a timer boundary. @@ -357,28 +437,18 @@ where Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } } else { - let timer = Self::query( - "SELECT * FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence=$2", - ) - .bind(&run_id) - .bind(sequence) - .fetch_optional(&mut *tx) - .await?; - if sequence < next && timer.is_none() { - return Err(refuse( - StatusCode::CONFLICT, - "cancellation_delivery_shape_mismatch", - )); - } - if let Some(timer) = timer { - if DB::string(&timer, "status")? != "pending" { - return Err(refuse( - StatusCode::CONFLICT, - "cancellation_delivery_not_eligible", - )); - } - Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()) - .await?; + // A timer: `validateCallBoundary`, then `cancelOpenWaitAtSequence` + // cancels it while pending. A wait's timeout timer is not a timer + // boundary, and a timer that fired after the request takes the + // delivery as it is. + Self::validate_call_boundary(&mut tx, &run_id, sequence, call_kind, sequence < next, request_sequence) + .await + .map_err(refusal)?; + if let Some(timer) = Self::query("SELECT * FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence=$2") + .bind(&run_id).bind(sequence).fetch_optional(&mut *tx).await? + && DB::string(&timer, "status")? == "pending" + { + Self::cancel_root_timer(&mut tx, &run, request_id, &timer, Some(task_id), now()).await?; } } let at = now(); @@ -541,6 +611,55 @@ where Ok(members) } + /// `validateCallBoundary` for an activity or timer call. Every typed step + /// at the sequence must have the call's shape + /// (`WorkflowStepHistory::assertCompatible`), and an authored sequence + /// must have typed history (`assertTypedHistoryRecorded`); either fault is + /// a shape mismatch. Then a call that resolved before the request is not + /// eligible (`resolvedBeforeRequest`). The next sequence may be a call not + /// yet authored. + async fn validate_call_boundary( + tx: &mut Transaction<'_, DB>, + run_id: &str, + sequence: i64, + call_kind: &str, + authored: bool, + request_sequence: i64, + ) -> Result<()> { + let events = Self::query("SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('CancellationScopeOpened','ActivityScheduled','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityCancelled','ActivityTimedOut','ChildWorkflowScheduled','ChildRunStarted','ChildRunCompleted','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','ServiceCallStarted','ServiceCallCompleted','ServiceCallFailed','ServiceCallCancelled','WorkflowContinuedAsNew','ConditionWaitOpened','ConditionWaitSatisfied','ConditionWaitTimedOut','SignalWaitOpened','SignalApplied','MemoUpserted','SearchAttributesUpserted','SideEffectRecorded','VersionMarkerRecorded','TimerScheduled','TimerFired','TimerCancelled','SelectionResolved','SelectionOperationCancelled') ORDER BY sequence LIMIT 10001") + .bind(run_id) + .fetch_all(&mut **tx) + .await?; + if events.len() > 10000 { + return Err(refuse(StatusCode::SERVICE_UNAVAILABLE, "cancellation_inventory_exceeds_budget")); + } + let (mut typed, mut conflicting, mut resolved) = (false, false, false); + for event in &events { + let kind = DB::string(event, "event_type")?; + let payload = DB::document_row(event, "payload")?; + let at = payload["sequence"].as_i64() == Some(sequence); + if at && STEP_EVENTS.contains(&kind.as_str()) { + typed = true; + conflicting |= !matches_call_shape(call_kind, &kind, &payload); + } + // A cancelled selection operation resolves each member it spans. + let covered = at + || (kind == "SelectionOperationCancelled" + && payload["member_base_sequence"].as_i64().zip(payload["member_size"].as_i64()) + .is_some_and(|(base, size)| base >= 1 && base <= sequence && size > sequence - base)); + resolved |= covered + && RESOLUTION_EVENTS.contains(&kind.as_str()) + && DB::number(event, "sequence")? < request_sequence; + } + if conflicting || (authored && !typed) { + return Err(refuse(StatusCode::CONFLICT, "cancellation_delivery_shape_mismatch")); + } + if resolved { + return Err(refuse(StatusCode::CONFLICT, "cancellation_delivery_not_eligible")); + } + Ok(()) + } + /// `validateCallBoundary` for a condition or signal wait: the authored /// step at the sequence must be that wait (a timed wait's timeout timer /// belongs to it), and it must not have resolved before the request. diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 6033f9c58..bc5734823 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -1906,8 +1906,10 @@ async fn deliver_cancellation( runtime .deliver_cancellation(&task_id, body, request_protocol(&headers)?) .await - // `WorkerProtocol::json`: every answer carries the worker envelope. + // `WorkerProtocol::json`: every answer carries the worker envelope, + // the bridge's refused deliveries included. .map(|body| Json(worker_response(body))) + .map_err(worker_refusal) } async fn complete_workflow( Extension(runtime): Extension, diff --git a/rust/tests/execution/activity_cancellation_ack.rs b/rust/tests/execution/activity_cancellation_ack.rs index a03d4ae7b..6ce97c951 100644 --- a/rust/tests/execution/activity_cancellation_ack.rs +++ b/rust/tests/execution/activity_cancellation_ack.rs @@ -73,3 +73,124 @@ async fn a_retried_activity_acknowledges_on_its_live_task() { runtime.close().await; database.remove().await; } + +/// The bridge's refused delivery (`DefaultWorkflowTaskBridge::deliverCancellation` +/// over `CooperativeCancellationDelivery::validate`), as published PHP answers +/// it under 409, without the capability map. +pub(super) fn refused_delivery(task: &Value, run_id: &Value, reason: &str) -> Value { + json!({"delivered":false,"task_id":task["task_id"],"workflow_run_id":run_id,"request_id":null,"sequence":null, + "call_kind":null,"sequence_span":null,"operation_sequence":null,"operation_sequence_span":null, + "reason":reason,"protocol_version":"1.20"}) +} + +pub(super) async fn deliver_at(app: &Router, task: &Value, request_id: &Value, sequence: i64, kind: &str) -> (StatusCode, Value) { + let (status, mut body) = request_protocol(app, "POST", + &format!("/api/worker/workflow-tasks/{}/deliver-cancellation", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "request_id":request_id,"sequence":sequence,"call_kind":kind}), "1.20").await; + let capabilities = body.as_object_mut().and_then(|body| body.remove("server_capabilities")); + assert!(capabilities.is_some_and(|map| map.is_object()), "every delivery answer carries the worker envelope: {body}"); + (status, body) +} + +/// A run whose activity at sequence 1 completed before the request and whose +/// timer at sequence 2 is still pending, with the request's workflow task +/// claimed. Returns the run, the claimed task and the request id. +async fn resolved_activity_then_timer(app: &Router, workflow: &str) -> (Value, Value, Value) { + let registered = request_protocol(app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["stop-activity"], + "capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(app, workflow).await; + let task = cooperative_poll(app, "workflow").await; + let scheduled = finish_task(app, &task, json!([{"type":"schedule_activity","activity_type":"stop-activity", + "arguments":envelope(Payload::Array(vec![])),"retry_policy":{"max_attempts":1,"backoff_seconds":[]}, + "start_to_close_timeout":300}])).await; + assert_eq!(scheduled.0, StatusCode::OK, "{}", scheduled.1); + let activity = cooperative_poll(app, "activity").await; + let completed = request_protocol(app, "POST", + &format!("/api/worker/activity-tasks/{}/complete", activity["task_id"].as_str().unwrap()), + json!({"activity_attempt_id":activity["activity_attempt_id"],"lease_owner":activity["lease_owner"], + "result":envelope(Payload::String("done".into()))}), "1.20").await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + let resumed = cooperative_poll(app, "workflow").await; + let timer = finish_task(app, &resumed, json!([{"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(timer.0, StatusCode::OK, "{}", timer.1); + let requested = request(app, "POST", &format!("/api/workflows/{workflow}/request-cancellation"), + json!({"reason":"stop after the activity","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + let claimed = cooperative_poll(app, "workflow").await; + assert_eq!(claimed["cancellation_request"]["request_id"], requested.1["cancellation_request"]["request_id"], "{claimed}"); + (started, claimed, requested.1["cancellation_request"]["request_id"].clone()) +} + +// Published Workflow 2.11.1 validates an authored activity boundary against +// its typed history before cancelling anything +// (CooperativeCancellationDelivery::validateCallBoundary): another call's +// sequence is a shape mismatch, and an activity that resolved before the +// request is not eligible. Neither refusal records a delivery, and the +// request can still be delivered at its real boundary. +#[tokio::test] +async fn an_activity_delivery_is_refused_at_a_resolved_activity_or_another_calls_sequence() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (started, task, request_id) = resolved_activity_then_timer(&app, "resolved-activity").await; + let run_id = &started["run_id"]; + let before = run_history(&app, &started["workflow_id"], run_id).await; + let undelivered = "SELECT COUNT(*) FROM workflow_runs WHERE cancellation_request_command_id IS NOT NULL AND cancellation_delivery_sequence IS NULL AND cancellation_delivered_at IS NULL"; + assert_eq!(task_retry::rows(&database, undelivered).await, 1); + + let resolved = deliver_at(&app, &task, &request_id, 1, "activity").await; + assert_eq!(resolved, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_not_eligible"))); + let timer_sequence = deliver_at(&app, &task, &request_id, 2, "activity").await; + assert_eq!(timer_sequence, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_shape_mismatch"))); + let activity_sequence = deliver_at(&app, &task, &request_id, 1, "timer").await; + assert_eq!(activity_sequence, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_shape_mismatch"))); + let beyond = deliver_at(&app, &task, &request_id, 4, "activity").await; + assert_eq!(beyond, (StatusCode::CONFLICT, refused_delivery(&task, run_id, "cancellation_delivery_sequence_mismatch"))); + + // Nothing was delivered or cancelled. + assert_eq!(run_history(&app, &started["workflow_id"], run_id).await, before); + assert_eq!(task_retry::rows(&database, undelivered).await, 1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_executions WHERE status='completed'").await, 1); + let duplicate = request(&app, "POST", "/api/workflows/resolved-activity/request-cancellation", json!({})).await; + assert_eq!((duplicate.0, duplicate.1["duplicate"].clone()), (StatusCode::OK, json!(true)), "{}", duplicate.1); + assert_eq!((duplicate.1["cancellation_request"]["delivery_sequence"].clone(), + duplicate.1["cancellation_request"]["delivered_at"].clone()), (Value::Null, Value::Null), "{}", duplicate.1); + + // The pending timer still takes the delivery. + let delivered = deliver_at(&app, &task, &request_id, 2, "timer").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone()), + (StatusCode::OK, json!(true), json!(2)), "{}", delivered.1); + let history = run_history(&app, &started["workflow_id"], run_id).await; + let events = history.as_array().unwrap(); + assert_eq!(&events[..before.as_array().unwrap().len()], before.as_array().unwrap().as_slice()); + let appended: Vec<_> = events[before.as_array().unwrap().len()..].iter().map(|event| event["event_type"].clone()).collect(); + assert_eq!(appended, [json!("TimerCancelled"), json!("CooperativeCancellationDelivered")], "{history}"); + runtime.close().await; + database.remove().await; +} + +// The next sequence may be a call the workflow has not authored yet, so it +// takes an activity delivery without typed history, as published PHP does. +#[tokio::test] +async fn an_activity_delivery_at_the_next_sequence_needs_no_authored_activity() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (started, task, request_id) = resolved_activity_then_timer(&app, "next-activity").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let delivered = deliver_at(&app, &task, &request_id, 3, "activity").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone(), delivered.1["call_kind"].clone()), + (StatusCode::OK, json!(true), json!(3), json!("activity")), "{}", delivered.1); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let appended: Vec<_> = history.as_array().unwrap()[before.as_array().unwrap().len()..].iter() + .map(|event| event["event_type"].clone()).collect(); + assert_eq!(appended, [json!("CooperativeCancellationDelivered")], "{history}"); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/cooperative_waits.rs b/rust/tests/execution/cooperative_waits.rs index 54ce76dab..832bcf919 100644 --- a/rust/tests/execution/cooperative_waits.rs +++ b/rust/tests/execution/cooperative_waits.rs @@ -68,3 +68,87 @@ async fn a_run_parked_in_a_signal_wait_takes_cancellation_there_and_cleans_up() assert_eq!(cancelled["principal"], Value::Null, "{cancelled}"); assert_eq!(cancelled["payload"]["cancellation_cleanup"]["outcome"], "completed", "{cancelled}"); } + +async fn cooperative_worker(app: &Router) { + let registered = request_protocol(app, "POST", "/api/worker/register", + json!({"worker_id":"cleanup-worker","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["cooperative_cancellation"]}), "1.20").await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); +} + +async fn request_root(app: &Router, workflow: &str) -> Value { + let requested = request(app, "POST", &format!("/api/workflows/{workflow}/request-cancellation"), + json!({"reason":"timer boundary","cleanup_timeout_seconds":60})).await; + assert_eq!(requested.0, StatusCode::ACCEPTED, "{}", requested.1); + requested.1["cancellation_request"]["request_id"].clone() +} + +// A timed condition wait's timeout timer belongs to the wait +// (WorkflowStepHistory::isPureTimerEvent), so published PHP refuses a timer +// delivery there as a shape mismatch without touching the timer; the +// condition delivery then cancels it. +#[tokio::test] +async fn a_timer_delivery_at_a_timed_waits_timeout_is_a_shape_mismatch() { + use super::activity_cancellation_ack::{deliver_at, refused_delivery}; + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + cooperative_worker(&app).await; + let run = start(&app, "timed-wait-timer").await; + let task = cooperative_poll(&app, "workflow").await; + let opened = finish_task(&app, &task, json!([{"type":"open_condition_wait","timeout_seconds":600}])).await; + assert_eq!(opened.0, StatusCode::OK, "{}", opened.1); + let request_id = request_root(&app, "timed-wait-timer").await; + let root = cooperative_poll(&app, "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let wrong = deliver_at(&app, &root, &request_id, 1, "timer").await; + assert_eq!(wrong, (StatusCode::CONFLICT, refused_delivery(&root, &run["run_id"], "cancellation_delivery_shape_mismatch"))); + assert_eq!(run_history(&app, &run["workflow_id"], &run["run_id"]).await, before); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 1); + let delivered = deliver_at(&app, &root, &request_id, 1, "condition").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone()), (StatusCode::OK, json!(true)), "{}", delivered.1); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers WHERE status='pending'").await, 0); + runtime.close().await; + database.remove().await; +} + +// Only a timer that resolved before the request is not eligible +// (resolvedBeforeRequest). One that fired after it still takes the delivery, +// as it is: published PHP's cancelOpenWaitAtSequence leaves a fired timer +// alone and records the delivery. +#[tokio::test] +async fn a_timer_that_fired_after_the_request_still_takes_the_delivery() { + use super::activity_cancellation_ack::deliver_at; + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + cooperative_worker(&app).await; + let run = start(&app, "fired-after-request").await; + let task = cooperative_poll(&app, "workflow").await; + let timer = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await; + assert_eq!(timer.0, StatusCode::OK, "{}", timer.1); + let request_id = request_root(&app, "fired-after-request").await; + let root = cooperative_poll(&app, "workflow").await; + assert_eq!(root["cancellation_request"]["request_id"], request_id, "{root}"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(20); + let fired = loop { + let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + if history.as_array().unwrap().iter().any(|event| event["event_type"] == "TimerFired") { + break history; + } + assert!(tokio::time::Instant::now() < deadline, "timer never fired: {history}"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + }; + let events = fired.as_array().unwrap(); + let position = |kind: &str| events.iter().position(|event| event["event_type"] == kind).unwrap(); + assert!(position("CooperativeCancellationRequested") < position("TimerFired"), "{fired}"); + let delivered = deliver_at(&app, &root, &request_id, 1, "timer").await; + assert_eq!((delivered.0, delivered.1["delivered"].clone(), delivered.1["sequence"].clone()), + (StatusCode::OK, json!(true), json!(1)), "{}", delivered.1); + let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let kinds: Vec<_> = history.as_array().unwrap().iter().map(|event| event["event_type"].clone()).collect(); + assert!(!kinds.contains(&json!("TimerCancelled")), "{history}"); + assert_eq!(kinds.last(), Some(&json!("CooperativeCancellationDelivered")), "{history}"); + runtime.close().await; + database.remove().await; +} diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs index 5568db624..c40e89805 100644 --- a/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs +++ b/scripts/conformance/server-parity/activity-cancellation-ack-contract.mjs @@ -12,6 +12,8 @@ const ACKS = ['before_request', 'before_delivery', 'foreign_owner', 'unknown_att 'terminated_ack']; const MAIN_EVENTS = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'CooperativeCancellationRequested', 'ActivityCancelled', 'CooperativeCancellationDelivered']; +const BOUNDARY_EVENTS = ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', 'ActivityCompleted', + 'TimerScheduled', 'CooperativeCancellationRequested']; // Milliseconds since the epoch of a UTC timestamp with any fraction length. function at(value, label) { @@ -243,5 +245,37 @@ export function checkActivityCancellationAcknowledgement(fixture, observation, w 'a terminated run still acknowledges'); equal(state.terminated_history, [...MAIN_EVENTS, 'TerminateRequested', 'WorkflowTerminated', 'ActivityCancellationAcknowledged'], 'the late acknowledgement follows the terminal event'); + + // An activity delivery is checked against the run's typed history before + // anything is cancelled (CooperativeCancellationDelivery::validateCallBoundary). + // The activity at sequence 1 completed before the request and sequence 2 is + // a timer. Both refusals carry the bridge's undelivered body and record + // nothing, and the pending timer then takes the delivery. + const boundary = state.boundary_claim; + equal(boundary.workflow_id, `${workflowId}-boundary`, 'boundary run'); + equal([state.boundary_complete, state.boundary_timer, state.boundary_request], [200, 200, 202], + 'activity completed, timer started, then cancellation requested'); + assert.match(state.boundary_task_id ?? '', ULID, 'boundary cancellation workflow task'); + const undelivered = reason => ({status: 409, acknowledgement_capability: true, response: {delivered: false, + task_id: state.boundary_task_id, workflow_run_id: boundary.run_id, request_id: null, sequence: null, call_kind: null, + sequence_span: null, operation_sequence: null, operation_sequence_span: null, reason, protocol_version: '1.20'}}); + equal(state.boundary_resolved_activity, undelivered('cancellation_delivery_not_eligible'), + 'an activity that resolved before the request is not eligible'); + equal(state.boundary_timer_as_activity, undelivered('cancellation_delivery_shape_mismatch'), + 'an activity delivery at a timer is a shape mismatch'); + equal(state.boundary_history_before.map(event => event.event_type), BOUNDARY_EVENTS, 'boundary history before delivery'); + equal(state.boundary_history_after_refusals, state.boundary_history_before, 'boundary refusals record nothing'); + const pendingRequest = state.boundary_pending.response?.cancellation_request; + equal([state.boundary_pending.status, state.boundary_pending.response?.duplicate, pendingRequest?.delivery_sequence, + pendingRequest?.delivered_at], [200, true, null, null], 'the request is still undelivered after the refusals'); + assert.match(pendingRequest.request_id ?? '', REQUEST, 'boundary request id'); + equal(state.boundary_delivery, {status: 200, acknowledgement_capability: true, response: {delivered: true, + task_id: state.boundary_task_id, workflow_run_id: boundary.run_id, request_id: pendingRequest.request_id, sequence: 2, + call_kind: 'timer', sequence_span: 1, operation_sequence: null, operation_sequence_span: 1, reason: null, + protocol_version: '1.20'}}, 'the pending timer takes the delivery'); + const boundaryHistory = state.boundary_history_after_delivery; + equal(boundaryHistory.slice(0, -2), state.boundary_history_before, 'delivery only appends'); + equal(boundaryHistory.slice(-2).map(event => event.event_type), ['TimerCancelled', 'CooperativeCancellationDelivered'], + 'the timer is cancelled under the delivery'); return projection; } diff --git a/scripts/conformance/server-parity/activity-cancellation-ack-probe.php b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php index 46af4b3c3..5cd84d983 100644 --- a/scripts/conformance/server-parity/activity-cancellation-ack-probe.php +++ b/scripts/conformance/server-parity/activity-cancellation-ack-probe.php @@ -14,7 +14,8 @@ // acknowledge the stop from the activity worker. The main run is also refused // for each ownership, protocol and validation fault, acknowledged, read back // and acknowledged again. Separate runs cover a claim polled on protocol 1.19, -// an immediate cancel, a retried attempt and a run terminated after delivery. +// an immediate cancel, a retried attempt, a run terminated after delivery and +// activity deliveries refused by the run's typed history. // Responses are recorded raw, without the protocol's capability map beyond // its acknowledgement flag. function finishHttpActivityCancellationAcknowledgement(Client $admin, array $fixture, string $workflowId, string $url): array @@ -189,5 +190,33 @@ function finishHttpActivityCancellationAcknowledgement(Client $admin, array $fix $state['terminated_ack'] = $ack($terminated, $own($terminated) + ['request_id' => $terminatedDelivered['request_id']]); $state['terminated_history'] = array_column($history($terminated), 'event_type'); + // An activity delivery is checked against typed history first: the + // activity completed before the request, then a timer was started at + // sequence 2. Delivering to the activity, or as an activity at the + // timer, is refused and records nothing; the timer then takes it. + $boundary = $state['boundary_claim'] = $claim('boundary'); + $boundaryPoll = ['worker_id' => $boundary['worker'], 'task_queue' => $boundary['queue'], 'timeout_seconds' => 0]; + $state['boundary_complete'] = $raw('POST', '/api/worker/activity-tasks/'.$boundary['task_id'].'/complete', + $own($boundary) + ['result' => $done])['status']; + $resumed = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $boundaryPoll), 200, 'boundary resume')['task']; + $state['boundary_timer'] = $raw('POST', '/api/worker/workflow-tasks/'.$resumed['task_id'].'/complete', [ + 'lease_owner' => $resumed['lease_owner'], 'workflow_task_attempt' => $resumed['workflow_task_attempt'], + 'commands' => [['type' => 'start_timer', 'delay_seconds' => 3600]]])['status']; + $state['boundary_request'] = $request($boundary)['status']; + $cancelling = $ok($raw('POST', '/api/worker/workflow-tasks/poll', $boundaryPoll), 200, 'boundary cancellation claim')['task']; + $state['boundary_task_id'] = $cancelling['task_id']; + $deliverAt = static fn (int $sequence, string $kind): array => $raw('POST', + '/api/worker/workflow-tasks/'.$cancelling['task_id'].'/deliver-cancellation', [ + 'lease_owner' => $cancelling['lease_owner'], 'workflow_task_attempt' => $cancelling['workflow_task_attempt'], + 'request_id' => $cancelling['cancellation_request']['request_id'] ?? null, 'sequence' => $sequence, + 'call_kind' => $kind]); + $state['boundary_history_before'] = $history($boundary); + $state['boundary_resolved_activity'] = $deliverAt(1, 'activity'); + $state['boundary_timer_as_activity'] = $deliverAt(2, 'activity'); + $state['boundary_history_after_refusals'] = $history($boundary); + $state['boundary_pending'] = $request($boundary); + $state['boundary_delivery'] = $deliverAt(2, 'timer'); + $state['boundary_history_after_delivery'] = $history($boundary); + return $state; } diff --git a/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json index bb922669c..9a060b57a 100644 --- a/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json +++ b/tests/Fixtures/ServerParityExpected/activity-cancellation-acknowledgement-observation.json @@ -1 +1 @@ -{"applicable": true, "namespace": "pr1-activity-cancellation-acknowledgement-cack", "discovery": {"activity_cancellation_acknowledgement": true, "cooperative_cancellation": true}, "main": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5pce4j95qsm4nh4kj68q5v9", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "worker": "pr1-activity-cancellation-acknowledgement-main-worker", "workflow_task_id": "01m5pce4jm5yjq52g1gh13zq17", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker"}, "before_request": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "not-yet-requested", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "request": {"status": 202, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5pce4j95qsm4nh4kj68q5v9", "accepted": true, "duplicate": false, "run_status": "waiting", "cancellation_request": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "reason": null}, "acknowledgement_capability": "absent"}, "before_delivery": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_recorded", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_delivery": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": true, "cancel_requested": false, "reason": null, "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T07:24:08.525226Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "running", "attempt_status": "running", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "poll_cancellation_request": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce56p4sjtyxzrv4e86j4d", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "heartbeat_after_delivery": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancel_requested": true, "can_continue": false, "reason": "attempt_closed", "run_closed_reason": null, "run_closed_at": null, "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_ack": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_before_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}], "foreign_owner": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity attempt lease is owned by another worker.", "reason": "lease_owner_mismatch", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_attempt": {"status": 404, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "no-such-attempt", "error": "Activity attempt not found.", "reason": "attempt_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_task": {"status": 404, "response": {"task_id": "no-such-task", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "task_mismatch": {"status": 409, "response": {"task_id": "01m5pce4jm5yjq52g1gh13zq17", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "other_namespace": {"status": 404, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_19": {"status": 422, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "minimum_protocol_version": "1.20", "unavailable": ["worker_protocol"], "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_missing": {"status": 400, "response": {"error": "Missing worker protocol version header.", "reason": "missing_protocol_version", "supported_version": "1.20", "requested_version": null, "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_21": {"status": 400, "response": {"error": "Unsupported worker protocol version.", "reason": "unsupported_protocol_version", "supported_version": "1.20", "requested_version": "1.21", "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "missing_fields": {"status": 422, "response": {"message": "The activity attempt id field is required. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "non_string_fields": {"status": 422, "response": {"message": "The activity attempt id field must be a string. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "long_request_id": {"status": 422, "response": {"message": "The request id field must not be greater than 255 characters.", "reason": "validation_failed", "errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "validation_errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "wrong_request_id": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "other-request", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}], "acknowledged": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_after_ack": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:08.525107Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "callback_state": "stopped", "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "acknowledged_at": "2026-10-24T07:19:10.136717Z", "received_after_deadline": false}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "duplicate": {"status": 200, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "acknowledged": true, "duplicate": true, "reason": null, "history_event_id": "01m5pce6htfpz4bjzf62naz6cf", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "completed_after_ack": {"status": 409, "response": {"task_id": "01m5pce4tp2gefawyg6dsb7jjd", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "outcome": "completed", "recorded": false, "reason": "stale_attempt", "next_task_id": null, "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:08.109895Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:08.113603Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "workflow_command_id": "01m5pce4jbwkapjjexqwbh5f30", "command": {"id": "01m5pce4jbwkapjjexqwbh5f30", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:08.106292Z", "applied_at": "2026-10-24T07:19:08.106296Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:08.371961Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4jm5yjq52g1gh13zq17", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.116242Z", "leased_at": "2026-10-24T07:19:08.222187Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:20:08.222141Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.131429Z", "last_dispatched_at": "2026-10-24T07:19:08.131429Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:08.529635Z", "principal": null, "payload": {"activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:24:08.525226Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:08.752568Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "command": {"id": "01M5PCE56BXRCEJMCPDGEE9VYC", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "resolved_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDowOC43NDc1ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFNTZCWFJDRUpNQ1BER0VFOVZZQx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U1NkJYUkNFSk1DUERHRUU5VllDMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVwY2U0ajk1cXNtNG5oNGtqNjhxNXY5InBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMDc6MTk6MDguNzQ3NTg5WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDA3OjIwOjA4Ljc0NzU4OVoObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVBDRTU2QlhSQ0VKTUNQREdFRTlWWUMod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlNGo5NXFzbTRuaDRrajY4cTV2OQAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5pce4j95qsm4nh4kj68q5v9/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:60bb4b291c48c20b0e2d6b5427756b62d603374b3927b36ad13dbd851c7e6700", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:08.747589Z", "applied_at": "2026-10-24T07:19:08.747589Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:09.184618Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T07:19:09.179606Z", "activity": {"id": "01m5pce4tk6q91g0ebyhxynvkq", "idempotency_key": "01m5pce4tk6q91g0ebyhxynvkq", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T07:19:08.371089Z", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "task_id": "01m5pce4tp2gefawyg6dsb7jjd", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T07:19:08.525107Z", "closed_at": "2026-10-24T07:19:09.179606Z"}, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:09.188357Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:08.747589Z", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "lineage": [{"request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5pce4j95qsm4nh4kj68q5v9"}]}, "task": {"id": "01m5pce56p4sjtyxzrv4e86j4d", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:08.747589Z", "leased_at": "2026-10-24T07:19:09.033536Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T07:19:19.033478Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:08.792234Z", "last_dispatched_at": "2026-10-24T07:19:08.792234Z"}}}, {"sequence": 8, "event_type": "ActivityCancellationAcknowledged", "timestamp": "2026-10-24T07:19:10.137480Z", "principal": null, "payload": {"sequence": 1, "activity_execution_id": "01m5pce4tk6q91g0ebyhxynvkq", "activity_attempt_id": "01M5PCE4ZD27PQ6J8E7P7YRYXT", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancellation_history_event_id": "01m5pce5m14rjbp50nfztp36pj", "request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "root_request_id": "01M5PCE56BXRCEJMCPDGEE9VYC", "cleanup_deadline_at": "2026-10-24T07:20:08.747589Z", "callback_state": "stopped", "evidence_source": "activity_worker", "acknowledged_at": "2026-10-24T07:19:10.136717Z", "received_after_deadline": false, "task": {"id": "01m5pce4tp2gefawyg6dsb7jjd", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:08.374316Z", "leased_at": "2026-10-24T07:19:08.525107Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}], "legacy_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-legacy", "run_id": "01m5pce71geceqgfrvb8mgzghc", "queue": "pr1-activity-cancellation-acknowledgement-legacy-queue", "worker": "pr1-activity-cancellation-acknowledgement-legacy-worker", "workflow_task_id": "01m5pce71tq57627pyxztvv08b", "task_id": "01m5pce79fbyqm23p77xmhqz4n", "activity_attempt_id": "01M5PCE7F9QG7DMWQKX1NQA5T6", "activity_execution_id": "01m5pce79b8nyyk1n89129btqt", "lease_owner": "pr1-activity-cancellation-acknowledgement-legacy-worker"}, "legacy_request": 202, "legacy_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce7mp7x45m0zy0mvynpxe", "workflow_run_id": "01m5pce71geceqgfrvb8mgzghc", "request_id": "01M5PCE7MFYSZ2T8SGGPZ5EEGB", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "legacy_ack": {"status": 409, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "required_capability": "cooperative_cancellation", "minimum_protocol_version": "1.20", "worker_id": "pr1-activity-cancellation-acknowledgement-legacy-worker", "unavailable": ["worker_claim_capability"], "remediation": "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5pce84zt7623a5j72s1g4p3", "queue": "pr1-activity-cancellation-acknowledgement-immediate-queue", "worker": "pr1-activity-cancellation-acknowledgement-immediate-worker", "workflow_task_id": "01m5pce8594pxswss94983p374", "task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "activity_execution_id": "01m5pce8cgkm09vw8y9grnzhgk", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker"}, "immediate_cancel": {"status": 200, "response": {"outcome": "cancelled", "workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5pce84zt7623a5j72s1g4p3", "requested_run_id": null, "resolved_run_id": "01m5pce84zt7623a5j72s1g4p3", "command_id": "01m5pce8men6x37hxx01ke5gc7", "command_sequence": 2, "target_scope": "instance", "workflow_type": "parity.v1.cancel_ack", "command_status": "accepted", "command_source": "control_plane", "reason": "acknowledgement fixture", "rejection_reason": null, "validation_errors": []}, "acknowledgement_capability": "absent"}, "immediate_ack": {"status": 409, "response": {"task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "request_id": "01m5pce8men6x37hxx01ke5gc7", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_status": {"status": 200, "response": {"task_id": "01m5pce8cm6h82fwzx9wvbhqj8", "activity_attempt_id": "01M5PCE8GHRT1S34B2XT4PC8YZ", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "can_continue": false, "cancel_requested": true, "reason": "run_cancelled", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "cancelled", "run_closed_reason": "cancelled", "run_closed_at": "2026-10-24T07:19:12.280883Z", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T07:24:12.145804Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first": {"workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5pce91n33d9v1cfb6ncvh2z", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5pce91z14aca5gjm9p1dj6k", "task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker"}, "stale_fail": {"status": 200, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "outcome": "failed", "recorded": true, "reason": null, "next_task_id": "01m5pce9m78fmfh88ce3jqee3e", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5pce91n33d9v1cfb6ncvh2z", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5pce91z14aca5gjm9p1dj6k", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g"}, "stale_request": 202, "stale_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pce9wk5q46atc83g1pd2sr", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_own_task": {"status": 409, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_acknowledgement_fence_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_new_task": {"status": 409, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt_old_task": {"status": 409, "response": {"task_id": "01m5pce9ae7cyss9tzhnbch75b", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt": {"status": 409, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_fenced", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_status": {"status": 200, "response": {"task_id": "01m5pce9m78fmfh88ce3jqee3e", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T07:24:13.443669Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T07:24:13.443566Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "cancellation_history_event_id": "01m5pcea5t4kt69sf5ns6661jy", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_history": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T07:19:12.696936Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5pce91q61dexcb48296z3nx", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5pce91q61dexcb48296z3nx", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:12.693941Z", "applied_at": "2026-10-24T07:19:12.693946Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T07:19:12.700510Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "workflow_command_id": "01m5pce91q61dexcb48296z3nx", "command": {"id": "01m5pce91q61dexcb48296z3nx", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T07:19:12.693941Z", "applied_at": "2026-10-24T07:19:12.693946Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T07:19:12.971905Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce91z14aca5gjm9p1dj6k", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:12.703025Z", "leased_at": "2026-10-24T07:19:12.817249Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:20:12.817202Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:12.718128Z", "last_dispatched_at": "2026-10-24T07:19:12.718128Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:13.160244Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.155017Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:24:13.155152Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 5, "event_type": "ActivityRetryScheduled", "timestamp": "2026-10-24T07:19:13.290467Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "retry_task_id": "01m5pce9m78fmfh88ce3jqee3e", "retry_of_task_id": "01m5pce9ae7cyss9tzhnbch75b", "retry_available_at": "2026-10-24T07:19:13.284568Z", "retry_backoff_seconds": 0, "retry_after_attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "retry_after_attempt": 1, "max_attempts": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "exception_type": "parity.retryable", "activity_attempt": {"id": "01M5PCE9G3T6M6C7Y3758CNQHA", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "task_id": "01m5pce9ae7cyss9tzhnbch75b", "status": "failed", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "closed_at": "2026-10-24T07:19:13.284791Z"}, "exception_class": "RuntimeException", "message": "retry me", "code": 0, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9G3T6M6C7Y3758CNQHA", "status": "pending", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.155017Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "completed", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 6, "event_type": "ActivityStarted", "timestamp": "2026-10-24T07:19:13.448417Z", "principal": null, "payload": {"activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "status": "running", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.443566Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "task": {"id": "01m5pce9m78fmfh88ce3jqee3e", "type": "activity", "status": "leased", "available_at": "2026-10-24T07:19:13.284568Z", "leased_at": "2026-10-24T07:19:13.443566Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:24:13.443669Z", "attempt_count": 2, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:13.322057Z", "last_dispatched_at": "2026-10-24T07:19:13.322057Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T07:19:13.551795Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:13.548489Z", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "lineage": [{"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z"}]}, "command": {"id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "resolved_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDoxMy41NDg0ODlaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UENFOVdDWjJHUEgwUVNNMjZEUUJGSx5yb290X3JlcXVlc3RfaWQKNDAxTTVQQ0U5V0NaMkdQSDBRU00yNkRRQkZLMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXnByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LXN0YWxlKHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cGNlOTFuMzNkOXYxY2ZiNm5jdmgyeiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDA3OjE5OjEzLjU0ODQ4OVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQwNzoyMDoxMy41NDg0ODlaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVQQ0U5V0NaMkdQSDBRU00yNkRRQkZLKHdvcmtmbG93X2luc3RhbmNlX2lkCl5wcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1zdGFsZR53b3JrZmxvd19ydW5faWQKNDAxbTVwY2U5MW4zM2Q5djFjZmI2bmN2aDJ6AAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-stale/runs/01m5pce91n33d9v1cfb6ncvh2z/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:209709fdff9285a483ef965eae0aa5b23a07df94ec273ea1fe3ad52822cd4701", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T07:19:13.548489Z", "applied_at": "2026-10-24T07:19:13.548489Z"}}}, {"sequence": 8, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T07:19:13.849540Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "activity_attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "cancelled_at": "2026-10-24T07:19:13.843676Z", "activity": {"id": "01m5pce9abzx3wnxp2fvtpqb6g", "idempotency_key": "01m5pce9abzx3wnxp2fvtpqb6g", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5PCE9S344EQGV6BHH016F2Y", "status": "cancelled", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T07:19:12.971049Z", "started_at": "2026-10-24T07:19:13.443566Z", "closed_at": "2026-10-24T07:19:13.843676Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "activity_attempt": {"id": "01M5PCE9S344EQGV6BHH016F2Y", "activity_execution_id": "01m5pce9abzx3wnxp2fvtpqb6g", "task_id": "01m5pce9m78fmfh88ce3jqee3e", "attempt_number": 2, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "started_at": "2026-10-24T07:19:13.443566Z", "closed_at": "2026-10-24T07:19:13.843676Z"}, "task": {"id": "01m5pce9ae7cyss9tzhnbch75b", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T07:19:12.974257Z", "leased_at": "2026-10-24T07:19:13.155017Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T07:19:13.853476Z", "principal": null, "payload": {"workflow_command_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T07:19:13.548489Z", "cleanup_deadline_at": "2026-10-24T07:20:13.548489Z", "lineage": [{"request_id": "01M5PCE9WCZ2GPH0QSM26DQBFK", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5pce91n33d9v1cfb6ncvh2z"}]}, "task": {"id": "01m5pce9wk5q46atc83g1pd2sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T07:19:13.548489Z", "leased_at": "2026-10-24T07:19:13.723070Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T07:19:23.723013Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T07:19:13.577280Z", "last_dispatched_at": "2026-10-24T07:19:13.577280Z"}}}], "terminated_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-terminated", "run_id": "01m5pceask5cer356gz7m630am", "queue": "pr1-activity-cancellation-acknowledgement-terminated-queue", "worker": "pr1-activity-cancellation-acknowledgement-terminated-worker", "workflow_task_id": "01m5pceaswp3wq87dt3xy1ftkq", "task_id": "01m5pceb2yhq2n7mhkg1kmwryh", "activity_attempt_id": "01M5PCEB8D36PD1YJ91T2PEK4B", "activity_execution_id": "01m5pceb2tga6mgbnp7k8yfca8", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker"}, "terminated_request": 202, "terminated_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5pcebcnbv13ep26mtw7y73c", "workflow_run_id": "01m5pceask5cer356gz7m630am", "request_id": "01M5PCEBCD5HM18A6YAE8QFA5E", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminate": 200, "terminated_ack": {"status": 200, "response": {"task_id": "01m5pceb2yhq2n7mhkg1kmwryh", "activity_attempt_id": "01M5PCEB8D36PD1YJ91T2PEK4B", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker", "request_id": "01M5PCEBCD5HM18A6YAE8QFA5E", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5pcebxrbwvbd8n7t8ezc0ks", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminated_history": ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityStarted", "CooperativeCancellationRequested", "ActivityCancelled", "CooperativeCancellationDelivered", "TerminateRequested", "WorkflowTerminated", "ActivityCancellationAcknowledged"]} \ No newline at end of file +{"applicable": true, "namespace": "pr1-activity-cancellation-acknowledgement-cack", "discovery": {"activity_cancellation_acknowledgement": true, "cooperative_cancellation": true}, "main": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "worker": "pr1-activity-cancellation-acknowledgement-main-worker", "workflow_task_id": "01m5qw91semrqyrf9d8btymy60", "task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker"}, "before_request": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "not-yet-requested", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "request": {"status": 202, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-main", "run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "accepted": true, "duplicate": false, "run_status": "waiting", "cancellation_request": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "reason": null}, "acknowledgement_capability": "absent"}, "before_delivery": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_recorded", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_delivery": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": true, "cancel_requested": false, "reason": null, "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T21:20:13.496866Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "running", "attempt_status": "running", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "poll_cancellation_request": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}, "delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw92edr1d6qfp9wpvk18p8", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "heartbeat_after_delivery": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancel_requested": true, "can_continue": false, "reason": "attempt_closed", "run_closed_reason": null, "run_closed_at": null, "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_before_ack": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_before_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}], "foreign_owner": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity attempt lease is owned by another worker.", "reason": "lease_owner_mismatch", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_attempt": {"status": 404, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "no-such-attempt", "error": "Activity attempt not found.", "reason": "attempt_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "unknown_task": {"status": 404, "response": {"task_id": "no-such-task", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "task_mismatch": {"status": 409, "response": {"task_id": "01m5qw91semrqyrf9d8btymy60", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "other_namespace": {"status": 404, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "error": "Activity task not found.", "reason": "task_not_found", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_19": {"status": 422, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "minimum_protocol_version": "1.20", "unavailable": ["worker_protocol"], "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_missing": {"status": 400, "response": {"error": "Missing worker protocol version header.", "reason": "missing_protocol_version", "supported_version": "1.20", "requested_version": null, "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "protocol_1_21": {"status": 400, "response": {"error": "Unsupported worker protocol version.", "reason": "unsupported_protocol_version", "supported_version": "1.20", "requested_version": "1.21", "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "missing_fields": {"status": 422, "response": {"message": "The activity attempt id field is required. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field is required."], "lease_owner": ["The lease owner field is required."], "request_id": ["The request id field is required."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "non_string_fields": {"status": 422, "response": {"message": "The activity attempt id field must be a string. (and 2 more errors)", "reason": "validation_failed", "errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "validation_errors": {"activity_attempt_id": ["The activity attempt id field must be a string."], "lease_owner": ["The lease owner field must be a string."], "request_id": ["The request id field must be a string."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "long_request_id": {"status": 422, "response": {"message": "The request id field must not be greater than 255 characters.", "reason": "validation_failed", "errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "validation_errors": {"request_id": ["The request id field must not be greater than 255 characters."]}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "wrong_request_id": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "other-request", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}], "acknowledged": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5qw9483fhm6p895f7baexdc", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "status_after_ack": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:13.496706Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "callback_state": "stopped", "history_event_id": "01m5qw9483fhm6p895f7baexdc", "acknowledged_at": "2026-10-24T21:15:15.585663Z", "received_after_deadline": false}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "duplicate": {"status": 200, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "acknowledged": true, "duplicate": true, "reason": null, "history_event_id": "01m5qw9483fhm6p895f7baexdc", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "completed_after_ack": {"status": 409, "response": {"task_id": "01m5qw922den79a15zyessm583", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "outcome": "completed", "recorded": false, "reason": "stale_attempt", "next_task_id": null, "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "history_after_ack": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:13.060924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:13.066371Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "workflow_command_id": "01m5qw91s1fswa97y4d5ye4mzg", "command": {"id": "01m5qw91s1fswa97y4d5ye4mzg", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:22a641e93cd78f0f3e63c39d489b05f93f27a98594e4a4ae63339aa5e73ad5c9", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:13.056053Z", "applied_at": "2026-10-24T21:15:13.056060Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:13.353604Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw91semrqyrf9d8btymy60", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.069961Z", "leased_at": "2026-10-24T21:15:13.201261Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:16:13.201169Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.076720Z", "last_dispatched_at": "2026-10-24T21:15:13.076720Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:13.502903Z", "principal": null, "payload": {"activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:20:13.496866Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 5, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:13.734742Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "command": {"id": "01M5QW92E0BK0GDWHT9G71MBNX", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "resolved_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoxMy43MjgwNTZaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5MkUwQkswR0RXSFQ5RzcxTUJOWB5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzkyRTBCSzBHRFdIVDlHNzFNQk5YMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ocm9vdF93b3JrZmxvd19ydW5faWQKNDAxbTVxdzkxcno2YjBzeHJyY3doejNxMW1jInBhcmVudF9yZXF1ZXN0X2lkAAxyZWFzb24KNGFja25vd2xlZGdlbWVudCBmaXh0dXJlIM67EnJlcXVlc3Rlcg4GCHR5cGUKFGF1dGg6dG9rZW4EaWQKGGxlZ2FjeS10b2tlbgpsYWJlbAoKQWRtaW4ADHNvdXJjZQoaY29udHJvbF9wbGFuZRhyZXF1ZXN0ZWRfYXQKNjIwMjYtMTAtMjRUMjE6MTU6MTMuNzI4MDU2WiZjbGVhbnVwX2RlYWRsaW5lX2F0CjYyMDI2LTEwLTI0VDIxOjE2OjEzLjcyODA1NloObGluZWFnZQwCDgYUcmVxdWVzdF9pZAo0MDFNNVFXOTJFMEJLMEdEV0hUOUc3MU1CTlgod29ya2Zsb3dfaW5zdGFuY2VfaWQKXHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LW1haW4ed29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5MXJ6NmIwc3hycmN3aHozcTFtYwAAAAA="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-main/runs/01m5qw91rz6b0sxrrcwhz3q1mc/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:6b4d6cab8528728fa1643859c9fb03337bfad407e63b12796c7eefbe9e589377", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:13.728056Z", "applied_at": "2026-10-24T21:15:13.728056Z"}}}, {"sequence": 6, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:14.220415Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "cancelled_at": "2026-10-24T21:15:14.213744Z", "activity": {"id": "01m5qw92282801x57927cvx33q", "idempotency_key": "01m5qw92282801x57927cvx33q", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "status": "cancelled", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "created_at": "2026-10-24T21:15:13.352133Z", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "activity_attempt": {"id": "01M5QW926RR5Y2GRX31DRP80P5", "activity_execution_id": "01m5qw92282801x57927cvx33q", "task_id": "01m5qw922den79a15zyessm583", "attempt_number": 1, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "started_at": "2026-10-24T21:15:13.496706Z", "closed_at": "2026-10-24T21:15:14.213744Z"}, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}, {"sequence": 7, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:14.225246Z", "principal": null, "payload": {"workflow_command_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "root_workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:13.728056Z", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "lineage": [{"request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-main", "workflow_run_id": "01m5qw91rz6b0sxrrcwhz3q1mc"}]}, "task": {"id": "01m5qw92edr1d6qfp9wpvk18p8", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:13.728056Z", "leased_at": "2026-10-24T21:15:14.098410Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "lease_expires_at": "2026-10-24T21:15:24.098332Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:13.776290Z", "last_dispatched_at": "2026-10-24T21:15:13.776290Z"}}}, {"sequence": 8, "event_type": "ActivityCancellationAcknowledged", "timestamp": "2026-10-24T21:15:15.586501Z", "principal": null, "payload": {"sequence": 1, "activity_execution_id": "01m5qw92282801x57927cvx33q", "activity_attempt_id": "01M5QW926RR5Y2GRX31DRP80P5", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "cancellation_history_event_id": "01m5qw92xd4yg1z9pd45cyfvzd", "request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "root_request_id": "01M5QW92E0BK0GDWHT9G71MBNX", "cleanup_deadline_at": "2026-10-24T21:16:13.728056Z", "callback_state": "stopped", "evidence_source": "activity_worker", "acknowledged_at": "2026-10-24T21:15:15.585663Z", "received_after_deadline": false, "task": {"id": "01m5qw922den79a15zyessm583", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:13.357541Z", "leased_at": "2026-10-24T21:15:13.496706Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-main-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-main-queue"}}}], "legacy_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-legacy", "run_id": "01m5qw95ncnf12s9cyr66q7pck", "queue": "pr1-activity-cancellation-acknowledgement-legacy-queue", "worker": "pr1-activity-cancellation-acknowledgement-legacy-worker", "workflow_task_id": "01m5qw95nx7rnbn6ngvve83q53", "task_id": "01m5qw95yrdg1xx4g57v6fytj5", "activity_attempt_id": "01M5QW963PD68K0ZPJB6BSP7WY", "activity_execution_id": "01m5qw95yh7rav0ej04snvxqfm", "lease_owner": "pr1-activity-cancellation-acknowledgement-legacy-worker"}, "legacy_request": 202, "legacy_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw968w13nt8j2a0w1pewnq", "workflow_run_id": "01m5qw95ncnf12s9cyr66q7pck", "request_id": "01M5QW968DTFETWENZ785VDCW1", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "legacy_ack": {"status": 409, "response": {"acknowledged": false, "reason": "activity_cancellation_acknowledgement_not_supported", "required_capability": "cooperative_cancellation", "minimum_protocol_version": "1.20", "worker_id": "pr1-activity-cancellation-acknowledgement-legacy-worker", "unavailable": ["worker_claim_capability"], "remediation": "Use a runtime with activity stop acknowledgements and an original claim from a cooperative worker on protocol 1.20 or newer.", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5qw96yehxek7x8mjp3fyqy6", "queue": "pr1-activity-cancellation-acknowledgement-immediate-queue", "worker": "pr1-activity-cancellation-acknowledgement-immediate-worker", "workflow_task_id": "01m5qw96yz3mwt8beb0cngmp8n", "task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "activity_execution_id": "01m5qw977rz8cspyd91hqxexv8", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker"}, "immediate_cancel": {"status": 200, "response": {"outcome": "cancelled", "workflow_id": "pr1-activity-cancellation-acknowledgement-immediate", "run_id": "01m5qw96yehxek7x8mjp3fyqy6", "requested_run_id": null, "resolved_run_id": "01m5qw96yehxek7x8mjp3fyqy6", "command_id": "01m5qw97p9xgyxzwfwmzbczkd1", "command_sequence": 2, "target_scope": "instance", "workflow_type": "parity.v1.cancel_ack", "command_status": "accepted", "command_source": "control_plane", "reason": "acknowledgement fixture", "rejection_reason": null, "validation_errors": []}, "acknowledgement_capability": "absent"}, "immediate_ack": {"status": 409, "response": {"task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "request_id": "01m5qw97p9xgyxzwfwmzbczkd1", "acknowledged": false, "duplicate": false, "reason": "cancellation_request_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "immediate_status": {"status": 200, "response": {"task_id": "01m5qw977yhxzhj2k2sz9y7k6w", "activity_attempt_id": "01M5QW97EMG38E01P5FKGVY82G", "lease_owner": "pr1-activity-cancellation-acknowledgement-immediate-worker", "can_continue": false, "cancel_requested": true, "reason": "run_cancelled", "heartbeat_recorded": false, "lease_expires_at": null, "last_heartbeat_at": null, "run_status": "cancelled", "run_closed_reason": "cancelled", "run_closed_at": "2026-10-24T21:15:19.139787Z", "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "cancelled", "deadlines": {"start_to_close": "2026-10-24T21:20:18.868117Z"}, "worker_session": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first": {"workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5qw98677c10kwqv7h4s9x4e", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5qw986qacg06ps3paph5624", "task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker"}, "stale_fail": {"status": 200, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "outcome": "failed", "recorded": true, "reason": null, "next_task_id": "01m5qw98qqqdsbe49ye8879md3", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_id": "pr1-activity-cancellation-acknowledgement-stale", "run_id": "01m5qw98677c10kwqv7h4s9x4e", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "worker": "pr1-activity-cancellation-acknowledgement-stale-worker", "workflow_task_id": "01m5qw986qacg06ps3paph5624", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m"}, "stale_request": 202, "stale_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9922842rpf9krn13t5dn", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_own_task": {"status": 409, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_acknowledgement_fence_mismatch", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_first_attempt_new_task": {"status": 409, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt_old_task": {"status": 409, "response": {"task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "error": "Activity attempt is not leased for this task.", "reason": "task_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_attempt": {"status": 409, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "acknowledged": false, "duplicate": false, "reason": "activity_cancellation_not_fenced", "history_event_id": null, "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_second_status": {"status": 200, "response": {"task_id": "01m5qw98qqqdsbe49ye8879md3", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "can_continue": false, "cancel_requested": true, "reason": "attempt_closed", "heartbeat_recorded": false, "lease_expires_at": "2026-10-24T21:20:20.353227Z", "last_heartbeat_at": null, "run_status": "waiting", "run_closed_reason": null, "run_closed_at": null, "activity_status": "cancelled", "attempt_status": "cancelled", "task_status": "leased", "deadlines": {"start_to_close": "2026-10-24T21:20:20.353020Z"}, "worker_session": null, "cancellation_acknowledgement": {"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "cancellation_history_event_id": "01m5qw99b2xay0h93x1sskjqbc", "callback_state": "unknown", "history_event_id": null, "acknowledged_at": null, "received_after_deadline": null}, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "stale_history": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:19.630000Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw986a3jv3c1jqyh29q0hh", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw986a3jv3c1jqyh29q0hh", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:19.624680Z", "applied_at": "2026-10-24T21:15:19.624687Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:19.635924Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "workflow_command_id": "01m5qw986a3jv3c1jqyh29q0hh", "command": {"id": "01m5qw986a3jv3c1jqyh29q0hh", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:1adaf9c094cfa4fe23ffbfb817b706da3e95b1fcfe3bef527a1d88a28554f8a8", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:19.624680Z", "applied_at": "2026-10-24T21:15:19.624687Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:19.891980Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw986qacg06ps3paph5624", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:19.639387Z", "leased_at": "2026-10-24T21:15:19.757708Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:16:19.757637Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:19.645942Z", "last_dispatched_at": "2026-10-24T21:15:19.645942Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:20.037645Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.030742Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:20:20.030966Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 5, "event_type": "ActivityRetryScheduled", "timestamp": "2026-10-24T21:15:20.187201Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "retry_task_id": "01m5qw98qqqdsbe49ye8879md3", "retry_of_task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "retry_available_at": "2026-10-24T21:15:20.178836Z", "retry_backoff_seconds": 0, "retry_after_attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "retry_after_attempt": 1, "max_attempts": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "exception_type": "parity.retryable", "activity_attempt": {"id": "01M5QW98JYWQV4RQKAQ6STS1JV", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "task_id": "01m5qw98eqxfwyc0zdfe2t0c3k", "status": "failed", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "closed_at": "2026-10-24T21:15:20.179184Z"}, "exception_class": "RuntimeException", "message": "retry me", "code": 0, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98JYWQV4RQKAQ6STS1JV", "status": "pending", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.030742Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": {"type": "parity.retryable", "message": "retry me", "code": 0, "non_retryable": false}}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "completed", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 6, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:20.359625Z", "principal": null, "payload": {"activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "status": "running", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.353020Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "task": {"id": "01m5qw98qqqdsbe49ye8879md3", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:20.178836Z", "leased_at": "2026-10-24T21:15:20.353020Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:20:20.353227Z", "attempt_count": 2, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:20.219594Z", "last_dispatched_at": "2026-10-24T21:15:20.219594Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:20.508622Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:20.501658Z", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "lineage": [{"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e"}]}, "command": {"id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw98677c10kwqv7h4s9x4e", "resolved_run_id": "01m5qw98677c10kwqv7h4s9x4e", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyMC41MDE2NThaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5OTFOVFRDN0JCWDhaVjVLWjVaUR5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzk5MU5UVEM3QkJYOFpWNUtaNVpRMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKXnByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LXN0YWxlKHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5ODY3N2MxMGt3cXY3aDRzOXg0ZSJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjIwLjUwMTY1OFomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyMC41MDE2NThaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzk5MU5UVEM3QkJYOFpWNUtaNVpRKHdvcmtmbG93X2luc3RhbmNlX2lkCl5wcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1zdGFsZR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzk4Njc3YzEwa3dxdjdoNHM5eDRlAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-stale/runs/01m5qw98677c10kwqv7h4s9x4e/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:97314033b81ef0fcb18ea2e9b0903b78227292617d2066fa5e74e3a5b31b3c0c", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:20.501658Z", "applied_at": "2026-10-24T21:15:20.501658Z"}}}, {"sequence": 8, "event_type": "ActivityCancelled", "timestamp": "2026-10-24T21:15:20.802210Z", "principal": null, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "activity_attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 2, "cancelled_at": "2026-10-24T21:15:20.795645Z", "activity": {"id": "01m5qw98ejttpbyjcrqgr6e12m", "idempotency_key": "01m5qw98ejttpbyjcrqgr6e12m", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW98X14S4NR9NWTFJ0HC3D", "status": "cancelled", "payload_codec": "avro", "attempt_count": 2, "retry_policy": {"snapshot_version": 1, "max_attempts": 2, "backoff_seconds": [0], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "created_at": "2026-10-24T21:15:19.890664Z", "started_at": "2026-10-24T21:15:20.353020Z", "closed_at": "2026-10-24T21:15:20.795645Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "exception": "wwHioz3/VYAiNw4KCmNsYXNzCiBSdW50aW1lRXhjZXB0aW9uCHR5cGUKIHBhcml0eS5yZXRyeWFibGUObWVzc2FnZQoQcmV0cnkgbWUIY29kZQQAGm5vbl9yZXRyeWFibGUCAAA="}, "activity_attempt": {"id": "01M5QW98X14S4NR9NWTFJ0HC3D", "activity_execution_id": "01m5qw98ejttpbyjcrqgr6e12m", "task_id": "01m5qw98qqqdsbe49ye8879md3", "attempt_number": 2, "status": "cancelled", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "started_at": "2026-10-24T21:15:20.353020Z", "closed_at": "2026-10-24T21:15:20.795645Z"}, "task": {"id": "01m5qw98eqxfwyc0zdfe2t0c3k", "type": "activity", "status": "cancelled", "available_at": "2026-10-24T21:15:19.895552Z", "leased_at": "2026-10-24T21:15:20.030742Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:20.807285Z", "principal": null, "payload": {"workflow_command_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "sequence": 1, "call_kind": "activity", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "root_workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:20.501658Z", "cleanup_deadline_at": "2026-10-24T21:16:20.501658Z", "lineage": [{"request_id": "01M5QW991NTTC7BBX8ZV5KZ5ZQ", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-stale", "workflow_run_id": "01m5qw98677c10kwqv7h4s9x4e"}]}, "task": {"id": "01m5qw9922842rpf9krn13t5dn", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:20.501658Z", "leased_at": "2026-10-24T21:15:20.679571Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-stale-worker", "lease_expires_at": "2026-10-24T21:15:30.679486Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-stale-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:20.550270Z", "last_dispatched_at": "2026-10-24T21:15:20.550270Z"}}}], "terminated_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-terminated", "run_id": "01m5qw9a29sq7n9y3p7j80psk2", "queue": "pr1-activity-cancellation-acknowledgement-terminated-queue", "worker": "pr1-activity-cancellation-acknowledgement-terminated-worker", "workflow_task_id": "01m5qw9a2qpwwjd0t2pt6c85by", "task_id": "01m5qw9aav3vemzppj1s2hd037", "activity_attempt_id": "01M5QW9AFB23KJQ4BBBQ1ZQ7V8", "activity_execution_id": "01m5qw9aapb0scpkta377s39ya", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker"}, "terminated_request": 202, "terminated_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9amdz410vdkgdh3sh41y", "workflow_run_id": "01m5qw9a29sq7n9y3p7j80psk2", "request_id": "01M5QW9AM19VRHEZSXDP4ZTD9K", "sequence": 1, "call_kind": "activity", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminate": 200, "terminated_ack": {"status": 200, "response": {"task_id": "01m5qw9aav3vemzppj1s2hd037", "activity_attempt_id": "01M5QW9AFB23KJQ4BBBQ1ZQ7V8", "lease_owner": "pr1-activity-cancellation-acknowledgement-terminated-worker", "request_id": "01M5QW9AM19VRHEZSXDP4ZTD9K", "acknowledged": true, "duplicate": false, "reason": null, "history_event_id": "01m5qw9b78rej9s74b5rxrdyjq", "heartbeat_recorded": false, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "terminated_history": ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityStarted", "CooperativeCancellationRequested", "ActivityCancelled", "CooperativeCancellationDelivered", "TerminateRequested", "WorkflowTerminated", "ActivityCancellationAcknowledged"], "boundary_claim": {"workflow_id": "pr1-activity-cancellation-acknowledgement-boundary", "run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "worker": "pr1-activity-cancellation-acknowledgement-boundary-worker", "workflow_task_id": "01m5qw9bkj8asvajdrtsaka0sk", "task_id": "01m5qw9bvq80x04md6pmhc1x1h", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker"}, "boundary_complete": 200, "boundary_timer": 200, "boundary_request": 202, "boundary_task_id": "01m5qw9cjkmv2nag3wyztp5se1", "boundary_history_before": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}], "boundary_resolved_activity": {"status": 409, "response": {"delivered": false, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": null, "sequence": null, "call_kind": null, "sequence_span": null, "operation_sequence": null, "operation_sequence_span": null, "reason": "cancellation_delivery_not_eligible", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_timer_as_activity": {"status": 409, "response": {"delivered": false, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": null, "sequence": null, "call_kind": null, "sequence_span": null, "operation_sequence": null, "operation_sequence_span": null, "reason": "cancellation_delivery_shape_mismatch", "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_history_after_refusals": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}], "boundary_pending": {"status": 200, "response": {"workflow_id": "pr1-activity-cancellation-acknowledgement-boundary", "run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "accepted": true, "duplicate": true, "run_status": "waiting", "cancellation_request": {"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "delivery_sequence": null, "delivered_at": null, "history_refresh_page_token": "MA=="}}, "acknowledgement_capability": "absent"}, "boundary_delivery": {"status": 200, "response": {"delivered": true, "task_id": "01m5qw9cjkmv2nag3wyztp5se1", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "call_kind": "timer", "sequence_span": 1, "operation_sequence": null, "operation_sequence_span": 1, "reason": null, "protocol_version": "1.20"}, "acknowledgement_capability": true}, "boundary_history_after_delivery": [{"sequence": 1, "event_type": "StartAccepted", "timestamp": "2026-10-24T21:15:23.112252Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "business_key": null, "visibility_labels": null, "memo": null, "search_attributes": null, "outcome": "started_new", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 2, "event_type": "WorkflowStarted", "timestamp": "2026-10-24T21:15:23.118189Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_class": "parity.v1.cancel_ack", "workflow_type": "parity.v1.cancel_ack", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "workflow_command_id": "01m5qw9bk4nrf2d75xrztxnq3s", "command": {"id": "01m5qw9bk4nrf2d75xrztxnq3s", "sequence": 1, "type": "start", "target_scope": "instance", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows", "request_route_name": "generated::4Es3mx7XcNqGD6Dd", "request_fingerprint": "sha256:b565ef783b906c66fa0c83ef3510bee8f7e052f4c9d05135cf070a1127c121df", "status": "accepted", "outcome": "started_new", "accepted_at": "2026-10-24T21:15:23.105479Z", "applied_at": "2026-10-24T21:15:23.105489Z"}}}, {"sequence": 3, "event_type": "ActivityScheduled", "timestamp": "2026-10-24T21:15:23.379716Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "status": "pending", "payload_codec": "avro", "attempt_count": 0, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bkj8asvajdrtsaka0sk", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.121999Z", "leased_at": "2026-10-24T21:15:23.237050Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.236985Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.127029Z", "last_dispatched_at": "2026-10-24T21:15:23.127029Z"}}}, {"sequence": 4, "event_type": "ActivityStarted", "timestamp": "2026-10-24T21:15:23.533501Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "running", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 5, "event_type": "ActivityCompleted", "timestamp": "2026-10-24T21:15:23.669885Z", "principal": null, "payload": {"activity_execution_id": "01m5qw9bvhcg9q1b72z7bgntsv", "activity_attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "activity_class": "parity.v1.cancel_ack_activity", "activity_type": "parity.v1.cancel_ack_activity", "sequence": 1, "attempt_number": 1, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "payload_codec": "avro", "activity": {"id": "01m5qw9bvhcg9q1b72z7bgntsv", "idempotency_key": "01m5qw9bvhcg9q1b72z7bgntsv", "sequence": 1, "type": "parity.v1.cancel_ack_activity", "class": "parity.v1.cancel_ack_activity", "local_activity": false, "attempt_id": "01M5QW9C0607Q3S71NZZ2YZ4Y2", "status": "completed", "payload_codec": "avro", "attempt_count": 1, "retry_policy": {"snapshot_version": 1, "max_attempts": 1, "backoff_seconds": [], "start_to_close_timeout": 300, "schedule_to_start_timeout": null, "schedule_to_close_timeout": null, "heartbeat_timeout": null, "non_retryable_error_types": []}, "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "created_at": "2026-10-24T21:15:23.378048Z", "started_at": "2026-10-24T21:15:23.526423Z", "closed_at": "2026-10-24T21:15:23.667874Z", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}}, "task": {"id": "01m5qw9bvq80x04md6pmhc1x1h", "type": "activity", "status": "leased", "available_at": "2026-10-24T21:15:23.383557Z", "leased_at": "2026-10-24T21:15:23.526423Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:20:23.526604Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue"}}}, {"sequence": 6, "event_type": "TimerScheduled", "timestamp": "2026-10-24T21:15:23.961514Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "task": {"id": "01m5qw9c4xdky5bmwep3wxr6sr", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:23.677513Z", "leased_at": "2026-10-24T21:15:23.828497Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:16:23.828420Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:23.710725Z", "last_dispatched_at": "2026-10-24T21:15:23.710725Z"}}}, {"sequence": 7, "event_type": "CooperativeCancellationRequested", "timestamp": "2026-10-24T21:15:24.110392Z", "principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "command_type": "request_cancellation", "reason": "acknowledgement fixture λ", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "command": {"id": "01M5QW9CJ9BYF4N31WFW484M2S", "sequence": 2, "type": "request_cancellation", "target_scope": "run", "requested_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "resolved_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "payload_codec": "avro", "payload": {"codec": "avro", "blob": "wwHioz3/VYAiNw4GDHJlYXNvbgo0YWNrbm93bGVkZ2VtZW50IGZpeHR1cmUgzrsmY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaGGNhbmNlbGxhdGlvbg4YDHNjaGVtYQpQZHVyYWJsZS13b3JrZmxvdy5jYW5jZWxsYXRpb24tY29udGV4dC92MRRyZXF1ZXN0X2lkCjQwMU01UVc5Q0o5QllGNE4zMVdGVzQ4NE0yUx5yb290X3JlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTMnJvb3Rfd29ya2Zsb3dfaW5zdGFuY2VfaWQKZHByMS1hY3Rpdml0eS1jYW5jZWxsYXRpb24tYWNrbm93bGVkZ2VtZW50LWJvdW5kYXJ5KHJvb3Rfd29ya2Zsb3dfcnVuX2lkCjQwMW01cXc5YmswMmJ6NHh6MzZ5Mmd4MmgzaiJwYXJlbnRfcmVxdWVzdF9pZAAMcmVhc29uCjRhY2tub3dsZWRnZW1lbnQgZml4dHVyZSDOuxJyZXF1ZXN0ZXIOBgh0eXBlChRhdXRoOnRva2VuBGlkChhsZWdhY3ktdG9rZW4KbGFiZWwKCkFkbWluAAxzb3VyY2UKGmNvbnRyb2xfcGxhbmUYcmVxdWVzdGVkX2F0CjYyMDI2LTEwLTI0VDIxOjE1OjI0LjEwNTExNVomY2xlYW51cF9kZWFkbGluZV9hdAo2MjAyNi0xMC0yNFQyMToxNjoyNC4xMDUxMTVaDmxpbmVhZ2UMAg4GFHJlcXVlc3RfaWQKNDAxTTVRVzlDSjlCWUY0TjMxV0ZXNDg0TTJTKHdvcmtmbG93X2luc3RhbmNlX2lkCmRwcjEtYWN0aXZpdHktY2FuY2VsbGF0aW9uLWFja25vd2xlZGdlbWVudC1ib3VuZGFyeR53b3JrZmxvd19ydW5faWQKNDAxbTVxdzliazAyYno0eHozNnkyZ3gyaDNqAAAAAA=="}, "source": "control_plane", "context": {"principal": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}}, "caller_label": "Standalone Server", "principal_type": "auth:token", "principal_id": "legacy-token", "principal_label": "Admin", "auth_status": "authorized", "auth_method": "token", "request_method": "POST", "request_path": "/api/workflows/pr1-activity-cancellation-acknowledgement-boundary/runs/01m5qw9bk02bz4xz36y2gx2h3j/request-cancellation", "request_route_name": "generated::o5AleEuIQaKbg9jg", "request_fingerprint": "sha256:ce4468b0c7d35504e44e71fbcad622d19b219082d18d66cf338a5d1fd2b36d53", "status": "accepted", "outcome": "cancellation_requested", "accepted_at": "2026-10-24T21:15:24.105115Z", "applied_at": "2026-10-24T21:15:24.105115Z"}}}, {"sequence": 8, "event_type": "TimerCancelled", "timestamp": "2026-10-24T21:15:24.751043Z", "principal": null, "payload": {"timer_id": "01m5qw9cdr7ywz4je4acrpfj9h", "sequence": 2, "delay_seconds": 3600, "fire_at": "2026-10-24T22:15:23.960019Z", "cancelled_at": "2026-10-24T21:15:24.750414Z", "task": {"id": "01m5qw9cjkmv2nag3wyztp5se1", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:24.105115Z", "leased_at": "2026-10-24T21:15:24.292591Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:15:34.292508Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:24.153076Z", "last_dispatched_at": "2026-10-24T21:15:24.153076Z"}}}, {"sequence": 9, "event_type": "CooperativeCancellationDelivered", "timestamp": "2026-10-24T21:15:24.757076Z", "principal": null, "payload": {"workflow_command_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "sequence": 2, "call_kind": "timer", "cancellation": {"schema": "durable-workflow.cancellation-context/v1", "request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "root_workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "root_workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j", "parent_request_id": null, "reason": "acknowledgement fixture λ", "requester": {"type": "auth:token", "id": "legacy-token", "label": "Admin"}, "source": "control_plane", "requested_at": "2026-10-24T21:15:24.105115Z", "cleanup_deadline_at": "2026-10-24T21:16:24.105115Z", "lineage": [{"request_id": "01M5QW9CJ9BYF4N31WFW484M2S", "workflow_instance_id": "pr1-activity-cancellation-acknowledgement-boundary", "workflow_run_id": "01m5qw9bk02bz4xz36y2gx2h3j"}]}, "task": {"id": "01m5qw9cjkmv2nag3wyztp5se1", "type": "workflow", "status": "leased", "available_at": "2026-10-24T21:15:24.105115Z", "leased_at": "2026-10-24T21:15:24.292591Z", "lease_owner": "pr1-activity-cancellation-acknowledgement-boundary-worker", "lease_expires_at": "2026-10-24T21:15:34.292508Z", "attempt_count": 1, "repair_count": 0, "connection": "sqlite", "queue": "pr1-activity-cancellation-acknowledgement-boundary-queue", "last_dispatch_attempt_at": "2026-10-24T21:15:24.153076Z", "last_dispatched_at": "2026-10-24T21:15:24.153076Z"}}}]} \ No newline at end of file diff --git a/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs index bd249b3ef..8066eb17e 100644 --- a/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs +++ b/tests/Unit/ServerParityActivityCancellationAckContractTest.mjs @@ -88,6 +88,28 @@ for (const [label, change, message] of [ ['a cleanup budget other than the requested one', state => { state.request.response.cancellation_request.cleanup_deadline_at = '2026-10-24T07:21:08.747589Z'; }, /the cleanup budget runs from the request/], + // The defect in Server #745's first native head: an activity delivery at + // an activity that already completed was recorded instead of refused. + ['a delivery recorded at an activity that resolved before the request', state => { + state.boundary_resolved_activity = structuredClone(state.boundary_delivery); + Object.assign(state.boundary_resolved_activity.response, {sequence: 1, call_kind: 'activity'}); + }, /an activity that resolved before the request is not eligible/], + ['a resolved activity refused as a shape mismatch', state => { + state.boundary_resolved_activity.response.reason = 'cancellation_delivery_shape_mismatch'; + }, /an activity that resolved before the request is not eligible/], + ['an activity delivery accepted at a timer', state => { + state.boundary_timer_as_activity.status = 200; + state.boundary_timer_as_activity.response.delivered = true; + }, /an activity delivery at a timer is a shape mismatch/], + ['a boundary refusal that records history', state => { + state.boundary_history_after_refusals.push(structuredClone(state.boundary_history_before.at(-1))); + }, /boundary refusals record nothing/], + ['a refusal that pins the delivery sequence', state => { + state.boundary_pending.response.cancellation_request.delivery_sequence = 1; + }, /the request is still undelivered after the refusals/], + ['a timer delivery that leaves the timer pending', state => { + state.boundary_history_after_delivery.splice(-2, 1); + }, /delivery only appends|the timer is cancelled under the delivery/], ]) { test(`the contract refuses ${label}`, () => { assert.throws(() => check(observation(change)), message);