diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 2552f6472..679c1d450 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -36,6 +36,10 @@ on: - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs + - tests/Unit/ServerParitySystemMetricsContractTest.mjs + - tests/Unit/ServerParityPrometheusMetricsContractTest.mjs + - tests/Unit/ServerParityPayloadCleanupContractTest.mjs + - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs @@ -86,6 +90,10 @@ on: - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs + - tests/Unit/ServerParitySystemMetricsContractTest.mjs + - tests/Unit/ServerParityPrometheusMetricsContractTest.mjs + - tests/Unit/ServerParityPayloadCleanupContractTest.mjs + - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs @@ -443,8 +451,8 @@ jobs: -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$layout" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt - # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). - docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \ + # Compiling the library tests with two jobs peaks near 4.2 GB; a 4 GB limit OOM-killed rustc (exit 101). + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=6g --memory-swap=6g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ -e DW_TEST_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \ -e DW_EXECUTION_MYSQL_URL=mysql://root:parity-disposable-password@parity-mysql:3306/php_ref \ @@ -955,8 +963,8 @@ jobs: -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_AUTH_TOKEN=parity-test-token \ "$layout" artisan server:bootstrap --force > parity-evidence/published-bootstrap.txt - # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). - docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=4g --memory-swap=4g \ + # Compiling the library tests with two jobs peaks near 4.2 GB; a 4 GB limit OOM-killed rustc (exit 101). + docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" --cpus=2 --memory=6g --memory-swap=6g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ -e DW_TEST_POSTGRES_URL=postgres://parity:parity-disposable-password@parity-pg:5432/php_ref \ -e DW_EXECUTION_POSTGRES_URL=postgres://parity:parity-disposable-password@parity-pg:5432/php_ref \ @@ -1470,8 +1478,8 @@ jobs: } trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence - # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). - docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ + # Compiling the library tests with two jobs peaks near 4.2 GB; a 4 GB limit OOM-killed rustc (exit 101). + docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=6g --memory-swap=6g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ -e DW_UPGRADE_TEST_WRITABLE=/evidence -e DW_UPGRADE_TEST_READONLY=/source/parity-evidence \ -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \ @@ -1510,7 +1518,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port-checkpoints/2026-10-22-payload-cleanup.md b/docs/rust-server-port-checkpoints/2026-10-22-payload-cleanup.md new file mode 100644 index 000000000..f5c962e8c --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-payload-cleanup.md @@ -0,0 +1,50 @@ +# October 22, external payload cleanup + +- **Source.** Server 2.11.1 `SystemController::externalPayloadCleanupStatus` + and `externalPayloadCleanupPass`, `RuntimeExternalPayloadCleanup::runPass`, + `RuntimeExternalPayloadCleanupMetrics::snapshot`, + `RuntimePayloadCompletionUploads::cleanup`, `RuntimePayloadCompletionLease:: + mayResume` and `ExternalPayloadBackupHold::deleting`. +- **Served.** Native answers admin-only + `GET /api/system/external-payload-cleanup` and + `POST /api/system/external-payload-cleanup/pass` in the default or a named + namespace. + - The status is the cleanup section native's operator metrics already + report: the expired backlog, batch bounds, totals and the last pass. + - The pass takes `limit` (`nullable|integer|min:1|max:1000`, default 100). + It first visits the namespace's expired completion-upload allowances: one + whose activity can still complete (its task is leased to the same owner + and its attempt can continue) gets another abandoned-upload period, and + any other is dropped. It then reclaims the namespace's expired, unclaimed + references, oldest first, each under its object lock. A reference that + shares its object with another loses only its row; a sole reference also + deletes its object. The report lists processed, deleted, preserved, + skipped and blocked references, the remaining backlog and scan pressure, + and answers 207 when anything was blocked. Each pass is added to the + namespace's totals, with whole-second times as Eloquent stores them. +- **Decisions.** + - Native stores objects only with the local driver. A reference without an + enabled local policy, or whose object lies outside the current local root, + is blocked rather than deleted. PHP would delete through its configured + remote driver or old root. + - Native never takes an online-backup hold. A hold row left active blocks + deletion as in PHP; a missing row means no hold, where PHP would report a + storage failure because its migration seeds the row. + - Native's completion allowances hold activity contexts only. An allowance + whose context native cannot read cannot resume and is dropped once + expired; PHP would fail the pass on a context it cannot parse. +- **Fixture.** `payload-cleanup` checks a fresh namespace's status, a pass, + the recorded totals, limit validation (0, 1001, text and a fraction answer + 422; null and numeric text are accepted) and the final count of five + passes. Deletion needs expired rows and stored objects, which an HTTP probe + cannot make within the abandoned-upload period, so native tests cover it on + SQLite, PostgreSQL and MySQL: shared, live and retained references, the + limit and its backlog, an active backup hold, a disabled policy, and an + allowance renewed while its activity runs and dropped after it completes. +- **Evidence.** The new tests pass on SQLite, PostgreSQL and MySQL, and the + native suite with `--all-targets` passes on SQLite (652 tests). A local + copy of the SQLite parity step passed all 54 fixtures on embedded PHP and + native, including `payload-cleanup`. The published PHP fixture answers + were checked against a fresh 2.11.1 server. Its record differed only on + `namespace-delete`, a retryable 503 `backend_lock_pressure` that its probe + did not retry; that probe and the storage diagnostic now retry it. diff --git a/docs/rust-server-port-checkpoints/2026-10-22-prometheus-metrics.md b/docs/rust-server-port-checkpoints/2026-10-22-prometheus-metrics.md new file mode 100644 index 000000000..b3e4feda1 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-22-prometheus-metrics.md @@ -0,0 +1,58 @@ +# October 22, Prometheus metrics summary + +- **Source.** Server 2.11.1 `SystemController::prometheusMetrics` and + `PrometheusMetricsSummary::snapshot`, with `BoundedMetricPolicy` and the + declared dimensions in `config/dw-bounded-growth.php`. +- **Served.** Native answers admin-only `GET /api/system/prometheus-metrics` + in the default or a named namespace: + - workflow series by task queue and workflow type: started, completed, + failed, cancelled and terminated totals, running runs and a latency + histogram from run summaries; + - activity series by task queue, workflow type and activity type, with the + same counters and a latency histogram from activity executions; + - task-queue runtime state: ready, due, delayed, leased and expired tasks, + tasks added and dispatched in the last minute, and fresh worker pollers, + slot capacity and available slots. + + Each family keeps the first 100 series in PHP's order. Native reads at + most 101 candidate label sets, then aggregates only the series up to the + last candidate. The disclosure reports observed, reported and suppressed + series and the suppressed starts, with `lower_bound` precision once the + limit is reached. Native refuses the series-limit settings, so the limit is + always PHP's default of 100. +- **Times.** Task and worker comparisons bind whole seconds, as Laravel does + and as native's operator metrics already do. A task made available within + the current second still counts as delayed. Workers are fresh within + `DW_WORKER_STALE_AFTER_SECONDS`, by default `max(3 × heartbeat, 30)`. +- **Decision: cumulative latency buckets.** Published PHP binds each bucket + bound as text. SQLite orders every number before every text value, so PHP + on SQLite counts every duration in every bucket + ([server#683](https://github.com/durable-workflow/server/issues/683)). + Native compares durations with numeric bounds and reports the cumulative + histogram on every backend. The fixture accepts PHP's SQLite shape as a + recorded reference defect; native's tests pin the cumulative buckets with + fixed durations on SQLite, PostgreSQL and MySQL. +- **Correction: undispatched activity tasks.** PHP's worker completion + publishes only the timer tasks it creates. An activity task scheduled by a + workflow task therefore keeps null `last_dispatched_at` and + `last_dispatch_attempt_at` until a poller claims it, and its queue's + dispatched-in-the-last-minute counter stays 0. Native stamped it + dispatched. It now leaves those columns null for that path, as PHP does. + Other task paths keep their stamps. +- **Fixture.** `prometheus-metrics` reads a fresh namespace, then: + - registers a worker with four workflow and three activity slots on the + first queue; + - completes a run there after one activity; + - terminates a second run there; + - starts a run on a second queue that has no worker. + + Counters, task-queue state, worker capacity and the disclosure are exact. + Each counted run and activity lasts at least 1.1 seconds, so latency sums + and the empty low buckets are checked against that bound. +- **Evidence.** Native tests pass on SQLite, PostgreSQL and MySQL, and the + native suite with `--all-targets` passes on SQLite (649 tests). + A local copy of the SQLite parity step passed all 53 fixtures on embedded + PHP and native, including `prometheus-metrics`. Published PHP passed + `prometheus-metrics` too; its only difference was `storage-test`, an + intermittent 503 from PHP's external payload registration on that loaded + host, which passed in the previous local run. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-activity-heartbeats.md b/docs/rust-server-port-checkpoints/2026-10-23-activity-heartbeats.md new file mode 100644 index 000000000..087db8ec9 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-activity-heartbeats.md @@ -0,0 +1,26 @@ +# October 23, activity heartbeats on a closed run + +- **The gap.** When a run was terminated or cancelled while its activity + ran, published Server 2.11.1 answers the activity's next heartbeat with + 200: + - `cancel_requested: true` and `can_continue: false`; + - the closing reason, such as `run_terminated`, with + `run_closed_reason` and `run_closed_at`; + - `heartbeat_recorded: false` and a null `lease_expires_at`. + + The worker then stops the activity as cancelled. Native answered 409 + with its activity-outcome body, which a worker can read as a transport + error rather than a cancellation. +- **What native does now.** For a closed run, native answers PHP's + structured 200 from the same observation its status route reads. + Nothing is recorded and no lease is renewed. An ordinary heartbeat also + carries PHP's fields: + - `reason`, `run_closed_reason` and `run_closed_at`, all null; + - `worker_session`, null; + - `protocol_version` and `server_capabilities`. +- **How it was checked.** A probe ran on native and the published image. + It scheduled an activity with all four timeouts, claimed it, sent a + heartbeat, terminated the run, and sent another heartbeat. The claim's + `deadlines` map and both heartbeat bodies matched. +- **Not checked yet.** A heartbeat while a cooperative cancellation is + pending on a run that is still open. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-activity-lease-expiry.md b/docs/rust-server-port-checkpoints/2026-10-23-activity-lease-expiry.md new file mode 100644 index 000000000..5e185d598 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-activity-lease-expiry.md @@ -0,0 +1,63 @@ +# October 23, activity lease expiry + +- **Source.** Server 2.11.1 `ActivityTaskPoller::claimReadyTask` and + `ActivityTaskClaimer`, `TaskRepair::recoverLeasedTask`, + `ActivityTimeoutEnforcer::enforceLeaseExpiry` (through + `enforceLockedTimeout`, `leaseLossMetadata`, `closeTimedOutAttempt`, + `settleTimeout`, `scheduleRetry` and `recordTerminalTimeout`) and + `ActivityRetryPolicy`. +- **Published behavior.** PHP claims only ready activity tasks, so a worker + never receives a task again after its lease expires. PHP's maintenance runs + the repair pass every second, and the pass settles each expired activity + lease: + - The original attempt closes as expired, and its task is cancelled with + `last_error` `lease_expired`. + - When the activity's retry policy has an attempt left, PHP records + `ActivityRetryScheduled`. It carries `retry_reason` and `failure_reason` + `lease_expired`, the lease's owner and expiry, and no `timeout_kind`. + The retry task waits out the policy's backoff. + - Otherwise PHP records a terminal `ActivityFailed` with + `Workflow\V2\Exceptions\ActivityLeaseExpiredException`, failure category + `internal` and propagation `activity`. Its `non_retryable` is true when + the policy lists that exception. The workflow then runs again to see the + failure. + - An activity whose own deadline also expired is left to the timeout sweep. + - Without a retry policy an activity has one attempt. +- **Evidence for PHP.** A fresh 2.11.1 server on SQLite, with its + one-second repair loop, claimed two activities and had their leases + expired in the database. The activity without a policy recorded + `ActivityFailed` as above. The one with two attempts recorded + `ActivityRetryScheduled` with `retry_reason` `lease_expired`, then attempt + two started on a new task. +- **Native before.** Native's claim query took a leased activity task whose + lease had expired and ran it again as the next attempt. That skipped the + retry policy, its backoff and the history events above. An activity + without retries ran twice. +- **Served.** Native's claim query and its cross-kind poll take only ready + activity tasks. Native's scheduler settles expired activity leases on each + tick, after its deadline sweep, exactly as PHP's enforcer does. The repair + pass now settles an expired running-activity lease the same way; it no + longer reports it as unported. The same check against native recorded the + same events, classes, reasons, lease fields and attempt numbers as PHP. + - Workflow task leases are unchanged: PHP's workflow poller also takes a + workflow task whose lease expired. +- **Decisions.** + - PHP's lease events also embed the closed task's snapshot. Native's + activity events do not carry task snapshots yet; this is the same known + payload difference as its timeout events. + - A run that already closed is skipped as `run_already_terminal`. Native + never delivers an activity task of a closed run, so it does not continue + an abandoned activity there. +- **Restart probe contract.** `restart.php` kills a native process while it + holds an activity lease, waits out the real five-minute lease and finishes + with a new worker. It expected the recovered worker to run attempt two of + an activity without a retry policy, with no event in between. That + expectation came from native's re-claim, not from PHP: published PHP fails + that activity instead. The probe now declares two attempts for the + activity in `current-activity-restart.json` (`activity_retry_policy`) and + expects `ActivityRetryScheduled` with `retry_reason` `lease_expired` + between the two `ActivityStarted` events. +- **Tests.** Native tests on SQLite, PostgreSQL and MySQL cover a retry, + a terminal failure, a non-retryable lease exception, a poll racing the + repair loop, the repair pass and an external result refused after the + lease expired. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-build-pins.md b/docs/rust-server-port-checkpoints/2026-10-23-build-pins.md new file mode 100644 index 000000000..173871f36 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-build-pins.md @@ -0,0 +1,58 @@ +# October 23, build pins at start and in polls + +- **The gap.** Published Server 2.11.1 pins each new run to a worker build, + and its pollers give a pinned run's tasks only to that build's workers. + Native stamped no pin and gave any task to any worker. A run started + under build `v1` could replay on a `v2` worker. A PHP database whose runs + were already pinned would have lost that routing after takeover. +- **What native does now.** + - **Start.** A start resolves the cohort as + `WorkflowStartVersionPin::resolveForStart` does, apart from promoted + rollouts, which native does not keep yet: + - the one build among the type's live workers; + - else the one build on the queue; + - else no pin. + + Only unversioned workers is also no pin. The run's `compatibility` is + that build. `WorkflowStarted` records it as `compatibility` when set. + The declared contract and the definition fingerprint come only from + workers in that cohort. + - **Tasks.** Every task carries its run's pin. + - **Polls.** A task pinned to a build goes only to workers registered + with that build id. The build id is trimmed; a blank one is + unversioned. An unpinned task goes to any worker. The workflow task + response includes `compatibility`, as PHP's does. + - **Children.** A child keeps its parent's pin. A child of an unpinned + parent is unpinned. + - **Redrive.** A redrive checks the definition fingerprint against the + source run's own cohort, its build or the unversioned workers. +- **Contract decision: unpinned activities.** On 2.11.1 an unpinned + activity task is never offered to a versioned worker. With only + versioned workers on the queue it waits forever. Native offers it to any + worker, as both PHP and native do for unpinned workflow tasks. This is + [Workflow #908](https://github.com/durable-workflow/workflow/issues/908). + If the package decides differently there, native follows that decision. +- **How it was checked.** Probes ran against native and the published + 2.11.1 image on SQLite, comparing: + - three workers (`v1`, `v2` and unversioned); + - starts with one cohort, two cohorts and a type served by one build; + - workflow and activity polls from each worker; + - a child of an unpinned parent and of a pinned parent. + + Pins, describe output, `WorkflowStarted` and which worker received each + task matched, apart from the unpinned activity above. + + Two integration tests pass on SQLite, PostgreSQL and MySQL: + `runs_are_pinned_to_their_start_cohort_s_build_like_php` and + `children_keep_their_parent_s_build_pin`. A first child test followed + my reading of the package source, which gives an unpinned parent's child + the starting worker's build. The live comparison showed PHP leaves it + unpinned, and the test now follows PHP. +- **Still missing.** Build-id rollouts: + - promote, drain, resume and rollback on the task-queue and + `/api/deployments` routes; + - worker drain status; + - the promoted cohort at start. + + Native refuses those routes with `task_queue_build_id_rollout_unqualified`. + Takeover refuses a database with rollout rows. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-cli-sweep-and-archive.md b/docs/rust-server-port-checkpoints/2026-10-23-cli-sweep-and-archive.md new file mode 100644 index 000000000..9e1eee716 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-cli-sweep-and-archive.md @@ -0,0 +1,37 @@ +# October 23, CLI 2.3.0 sweep and archive + +- **What ran.** The published `dw` 2.3.0 phar, with its checksum verified + against the release's `SHA256SUMS`, ran 60 commands against native and + against published Server 2.11.1. Both servers ran on SQLite with the same + seed data and used `--output=json`. The commands covered: + - server and workflow reads, start, signal, query, repair, cancel and + terminate; + - archive and redrive; + - standalone activities; + - namespaces, schedules and search attributes; + - task queues and build ids, and workers; + - each `system:*` pass and status, storage, and worker polls. +- **Matched.** 53 commands had the same exit code on both servers: + - `workflow:query` against a registered worker that never polls is + refused by both with `query_worker_unavailable`, after the same + 45-second wait. The CLI's own timeout races that wait, so a run can + show either outcome on either server. + - `namespace:delete` succeeded on native. On the SQLite reference it hit + a backend lock refusal. +- **Ported here: archive.** `POST /api/workflows/{id}/archive` and its + run-targeted route now behave as `WorkflowStub::attemptArchive`: + - An open run is refused durably, 409 `rejected_run_not_closed`. + - A closed run records an archive command and `ArchiveRequested`. The + first archive also sets the run's archive columns and records + `WorkflowArchived`. A repeat answers `archive_not_needed`. + - Laravel's rules (`reason` up to 1000 characters, `request_id` a + string) refuse invalid fields with PHP's messages. + + A live comparison matched PHP's statuses, outcomes, history events and + command payload bytes. Native's describe still reports an accepted + command's `reason` as null, where PHP reads it from the payload. That is + a describe-wide difference, tracked with the other response shapes. +- **Still missing for the CLI.** + - `workflow:redrive` (Server 2.11's redrive of a failed run). + - `task-queue:promote`, `task-queue:drain` and `task-queue:resume`, which + publish PHP's worker-deployment lifecycle and its start routing. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-command-batches.md b/docs/rust-server-port-checkpoints/2026-10-23-command-batches.md new file mode 100644 index 000000000..cc843a6ea --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-command-batches.md @@ -0,0 +1,106 @@ +# October 23, command batches and open work + +Live probes sent the same workflow task completions to published Server +2.11.1 and to native. Native refused several batches that PHP applies: + +| Completion | Published PHP | Native before | +| --- | --- | --- | +| `complete_workflow` before a `start_timer` | timer scheduled, then the run completes | 422 `unsupported_workflow_command` | +| two children, or a child with a timer and an activity | all started, run waiting | 422 `unsupported_workflow_command`, or 409 `pending_durable_operations` with other work open | +| an activity, `continue_as_new` and a timer | activity and timer recorded on the old run, then it continues | 422 `unsupported_workflow_command` | +| `continue_as_new` with an open timer, activity, child or signal wait | the run continues; the open work stays with it | 409 `continue_as_new_transfer_unqualified` | +| `continue_as_new` from a child run | the child continues; its parent follows the new run | 409 `continue_as_new_transfer_unqualified` | +| `complete_update` with other commands | all applied | 422 `invalid_update_task_commands` | +| an update task completed without its update | the update stays accepted for a later task | 409 `update_task_mismatch` | +| two terminal commands | 409 `invalid_commands`, nothing recorded | 422 `invalid_commands` | + +## Source + +Workflow 2.11's `DefaultWorkflowTaskBridge`: + +- **`parseCommands`.** A batch may hold one terminal command + (`complete_workflow`, `fail_workflow`, `continue_as_new`) anywhere. Every + other command applies first, in order, then the terminal one. A second + terminal command makes the batch `invalid_commands`. +- **`validateUpdateCommands`.** Each `complete_update` or `fail_update` must: + - name a different update accepted on the run; + - name the task's own update when the task carries one; + - have no earlier signal still to apply. + + Otherwise the batch is `invalid_commands`. Nothing requires an update + task to complete its update. +- **`applyContinueAsNew`.** It closes the run as `continued` whatever is + open. Activities, timers, children and waits stay with the predecessor, and + no parent-close policy applies. For a child run: + - each parent link gets a new `child_workflow` link to the continued run; + - the parent's open child call follows the new run + (`markChildCallContinued`), counting the continuation; + - an open parent records `ChildRunStarted` for the new run, with the + original `child_call_id` and the new link. + + The continued run's `WorkflowStarted` carries `child_call_id` and the + parent fields, all null for a run without a parent. A child's result is + recorded on its parent with the parent's latest link for that call. + +## What native now does + +- A batch's terminal command applies after every other command, wherever + it sits. Two terminal commands get PHP's 409 `invalid_commands` body, + after the ownership checks. +- A task may start any number of children, alongside activities, timers and + waits. A child's result gets its own parent resume task even when the + parent already has one, as activity and timer results do. +- Update commands follow `validateUpdateCommands`, with PHP's + `invalid_commands` refusal. +- Continue-as-new leaves open activities, timers, children and waits with + the closed predecessor. A continued child stays its parent's child, as + above. +- Continue-as-new takes the command's `workflow_type` and `queue`, as PHP + does. The new type also becomes the run's and the instance's class. Native + used to refuse either with `continue_as_new_routing_unqualified` + (`continue_as_new_takes_a_new_type_and_queue_like_php`). + A child continued into another type still resolves its parent's call, + which keeps the type it started with, and the parent records the new type + (`a_child_continued_into_another_type_resolves_its_parent_like_php`). + +## Completion responses + +PHP's successful completion response lists `created_task_ids`: the tasks the +batch's own commands created, in the order applied. That is each activity's +task, each timer's task, each child's first workflow task, then the continued +run's first task. A resume task that a child's completion creates for its +parent is not listed. The response also carries `protocol_version` and +`server_capabilities`, as every PHP worker response does. Native's response +had none of these three fields, and now has all of them. + +The same probe found the worker envelope missing from workflow task +failures, workflow task heartbeats and activity results and failures. Native +now adds it to each of them. A workflow task heartbeat also reports PHP's +`run_status`, `task_status` and `reason`. Timestamps differ in precision: +native's lease expiry has milliseconds, PHP's microseconds. + +## Differences that remain + +- **Resume tasks.** PHP folds a child's result into the parent's ready or + leased workflow task. Native creates a task for each result, as it already + does for activities and timers. +- **Continue-as-new transfers.** Native still refuses continue-as-new + (`continue_as_new_transfer_unqualified`) while the instance has durable or + inbound streams, the run has service calls, or a query is live. +- **Open waits.** A task that opens a signal or condition wait while + another is open is still refused with `409 pending_durable_operations`. + +## Tests + +`tests/execution/command_batches.rs` checks each PHP answer above on SQLite, +PostgreSQL and MySQL: terminal order, two terminal commands, mixed children, +continue-as-new over open work, a continued child's parent records, and an +update completed alongside other commands after its update task left it open. +It also checks `created_task_ids` and `protocol_version` on the responses. +Two older tests asserted refusals that PHP does not give: + +- `updates_preserve_original_receipts_and_fence_recovered_tasks_across_pools` + now expects PHP's `invalid_commands` for another update in an update task. + It no longer refuses an update task completed with `complete_workflow`. +- `continuation_inherits_committed_upserts_before_successor_commands` no + longer refuses a continuation placed before the memo upsert. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-command-payloads.md b/docs/rust-server-port-checkpoints/2026-10-23-command-payloads.md new file mode 100644 index 000000000..cd8f03f08 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-command-payloads.md @@ -0,0 +1,39 @@ +# October 23, workflow task command payloads must decode + +- **The gap.** Native recorded a workflow task completion whose command + payloads were not Avro values. A run could complete with a result no + client can decode, or schedule an activity with arguments no worker can + read. Published Server 2.11.1 refuses the whole completion first, with a + Laravel validation failure naming the field. Its checks are + `resolveWorkflowTaskCommandPayloadReferences` and then the command + normalizer. +- **What native does now.** Before it applies any command, native checks + each payload field PHP lists in `PAYLOAD_ENVELOPE_FIELDS`, in command and + field order, and stops at the first invalid one: + - A string must decode as one complete Avro value. Otherwise the refusal + names `commands.N.field`. + - In an envelope, the codec must be allowed, or the refusal names + `.codec` with PHP's `unsupported_payload_codec` message. + - An exact `{codec, blob}` envelope needs a non-empty codec and a string + blob, and the blob must decode, or the refusal names `.blob`. + - A `complete_workflow` result or `schedule_activity` arguments that are + neither a string nor an envelope are refused as PHP refuses them. + + External references keep their own path. A refused completion records + nothing, and the task stays leased. +- **How it was checked.** A probe sent 18 completions to native and the + published image. They covered garbage, truncated and non-Avro blobs, + other codecs, plain values, envelopes missing a blob, blobs that are not + strings, empty codecs, and a bad second command. The statuses, reasons + and field errors matched, with three exceptions that predate this + change: + - PHP accepts a bare string `complete_workflow` result; native refuses it + with `unsupported_payload_codec`. + - PHP accepts plain continue-as-new arguments; native refuses them the + same way. + - PHP reports a non-integer timer delay as a Laravel field error; native + reports `invalid_timer_delay`. + + SDKs send envelopes, so the first two do not affect them. + `completion_payloads_must_decode_like_php` passes on SQLite, PostgreSQL + and MySQL. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-completion-rules.md b/docs/rust-server-port-checkpoints/2026-10-23-completion-rules.md new file mode 100644 index 000000000..03197fd79 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-completion-rules.md @@ -0,0 +1,139 @@ +# October 23, workflow task completion rules + +Before published Server 2.11.1 applies a workflow task completion, it checks +it in this order: + +1. Laravel's request middleware trims every string and turns empty ones into + null. +2. `WorkflowTaskCompletionValidation::rules` and the Workflow package's + parallel metadata rules check the top-level fields and every `commands.*` + field. +3. Digit strings in its integer fields become integers. +4. `validateWorkflowTaskCommandScopes` checks options used on the wrong + command, and activity and child timeout envelopes. +5. The task ownership guard runs. +6. The payload pass canonicalizes each command's `payload_codec` and checks + each payload. +7. The command normalizer checks field scopes and per-command values. + +Native had no equivalent of steps 1 to 4 and few of PHP's messages for step 7. +A malformed completion got a native reason such as `invalid_request`, +`invalid_commands`, `invalid_timer_delay` or `unsupported_request_field`, or +a 409 about the attempt. PHP answers 422 `validation_failed` with field errors. +Native now runs the same stages with PHP's messages. + +## What native now does + +- **Trimming.** Strings are trimmed with Laravel's `Str::trim` set (PHP's + trim characters, `\s` and `Str::INVISIBLE_CHARACTERS`), and empty strings + become null. A non-retryable error type padded with em spaces therefore + still matches, and a padded failure message is stored trimmed, as PHP + stores it. The same pinned set replaces the Unicode whitespace predicate + native used for control-plane bodies. +- **Transport rules.** Errors are listed as Laravel lists them: + - commands are validated in chunks of 25, keeping each command's index; + - within a chunk, rule by rule, each rule's expansions in data order; + - the first error is the response message; + - a wildcard attribute is named as written (`commands.0.type`), a named + dotted one with spaces (`sticky cache.workflow id`). + + The rules include `sometimes`, `prohibited` (the five `principal_*` + fields), `required_with` (cancellation cleanup, workflow streams, sticky + cache) and `array:keys`. These checks come before ownership, so a non-owner + gets the field error, as from PHP. +- **Digit strings.** After the rules pass, PHP's integer fields, the attempt, + retry policy and worker session fields turn digit strings into integers. + `"delay_seconds": "7"`, `"max_attempts": "2"`, `"version": "1"` and a wait + timeout of `"1"` are accepted. `"+5"` and `5.0` pass Laravel's rule but stay + non-integers, so the normalizer refuses them, as PHP's does. +- **Command scopes.** PHP's controller messages: + - `retry_policy is only supported for schedule_activity and + start_child_workflow commands.`; + - the same for activity timeouts, worker sessions, child timeouts, + cancellation policy, `non_retryable`, `exception` and failure identity; + - the activity and child timeout envelopes, such as `heartbeat_timeout + cannot exceed start_to_close_timeout.` +- **Normalizer checks.** Each command gets: + - `FIELD_SCOPES` (`Workflow task command field [delay_seconds] is not valid + on a record_version_marker command. …`); + - the timer delay message; + - `Open signal|condition wait timeout_seconds must be a non-negative integer + when provided.`; + - the version marker messages under `commands.N` (`change_id is required`, + `version must be an integer`, …). + + A marker version outside its range is PHP's bridge refusal: 409 with + `outcome: completed`, `recorded: false` and `reason: invalid_commands`. +- **Payload codecs.** Any command's `payload_codec` is canonicalized first, + so an unknown codec is refused with PHP's `unsupported_payload_codec` + message, on a marker too. +- **Bare payload strings.** A bare string `complete_workflow` result, or bare + string activity, child or continue-as-new arguments, is the Avro blob + itself, as PHP's normalizer reads it. An empty string is no payload. +- **Plain continue-as-new arguments.** An array or object without a codec is + no arguments to PHP, so the next run keeps the current input. A number is + refused with PHP's message. +- **Fields PHP ignores.** PHP's normalizer reads a fixed set of fields for + each command type. It ignores any other field unless a scope check refuses + it, and ignores unknown retry policy keys. Native's per-command field lists + used to refuse those fields with `unsupported_request_field`. Native now + drops a field before its own checks only when PHP neither reads nor + scope-checks it for that command type. Examples are an unknown field, a + null option, `queue` on a timer and `priority` on an activity. A field + PHP reads stays. So native still refuses one it cannot run, such as + `worker_session`, rather than silently ignoring a feature. +- **An open run is waiting after its task.** PHP's executor leaves every run + that a completed task did not close `waiting`, even when the task recorded + only version markers. Native left such a run `pending`. + +## Old expectations PHP contradicted + +Eleven native tests asserted answers the published image does not give. Each +was compared live with 2.11.1 and corrected: + +| Case | Old expectation | Published PHP | +| --- | --- | --- | +| Timer and wait delays `"1"`, `"7"` | refused | accepted | +| Marker `version: "1"`, `min_supported: "-1"` | refused | accepted | +| Retry policy `max_attempts: "2"` | refused | accepted | +| Child `cancellation_policy: 1` | one message | both the `string` and `in` messages | +| Partial `sticky_cache` | accepted | `required_with` errors | +| Marker version outside its range | 422 | 409 `invalid_commands` | +| Marker `payload_codec: newer-ignored-codec` | ignored | refused | +| Em-space padded non-retryable type | no match | trimmed, matches | +| Padded exception message | kept padded | trimmed | +| `heartbeat_timeout` over `start_to_close_timeout` | accepted | refused | +| Invalid delays and timeouts | native reasons | Laravel and normalizer messages | + +## Differences that remain + +- **Features native cannot run.** A field PHP reads for a feature native does + not have, such as `worker_session`, local activity attempts or workflow + streams, is refused with `unsupported_request_field`. PHP runs it, or + refuses it with its own capability reason when the worker lacks the + capability (409 `worker_capability_not_supported` in the probe). +- **Numbers too large for a timestamp.** PHP accepts `9223372036854775807` as + a timer delay, retry backoff or wait timeout. For a timer it wraps the fire + time into the past (year -292277022601), so the timer fires at once. Native + refuses these as before. +- **Errors across commands.** PHP's normalizer collects errors from every + command. Native reports the first command's normalizer errors. +- **An object `commands` value.** PHP answers 500. Native refuses it with + `invalid_commands`. + +## How it was checked + +Probes sent the same completions to native and the published image: + +- 34 malformed or edge-case completions: top-level fields, wrong types and + bounds, nested wildcards, `prohibited`, `required_with`, `array:keys`, a + 30-command batch spanning two chunks, unknown and null fields; +- every input from the eleven tests above; +- timer delays, bare strings and plain continue-as-new arguments; +- version markers; +- trimming. + +Statuses, reasons, messages and field errors match, apart from the +differences above. Error map key order differs, since native's JSON objects +sort their keys. `tests/execution/completion_rules.rs` covers the new +behavior on SQLite, PostgreSQL and MySQL. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-deployments.md b/docs/rust-server-port-checkpoints/2026-10-23-deployments.md new file mode 100644 index 000000000..ba235d759 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-deployments.md @@ -0,0 +1,60 @@ +# October 23, build-id rollouts and deployments + +- **What is served.** The rollout surface of published Server 2.11.1: + - `POST /api/task-queues/{queue}/build-ids/{promote,drain,resume}`; + - `GET /api/deployments`, `GET /api/deployments/{namespace/queue@build}`; + - `POST /api/deployments/{name}/{promote,drain,resume,rollback}`. + + They write the same `workflow_worker_build_id_rollouts` rows PHP writes. +- **What the rows change.** + - **Start.** A promoted build pins new starts on its queue before the + live fleet's cohort is considered (`WorkflowStartVersionPin`). The + latest promotion that is active and not rolled back wins. A promoted + unversioned cohort pins nothing, and takes the unversioned workers' + contract and fingerprint. + - **Workers.** Draining a build marks its registrations `draining`. + Registration and heartbeat keep them draining. Their workflow, + activity and query polls answer 409 `worker_draining`. Resume returns + them to `active`. + - **Admission.** A queue whose live workers all drain refuses workflow + starts and standalone activity starts with 409 `task_queue_draining`. + The response carries PHP's worker counts and draining build ids. + - **Promotion checks.** Promotion runs PHP's lifecycle plan. Refusals + come with PHP's ordered blockages: + - unknown deployment; + - a draining or terminal deployment; + - no compatible or no freshly heartbeating workers. + + The fleet joins PHP's two views. Compatibility heartbeats last 30 + seconds and include draining workers; native reads them from the + registration's heartbeat time. Registrations count only live, + undrained workers. + - **Reads.** The build-ids snapshot reports each rollout's drain intent, + times and new-start selection. A rollout without workers gets its own + row. + - **Takeover.** Takeover no longer refuses a PHP database with rollout + rows. +- **How it was checked.** A probe ran the same lifecycle against native and + the published 2.11.1 image, with builds `v1` and `v2` on one queue. It + covered: + - each route's validation, refusals and successes; + - draining polls and re-registration; + - the draining start refusal; + - promotion while draining; + - rollback; + - the deployments list. + + Every status, reason, blockage and timestamp format matched. The + timestamp formats are `Z` on task-queue routes and `+00:00` on + deployment routes. The remaining differences are the worker response + fields native already lacks on every poll, such as `protocol_version`. + + The published `dw` 2.3.0 CLI now matches PHP's exit code on 58 of its 60 + sweep commands, including `task-queue:promote`, `drain` and `resume`. The + other two are not gaps: + - a lock refusal on the SQLite reference; + - the query timing race in the [CLI sweep](2026-10-23-cli-sweep-and-archive.md). + + `build_ids_promote_drain_resume_and_roll_back_like_php` passes on SQLite, + PostgreSQL and MySQL. Two build-ids tests that expected the rollout + routes to be refused now expect PHP's answers. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-deregistration-activity-leases.md b/docs/rust-server-port-checkpoints/2026-10-23-deregistration-activity-leases.md new file mode 100644 index 000000000..00d66679b --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-deregistration-activity-leases.md @@ -0,0 +1,41 @@ +# October 23, deregistration settles activity leases + +- **Source.** Server 2.11.1 `WorkerRegistrationDeregistrar`, + `FencedWorkerDeregistration`, `ActivityTaskLeaseRecovery::recoverForStoppedWorker` + and `ActivityTimeoutEnforcer::revokeWorkerLease` (through + `enforceExpiredAttempt` and `leaseLossMetadata`). +- **Published behavior.** Both deregistration routes, + `DELETE /api/worker/registrations/{id}` and the fenced + `POST /api/worker/registrations/{id}/deregister`, settle each activity + task the departing worker still leases, in the same transaction: + - The original attempt closes as expired, and its task is cancelled with + `last_error` `worker_deregistered`. + - The activity's retry policy then records `ActivityRetryScheduled` or a + terminal `ActivityFailed`, as for an expired lease. `failure_reason` and + `retry_reason` are `worker_deregistered`. The event also carries + `lease_revoked_at` beside the original `lease_expires_at` and + `lease_owner`. + - A lease that had already expired says `lease_expired`, without + `lease_revoked_at`. + - An activity whose deadline also expired settles as that timeout. + - A task the worker no longer holds is left alone. + - The deregistration response is unchanged. +- **Evidence for PHP.** On a fresh 2.11.1 SQLite server, two activities were + claimed by worker `w`, then `w` was deregistered. The activity without a + policy recorded `ActivityFailed` with `failure_reason` + `worker_deregistered` and `lease_revoked_at`. The one with two attempts + recorded `ActivityRetryScheduled` with `retry_reason` + `worker_deregistered`. The same script against native recorded the same + events, reasons and lease fields. +- **Native before.** Native's deregistration released only workflow-task + leases. A departed worker's activity leases stayed until they expired, + five minutes by default. +- **Served.** Both native deregistration routes now settle the departing + worker's activity leases with the lease enforcer, inside their own + transaction, before workflow-task recovery. +- **Tests.** Native tests on SQLite, PostgreSQL and MySQL cover: + - a plain deregistration that schedules a retry, which a new registration + then claims as attempt two; + - a fenced deregistration that fails an activity without retries. + + The existing deregistration tests still pass. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-external-command-input.md b/docs/rust-server-port-checkpoints/2026-10-23-external-command-input.md new file mode 100644 index 000000000..ed38eb237 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-external-command-input.md @@ -0,0 +1,87 @@ +# October 23, external payload references on signal, update and query input + +Published Server 2.11.1 accepts a runtime external payload reference, +`{codec, external_payload}`, as the input of a signal, an update or a query, +as it does for a start. Native accepted one only on start, and refused it on +the other three with `invalid_payload_envelope`. Native now accepts it on all +four. + +- **Signals keep the reference.** The signal's history shows the external + envelope (`{codec, external_payload}`), and its worker reads the bytes as it + does for a start input. +- **Updates and queries carry the bytes inline.** PHP's `resolveToArray` + (update) and `resolve` (query) fetch the referenced bytes, so the update's + `UpdateAccepted` arguments and the query task's `query_arguments` are the + plain `{codec, blob}`. Native now does the same. +- **Each claims the object.** The referenced object is retained, as PHP's + `verifyFetchedBytesAndClaim` retains it. A signal or update claims it in the + transaction that records the command. A query claims it in a transaction of + its own, since it records nothing else. +- **Refusals are PHP's typed external payload errors.** A reference that is + missing, expired, malformed, from another provider or for another codec is + refused with PHP's `durable-workflow.v2.runtime-external-payload-error.v1` + body. That body has `schema`, `reason`, `message`, `retryable` and + `status`, plus the route's envelope: `workflow_id` and the `control_plane` + block on a workflow command, and the protocol fields on a worker route. + SDKs read `retryable` from this body. Native used to answer + `{reason, message}` with the reason as the message. +- **Check order follows PHP's `incomingEnvelope`.** + 1. A provider `external_storage` key is refused. + 2. Then an envelope that is not exactly `codec` and `external_payload`. + 3. Then the reference itself: its fields, size, registry row and expiry. + 4. Only then a codec that differs from the registered reference, as + `external_payload_integrity_mismatch`. + + Native used to refuse a codec other than `avro` before the lookup, as + `external_payload_unsupported`. +- **An input object naming a reference stays an object.** Native's input + reader turns a plain object into its values, as PHP's controller does. PHP's + transport middleware looks first, so `{codec, external_payload, extra}` is + refused there. Native took it as three plain arguments and accepted the + signal. Native now keeps any input object that names `external_payload` or + `external_storage`, and refuses it. + +## Differences that remain + +- **When the claim happens.** PHP claims the object once its bytes are + fetched, before the command's own transaction. A command refused after that + point, such as a signal to a missing workflow, still leaves the object + retained. Native claims inside the command's transaction, so a refused + command leaves the upload unclaimed, and it expires as an abandoned upload. + Native already did this for start. The reference, the bytes and every + accepted command are the same. +- **Stored form of a signal's reference.** PHP stores the provider reference + (`dw-external-payload:v1:`). Native stores its opaque runtime reference + (`dw-native-external-payload-v1:`), as it does for start input. The + [storage audit](../rust-server-storage-audit.md) records both forms. + Takeover still refuses a PHP database with runtime external payload rows + (`upgrade_source_external_payloads_unsupported`). +- **Two refusal messages.** PHP words an invalid reference codec and some + transport-route refusals by context. Native gives one message per reason + there. The reason, status and retryability are the same. + +## How it was checked + +A probe sent the same requests to native and the published image, each +against its own SQLite database. The requests were: + +- a signal, an update and a query, each carrying an uploaded reference; +- a signal reusing a claimed reference; +- refusals: + - a missing reference on start and query; + - a codec mismatch; + - an extra envelope key; + - a provider reference; + - a malformed reference object. + +It then compared: + +- the stored update arguments; +- the `UpdateAccepted` and `query_arguments` payloads; +- the query answer; +- the objects' retention state; +- every refusal body, field by field. + +All matched, apart from the claim timing described above. +`tests/execution/external_payloads/commands.rs` covers the same cases on +SQLite, PostgreSQL and MySQL. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-named-namespace-commands.md b/docs/rust-server-port-checkpoints/2026-10-23-named-namespace-commands.md new file mode 100644 index 000000000..39002aa74 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-named-namespace-commands.md @@ -0,0 +1,76 @@ +# October 23, workflow commands in named namespaces + +A live probe ran the same workflow in a named namespace (`ns-live`) against +published Server 2.11.1 and native. PHP served every request. Native refused +these: + +| Request | Published PHP | Native before | +| --- | --- | --- | +| a completion with a version marker and a side effect | 200 | 422 `development_namespace_command_unqualified` | +| `continue_as_new` | 200, the run continues | the same 422 | +| a signal wait batched with a timer | 200 | the same 422 | +| an `upsert_search_attributes` command | PHP's own refusal, 409 `workflow_metadata_capability_not_advertised` | the same 422 | +| `POST /update/{name}`, then `complete_update` | 202, then 200 | 422 `development_namespace_operation_unqualified` | +| `POST /repair` | 200 `repair_not_needed` | the same 422 | +| `POST /request-cancellation` | 202 | the same 422 | + +The completion refusal was a guard in the batch checks, and the route +refusals were missing entries in the named-namespace route table. + +Native's update admission and cooperative cancellation request looked the +instance up in the `default` namespace. Repair refused any other namespace +(`development_workflow_task_repair_namespace_unqualified`), although its +queries already used the request's namespace, as redrive's do. + +## What native now does + +- Every workflow task command is accepted in any namespace, under the same + checks as in the default namespace. +- Named namespaces route the following, instance- and run-scoped: + - `update/{name}`; + - `repair`; + - `request-cancellation`; + - the run's `redrive`. +- Update admission and cooperative cancellation requests read the instance + and run in the request's namespace. + +## Search attribute upserts need the typed capability + +The same probe showed a difference in every namespace. PHP's +`validateWorkflowTaskSearchAttributeCommands` fills in the registered type of +each attribute an upsert names. Its second metadata capability check therefore +requires the worker to advertise `typed_search_attributes` for any +`upsert_search_attributes` with attributes. Without it, PHP answers 409 +`workflow_metadata_capability_not_advertised`. Native required the capability +only when the worker sent `attribute_types`, and now requires it as PHP does +(`an_upsert_needs_the_typed_capability_like_php`). + +## Differences that remain + +- A route native does not implement at all, such as message streams or Nexus + operations, answers `501` in the default namespace but + `422 development_namespace_operation_unqualified` in a named one. +- A task that opens a wait while another wait is open is refused with + `409 pending_durable_operations` in every namespace (see the port log). + +## Tests + +`named_namespace_commands_like_php` runs, in a named namespace: + +- a version marker and a side effect; +- an update and its completion; +- repair; +- a cooperative cancellation request; +- continue-as-new. + +Three older tests expected the named-namespace refusal: + +- `named_namespaces_preserve_original_runs_and_global_workflow_id_reservation` + now checks that a refused completion in the namespace changes nothing, + using two terminal commands. +- `run_observation_signals` opens two signal waits at once, and now expects + native's `pending_durable_operations`. +- `timed_signal_waits` batches a wait whose timeout is 0 with a second wait. + The default namespace already admits that batch, and the named namespace + now does too (renamed + `named_batched_timeout_wait_is_admitted_like_the_default_namespace`). diff --git a/docs/rust-server-port-checkpoints/2026-10-23-priority-fairness.md b/docs/rust-server-port-checkpoints/2026-10-23-priority-fairness.md new file mode 100644 index 000000000..9fa2b52ec --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-priority-fairness.md @@ -0,0 +1,30 @@ +# October 23, task queue priority and fairness + +- **What is served.** `GET /api/task-queues/{queue}/priority-fairness`, as + published Server 2.11.1's `TaskQueuePriorityFairnessSurface` answers it. + For workflow and activity tasks it reports: + - the ready count; + - priority tiers, most urgent first, each with its fairness classes + sorted by key; + - each class's count and its first task's weight; + - each class's recent dispatch score. +- **Why every score is 0.** PHP keeps dispatch scores in + `InMemoryTaskFairnessState`, which its Apache server builds fresh for + every request. So every class scores 0. PHP's fairness reorder then never + moves a poll's first candidate: within the most urgent tier, the first + class it picks is the oldest task's class. A poll takes the oldest ready + task of the most urgent tier, which is native's existing order. +- **How it was checked.** A probe started workflows on the published image + and on native with these settings: + - two fairness keys, one mixed-case; + - fairness weights; + - a task with neither key nor weight; + - an urgent priority; + - a low priority; + - an invalid key; + - an invalid weight. + + It then read the snapshot and drained the queue with one worker. The + snapshot, the validation refusals and the dispatch order matched. + `priority_fairness_reports_tiers_and_classes_like_php` passes on SQLite, + PostgreSQL and MySQL. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-redrive.md b/docs/rust-server-port-checkpoints/2026-10-23-redrive.md new file mode 100644 index 000000000..bfc229665 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-redrive.md @@ -0,0 +1,45 @@ +# October 23, run redrive + +- **What is served.** `POST /api/workflows/{id}/runs/{run}/redrive` now + behaves as published Server 2.11.1's `DefaultWorkflowControlPlane::redrive` + and `FailedRunRedrivePlan`: + - A run that failed right after an activity failed continues in a new + run. Its history is `StartAccepted`, `WorkflowStarted` and an + `ActivityCompleted` for each completed step before the failed one. + Each reused event names the source run. + - `WorkflowStarted` carries the source run, `resume_step_sequence`, + `recovery_kind: redrive`, the source deadlines, and the source's + definition fingerprint. The new run gets a workflow task, so the worker + replays the reused steps and schedules only the failed activity again. + - A repeated `request_id` answers 200 with the run it already created. + - Each refusal PHP makes is made here with PHP's reason and status: + - `run_not_failed`; + - a failure without an activity boundary, `unrecorded_failure_boundary`; + - `source_run_not_current`; + - memos or search attributes on the source, + `unsupported_visibility_metadata`; + - a child link or external arguments, + `unsupported_source_link_or_payload`; + - a passed execution deadline, `execution_deadline_elapsed`; + - a missing or changed definition fingerprint, + `workflow_definition_unavailable_or_changed`; + - a reused `request_id`, `request_id_already_used`. + - `request_id` longer than 191 characters is refused with Laravel's + message. + - Describe on the redriven run shows the source run as + `continued_from_run_id`. +- **How it was checked.** A probe ran the same scenario against native and + the published 2.11.1 image on SQLite: + - a failed run with a recorded boundary; + - each refusal; + - the redrive, and its repeat; + - the resumed activity; + - completion. + + Statuses, reasons, the replayed history and the resumed step matched. + The integration test `redrive_resumes_a_failed_run_at_its_failed_activity_like_php` + passes on SQLite, PostgreSQL and MySQL. +- **Remaining difference.** `POST /api/workflows/{id}/redrive` without a run + is not a route on PHP. PHP answers its general 404; native answers its + general 501 for an unknown API route. That fallback is server-wide, and is + tracked with the other response shapes. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-repair-pass.md b/docs/rust-server-port-checkpoints/2026-10-23-repair-pass.md new file mode 100644 index 000000000..b3058c0e8 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-repair-pass.md @@ -0,0 +1,65 @@ +# October 23, task repair pass + +- **Source.** Server 2.11.1 `SystemController::repairPass`, + `TaskWatchdog::runPass`, `TaskRepairCandidates`, `TaskRepair`, + `TaskRepairPolicy`, `RequestedTaskScope` and + `ActivityTimeoutEnforcer::expiredExecutionIds`. +- **Served.** Native answers admin-only `POST /api/system/repair/pass` in the + default or a named namespace. Like PHP's, the pass is server-wide. + - The request takes `run_ids` (at most 100 strings of 1 to 128 + characters), `instance_id`, `connection`, `queue` (comma-separated + values; `default` also matches a null or empty column) and + `respect_throttle`. Strings are trimmed and empty ones read as null. + - A pass that respects the throttle runs nothing within 5 seconds of the + previous pass and reports `throttled: true`. Any other pass runs and + restarts the throttle. The throttle is shared by the server's namespaces. + - Selection is PHP's poll-mode selection. It takes up to 25 repairable + tasks and up to 25 runs that need a task. Candidates come one from each + connection, queue and compatibility scope in turn, oldest first. + - A ready task is repairable once it is due and its dispatch failed, its + claim failed, or its dispatch and availability are more than 3 seconds + old. A leased task is repairable once its lease expired. + - Each selected task is recovered under its run. A ready task counts one + more repair. An expired lease returns its task to ready and closes the + lease's activity attempt. A closed run's task is settled instead. The + task is then marked dispatched. + - A run whose projected summary still needs repair gets its recoverable + task back. A replay-blocked task comes first. Otherwise the run gets a + new workflow task. + - An open run past its execution or run deadline, with no open workflow + task, gets a task with reason `deadline_expired`; native closes the run + when that task runs. An expired cancellation cleanup deadline is + enforced under a control task. + - Expired activity deadlines in the requested connections and queues are + enforced as by the activity timeout pass. +- **Decisions.** + - Two recoveries are not ported. They are reported as their candidate's + failure, so the pass answers 207 rather than claiming a repair: + - An expired lease on a running activity. PHP consumes the activity's + retry policy. The failure is also kept as the task's last error, as + PHP keeps a recovery exception. + - A missing task for a run waiting on an activity, a timer, a timer-backed + condition, an update, a signal or a child. + - Native has no service calls and no command contracts to backfill, so + those counts are always zero. + - A closed run leaves an activity open only when it completed without + waiting for it. Native treats such an open activity as abandoned, as + `ActivityAbandonment::allows` would. + - Native runs no repair loop. The pass runs only when requested. +- **Shared rule.** Repair status and the pass now share one candidate query. + The overdue branch also requires availability before the redispatch + cutoff. The cutoff binds with microseconds, as PHP 2.11 binds it. Native's + per-task recheck takes availability after the last dispatch as the + redispatch reference, as `dispatchOverdue` does. +- **Fixture.** `repair-pass` checks nine requests. Three passes are scoped to + run ids and a queue that do not exist, so each selects nothing and echoes + its trimmed filters, with empty strings read as null. Six invalid requests + answer Laravel's 422. The published PHP answers were checked against a + fresh 2.11.1 server. Native tests cover the recoveries on SQLite, + PostgreSQL and MySQL: + - the queue filter; + - the throttle; + - an overdue ready task and an expired workflow lease; + - a lost workflow task; + - a deadline task; + - the 207 report for an expired running activity lease. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-request-bodies.md b/docs/rust-server-port-checkpoints/2026-10-23-request-bodies.md new file mode 100644 index 000000000..4d7644220 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-request-bodies.md @@ -0,0 +1,30 @@ +# October 23, empty and malformed request bodies + +- **Empty bodies.** Published Server 2.11.1 reads an empty POST, PUT or + PATCH body as empty input, as it reads `[]`, so validation answers it. An + empty start gets 422 `The workflow type field is required.` Native + answered with axum's plain-text JSON parse error. Native now treats an + empty body as `{}`, as it already did `[]`. +- **Operation fields on admission refusals.** PHP adds the operation's + fields, such as `workflow_id`, to refusals made before the controller + runs: + - malformed JSON; + - a bad token; + - an unknown namespace; + - a missing version header. + + It does this for start, signal, update and query as for the other + workflow commands. Native's admission responses now recognise those four + operations too. +- **How it was checked.** A probe sent these to native and the published + image: + - malformed bodies on start, signal, poll and register; + - an empty start; + - an array body; + - plain and scalar signal input; + - a plain query input; + - a bad token, an unknown namespace and a missing version header on a + signal; + - an unknown namespace on a start. + + Every status and body matched. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-route-fallback.md b/docs/rust-server-port-checkpoints/2026-10-23-route-fallback.md new file mode 100644 index 000000000..8efe394fd --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-route-fallback.md @@ -0,0 +1,28 @@ +# October 23, unknown routes and methods answer like PHP + +- **The gap.** Native answered every request it had no route for with + `501 rust_capability_not_implemented`, including paths published Server + 2.11.1 does not serve at all. A known path with the wrong method got an + empty 405. PHP answers: + - `404 not_found`: `The route {path} could not be found.` + - `405 method_not_allowed`: `The {METHOD} method is not supported for + route {path}. Supported methods: …` + + Worker-plane answers also carry `protocol_version` and + `server_capabilities`. +- **What native does now.** Native embeds published 2.11.1's route table + (`rust/src/runtime/php-routes.v2.11.1.json`, from `php artisan + route:list`). A request no native route serves is matched against it: + - a path PHP serves with this method is still `501 + rust_capability_not_implemented`: PHP has the capability and native + does not yet; + - a path PHP serves with other methods gets PHP's 405, listing those + methods; + - any other path gets PHP's 404. + + The deployment name is the one path parameter that spans segments, as + its `->where('name', '.+')` does. +- **How it was checked.** A probe sent seven requests to native and the + published image: unknown control-plane, run and worker paths, two wrong + methods, and the root-level unknown path. Bodies and statuses matched + exactly. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-standalone-activities.md b/docs/rust-server-port-checkpoints/2026-10-23-standalone-activities.md new file mode 100644 index 000000000..2c8d32897 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-standalone-activities.md @@ -0,0 +1,46 @@ +# October 23, standalone activities + +- **Source.** Server 2.11.1 `ActivityController` (`POST /api/activities`, + `GET /api/activities`, `GET /api/activities/{id}`), Workflow 2.9 + `StandaloneActivityStartService`, and the host-run paths in + `ActivityOutcomeRecorder::closeStandaloneHostRun` and + `ActivityTimeoutEnforcer::closeStandaloneHostRunAfterTimeout`. +- **Published behavior.** Observed on a fresh 2.11.1 SQLite server: + - **Start.** The activity runs inside a host run of type + `dw.standalone_activity`. The run starts `running` and records + `StartAccepted`, `WorkflowStarted` (with `business_key`) and + `ActivityScheduled`. It never gets a workflow task, only a ready + activity task. The retry policy is always snapshotted, with one + attempt by default. Without `input` the activity has no arguments, and + its poll omits them. The answer is 201 with the activity, run and + deadline fields. + - **Identifiers.** An omitted `activity_id` is generated. A running one + is refused, 422 on `activity_id`. A closed one starts the next run of + the same instance. A workflow's id is refused, also 422 on + `activity_id`. An id reserved in another namespace answers 409 + `activity_id_reserved_in_namespace`. + - **Outcomes.** A result completes the host run, with `WorkflowCompleted` + and the result as output. A terminal failure fails it with + `WorkflowFailed`, carrying the activity's failure fields. A timeout + fails it with `closed_reason` `timed_out` (`failed` for a lost lease) + and PHP's full exception shape. The worker is told `next_task_id` null. + Cancelling the host run cancels the activity. + - **Reads.** `GET /api/activities/{id}` describes the current run of a + host, with its attempts from the run's activity view. Any other id + answers 404 `activity_not_found`. The list is the run list for + `dw.standalone_activity`, newest first, filtered by status bucket and + paged by an offset token. +- **Evidence.** One script drove both servers through validation + failures, a completion, a failure, a schedule-to-start timeout, reuse of + an id, a generated id, a workflow's id, listing with filters and pages, + and a cancel. After masking ids and times, native and PHP gave the same + statuses, messages, bodies and host-run histories. What still differs is + native-wide, not specific to these routes: activity snapshot details in + history, and the extra fields in native's activity poll answer. +- **Also fixed.** The run list now reports a run's own business key, then + its instance's. A host instance has none, so PHP's list shows the run's. +- **Tests.** `standalone_activities` covers the lifecycle, failure and + timeout, refusals and listing on SQLite, PostgreSQL and MySQL. +- **Not ported.** PHP's task-queue drain refusal (`task_queue_draining`), + which native's workflow start does not apply either, and its durable + quotas for host runs, which native enforces nowhere. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-structural-limits.md b/docs/rust-server-port-checkpoints/2026-10-23-structural-limits.md new file mode 100644 index 000000000..29d98f9e7 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-structural-limits.md @@ -0,0 +1,60 @@ +# October 23, structural limits on signals, updates and activity results + +- **Source.** Workflow 2.9 `StructuralLimits`, `StructuralLimitExceededException`, + `WorkflowStub` (signal and update intake, `rejectOversizedSignal`, + `rejectUpdateCommand`), Server 2.11.1 `ExternalWorkflowUpdateAdmission` + and `ActivityOutcomeRecorder`. +- **Settings.** `DW_V2_LIMIT_PENDING_SIGNALS` (default 5000), + `DW_V2_LIMIT_PENDING_UPDATES` (500) and `DW_V2_LIMIT_PAYLOAD_SIZE_BYTES` + (2097152), with their `WORKFLOW_V2_LIMIT_*` names. They are read with the + same casts as the memo limit. A value of zero or less disables that check. + Native used to refuse to start when one was changed. It now honors all + three, and `structural_limits.configured` in the operator metrics reports + them, and the configured memo limit, as PHP's snapshot does. +- **Published behavior.** Observed on a fresh 2.11.1 SQLite server with + limits of 2 signals, 1 update and 300 bytes: + - **Signals.** After validation, a signal is refused when the run already + holds that many received signals, then when its serialized arguments + are longer than the payload limit. Either answers 409 with + `reason`/`rejection_reason` `structural_limit_exceeded`, `outcome` null + and `validation_errors` `[]`. A rejected command and signal row are + recorded, with no history event. An oversized signal keeps no + arguments. Its command carries the limit's kind, value and configured + value, and its signal row carries the limit's message. + - **Updates.** After validation, an update is refused when the run + already holds that many accepted updates, then when its serialized + arguments are too long. Either is a durable rejection: a rejected + command and update row with no arguments, and an `UpdateRejected` event + with the limit's message as `validation_errors.arguments`. The 409 + answer has `outcome` null and also carries `structural_limit_kind`, + `structural_limit_value` and `structural_limit_configured`. Retrying + the same `request_id` returns that answer again. + - **Activity results.** A completed result longer than the payload limit + fails the activity instead, with no retry: `ActivityFailed` with + category `structural_limit`, `non_retryable` true, + `Workflow\V2\Exceptions\StructuralLimitExceededException`, the limit's + message and fields, and a `structural_limit` failure row. The worker + still receives `outcome` `completed` and `recorded` true. +- **Evidence.** The same script ran against the published image and this + branch, with the same limits. Signals, updates (including the retried + request) and the activity result produced the same statuses, outcomes, + reasons, messages, limit fields, rows and history events. The measured + payload was 556 bytes on both. +- **Native before.** Native applied no pending-signal or pending-update + limit. It refused update arguments over 64 KiB with a 413 + `update_arguments_exceed_budget` that left no record. PHP has no such + budget: the payload limit rejects them durably. That native budget is + removed. +- **Tests.** `structural_limits` covers the three cases on SQLite, + PostgreSQL and MySQL. Unit tests cover the setting casts and messages. +- **Remaining differences.** + - PHP applies the pending-activity, pending-child and pending-timer limits + only when its own PHP workflows run, not to external workers' commands. + Native's handling of those settings and of the command batch size is + unchanged. + - Native still caps a run at 64 updates over its lifetime, with a 503 + `update_inventory_exceeds_budget`. PHP has no lifetime cap. +- **Also fixed.** The same comparison showed three older differences in + updates, now matched: the answer names the requester's `principal` and + carries `rejected_at`, and `UpdateAccepted` records `request_id` only + when the request named one. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-update-lifetime-cap.md b/docs/rust-server-port-checkpoints/2026-10-23-update-lifetime-cap.md new file mode 100644 index 000000000..1e0291dbc --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-update-lifetime-cap.md @@ -0,0 +1,13 @@ +# October 23, no lifetime cap on updates + +- **Change.** Native no longer refuses a run's 65th update with 503 + `update_inventory_exceeds_budget`. That cap came from the first update + slice, before native enforced the Workflow package's structural limits. +- **Why.** Published PHP 2.11.1 bounds only updates still pending, with + `DW_V2_LIMIT_PENDING_UPDATES` (default 500). It has no limit on how many + updates a run receives over its life. Native has enforced that pending + limit like PHP since the structural limits checkpoint + ([2026-10-23-structural-limits](2026-10-23-structural-limits.md)). +- **Test.** `updates_have_no_lifetime_cap_and_preserve_accepted_work_after_wait_timeout` + completes 64 updates, checks that a repeated request id still answers + with its stored result, then admits a 65th update with 202. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-waterline-service-mode.md b/docs/rust-server-port-checkpoints/2026-10-23-waterline-service-mode.md new file mode 100644 index 000000000..528036454 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-waterline-service-mode.md @@ -0,0 +1,30 @@ +# October 23, Waterline service mode against native + +- **What ran.** The published `durableworkflow/waterline:2.12.3` image ran in + service mode, as `SERVICE_MODE.md` describes, with operator access. It was + pointed first at native and then at published Server 2.11.1, both on + SQLite. +- **Data.** A completed, a failed, a cancelled, a terminated and a waiting + workflow; one that completed an activity; a schedule; and a standalone + activity. +- **Calls.** Every Waterline read route was called, plus query, signal, + repair and cancel through Waterline: + - stats, health and capacity evidence; + - the five flow lists; + - instance and run detail, history export and the activity result; + - the service catalog lists; + - schedules, one schedule and its history. +- **Found.** Waterline's Services pages answered 501 against native and 200 + against PHP. The PHP SDK reads the catalog only when cluster info + advertises `service_catalog_read_contract` version 1. Native served the + eight catalog routes but did not advertise them. Native's cluster info now + carries PHP's contract unchanged, and the pages load. +- **Result.** Every route now gets the same status from both servers: + - Capacity evidence is unavailable on both. Published PHP does not expose + that contract either. + - A query with no polling worker is refused on both with + `query_worker_unavailable`. + - The other routes answer 200 on both. +- **Not covered.** This checks Waterline's JSON routes, not its rendered + pages. Response bodies differ in fields tracked elsewhere: timestamp + precision, identifier case and several describe fields. diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 91089e775..96247ce9f 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -99,13 +99,43 @@ Updated October 22, 2026. ## API surface still missing These published PHP routes on this tree have no native handler yet, by family. - -- **Used by the CLI or Waterline.** - - Namespace delete. - - Standalone activities: list, describe and start. - - `system` metrics (JSON and Prometheus), the repair pass, and external - payload cleanup and its pass. - - Task queue priority and fairness. +A request for one of them answers `501 rust_capability_not_implemented`; +any other unknown path or method answers PHP's own 404 or 405 +([checkpoint](rust-server-port-checkpoints/2026-10-23-route-fallback.md)). + +- **Used by the CLI or Waterline.** None remain. Served from this family + since: task queue priority and fairness + ([checkpoint](rust-server-port-checkpoints/2026-10-23-priority-fairness.md)); standalone activities (start, list and + describe), compared live with published 2.11.1 + ([checkpoint](rust-server-port-checkpoints/2026-10-23-standalone-activities.md)); + run archive, instance and run-targeted + ([checkpoint](rust-server-port-checkpoints/2026-10-23-cli-sweep-and-archive.md)); + run redrive of a failed run at its failed activity + ([checkpoint](rust-server-port-checkpoints/2026-10-23-redrive.md)); + build pins at start and in polls, where unpinned activities follow + [Workflow #908](https://github.com/durable-workflow/workflow/issues/908) + ([checkpoint](rust-server-port-checkpoints/2026-10-23-build-pins.md)); + build-id rollouts and deployments: promote, drain, resume and rollback, + draining polls and starts + ([checkpoint](rust-server-port-checkpoints/2026-10-23-deployments.md)); + the service catalog read contract in cluster info, which Waterline 2.12.3's + Services pages require; with it, every Waterline route answered native as it + answered PHP + ([checkpoint](rust-server-port-checkpoints/2026-10-23-waterline-service-mode.md)); + namespace delete, with the + `namespace-delete` fixture. The JSON + `GET|POST /api/system/metrics` is served with the `system-metrics` + fixture: namespace failure, durable-state and payload counts are exact; + native enforces no admission, durable-state or payload quota, so limits + are null and rejections zero, as on an unconfigured PHP server. + Projection drift is server-wide, and native's only drift source is a run + without its summary. The Prometheus summary + `GET /api/system/prometheus-metrics` is served with the + `prometheus-metrics` fixture, with PHP's default limit of 100 series per + family. External payload cleanup and its pass are served with the + `payload-cleanup` fixture. The task repair pass is served with the + `repair-pass` fixture; a recovery native does not perform yet is + reported as its candidate's failure, with 207. - **Worker protocol features.** - Worker sessions. - Update validation tasks. @@ -116,8 +146,6 @@ These published PHP routes on this tree have no native handler yet, by family. is served: `POST /api/worker/activity-tasks/{task}/renew-lease`, with the `activity_lease_renewal` capability and the `activity-lease-renewal` fixture.) - **Run operations.** - - Archive (instance and run). - - Redrive. - Imports from embedded v2 and Waterline v1. - **Other families.** - Message streams. @@ -126,7 +154,6 @@ These published PHP routes on this tree have no native handler yet, by family. native never holds catalog rows, so every list is PHP's empty page and every identity its not-found answer; the `service-catalog` fixture checks 37 cases.) - Nexus operations. - - Deployments. - Bridge adapters. ## Known gaps found in review @@ -154,22 +181,41 @@ contract fixture before its correction. arguments at admission and refuses an undecodable blob with `invalid_query_arguments`. Published PHP first looks for a query worker, and answers `409 query_worker_unavailable` when there is none. -- **External payload references on signal, update and query.** Published - PHP resolves `{codec, external_payload}` input on these routes; native - supports it only on start. +- **When an external payload reference is claimed.** Published PHP retains + a referenced object once it fetches the bytes, before the command is + recorded, so a refused command still retains it. Native claims it in the + command's transaction, so the upload of a refused command expires + unclaimed. Signal, update and query input otherwise resolve references as + PHP does + ([checkpoint](rust-server-port-checkpoints/2026-10-23-external-command-input.md)). +- **Which worker receives an accepted update.** Published PHP gives the + workflow task that carries an update to any worker of the type. During a + rolling deploy that adds the handler, an older worker can take the update + and fail it (`UnknownUpdate`). + Native offers that task only to a worker that advertises + `workflow_updates` and declares the update for the run's type. The PHP, + Python and Rust SDKs do both exactly when they register the handler. This + is a contract decision, recorded in + [Server #690](https://github.com/durable-workflow/server/issues/690) + (`tests/execution/update_routing.rs`). - **Repair of a run without a recoverable task.** Published PHP's repair - creates a run's missing task, and recovers an expired activity-task lease - with its attempt. Native refuses both with - `development_workflow_task_repair_unqualified`. + command creates a run's missing task, and settles an expired activity-task + lease through the activity's retry policy. Native's repair command refuses + both with `development_workflow_task_repair_unqualified`. Native's + scheduler and the repair pass settle expired activity leases as PHP's + repair loop does, so such a lease rarely remains for the command + ([checkpoint](rust-server-port-checkpoints/2026-10-23-activity-lease-expiry.md)). - **Structural limits.** For external workers, published PHP enforces four of the Workflow package's `DW_V2_LIMIT_*` limits: the memo size, the command batch size as a bound on parallel group metadata, the pending signal and update counts with their payload size at intake, and the - payload size of an activity's result. Native enforces only the memo size. - #612 tracks the rest. -- **Environment settings.** Of PHP's 164 contract settings, native honors - 40 and ignores 57 that change nothing it serves. It refuses to start on a - non-default value of the other 67 + payload size of an activity's result. Native enforces all but the command + batch size, and still caps parallel groups at 100 members and a run at 64 + updates over its lifetime. #612 tracks the rest + ([checkpoint](rust-server-port-checkpoints/2026-10-23-structural-limits.md)). +- **Environment settings.** Of PHP's 166 contract settings, native honors + 43 and ignores 57 that change nothing it serves. It refuses to start on a + non-default value of the other 66 ([checkpoint](rust-server-port-checkpoints/2026-10-22-environment-contract.md)). Several of those are cheap to honor: repair timing and the multi-node declaration. @@ -186,11 +232,10 @@ contract fixture before its correction. - **Child completion while the parent has an open workflow task.** Published Workflow records the child resolution and reuses the parent's existing ready or leased workflow task (`WorkflowExecutor::dispatchParentResumeTasks`). - Native gives the resolution its own resume task when the parent's open - tasks only deliver updates, as activity and timer resolutions do. With any - other open task it refuses the child's completion with - `409 child_parent_task_conflict`, unless the parent has a cooperative - cancellation request. Signals still create no resume while the parent + Without a cooperative cancellation request, native gives the resolution + its own resume task, as activity and timer resolutions do + ([checkpoint](rust-server-port-checkpoints/2026-10-23-command-batches.md)). + Signals still create no resume while the parent waits on a child, because only signal and condition waits count as open waits. - **Update delivery into a ready task.** Published PHP admits an update @@ -239,18 +284,11 @@ contract fixture before its correction. build id: a promoted build-id rollout, else the single build id among the live workers for the type, else among all the queue's live workers. Native starts every run unpinned, and has no build-id rollouts. -- **Named-namespace workflow commands.** Published PHP serves every workflow - route in any namespace. Native refuses updates, repair and cooperative - cancellation requests (`/update/{name}`, `/repair`, - `/request-cancellation`, instance- or run-scoped) in a named namespace with - `422 development_namespace_operation_unqualified`. -- **Named-namespace worker commands.** In a named namespace, native - accepts only `complete_workflow`, `fail_workflow`, `schedule_activity`, - `start_timer`, `start_child_workflow` and a single wait in a workflow task. - It refuses any other command, such as side effects, version markers, - memo and search attribute upserts, updates and continue-as-new, with - `422 development_namespace_command_unqualified`. Published PHP accepts - them in any namespace. +- **Named-namespace routes native does not serve.** Native now serves every + workflow command and operation it implements in any namespace, as PHP + does ([checkpoint](rust-server-port-checkpoints/2026-10-23-named-namespace-commands.md)). + A route native does not implement answers `501` in the default namespace, + but `422 development_namespace_operation_unqualified` in a named one. - **Child cancellation cascade, remaining parts.** Native propagates a parent's cooperative cancellation to its child under `try_cancel` and `wait_cancellation_completed`, as PHP does. The rest is open: @@ -278,10 +316,18 @@ contract fixture before its correction. - **Root conflicts.** PHP records a rejected command when a child refuses a request under another root. Native records only the refusal on the parent's events. -- **Several open operations at once.** Published PHP lets a run hold any - number of open activities, timers, children and waits. Native refuses +- **Several open waits at once.** Published PHP lets a run hold any number + of open activities, timers, children and waits. Native now starts children + alongside other open work, but still refuses `409 pending_durable_operations` when a task opens a wait while another - wait is open, or starts a child while other work is outstanding. + wait is open. +- **Continue-as-new transfers.** Native continues a run over open + activities, timers, children and waits, and from a child run, as PHP does + ([checkpoint](rust-server-port-checkpoints/2026-10-23-command-batches.md)). + It still refuses continue-as-new while the instance has durable or inbound + streams, the run has service calls, or a query is live + (`continue_as_new_transfer_unqualified`); PHP continues it. A new workflow + type or queue on the command now applies, as in PHP. - **Trace context (Workflow #773 / workflow#788, not yet published).** A Workflow minor will add nullable `observability_context` columns on runs, commands and tasks, plus an entry in schedule metadata. Native-owned @@ -319,10 +365,16 @@ contract fixture before its correction. as that wall-clock time in UTC (`12:34:56Z`), which is a different instant ([workflow#898](https://github.com/durable-workflow/workflow/issues/898)). Native stores the instant written (`10:34:56Z`). -- **Worker payload datums.** Published PHP refuses a worker command payload, - such as a `complete_workflow` result, that is not one complete Avro Value - (`422 validation_failed` on `commands.N.result.blob`). Native checks only - the envelope's transport shape and accepts it. +- **Workflow task command fields.** Native checks a completion with PHP's + transport rules, then its payload and normalizer rules + ([checkpoint](rust-server-port-checkpoints/2026-10-23-completion-rules.md)). + Two differences remain: + - A field for a feature native cannot run, such as `worker_session`, is + refused with `unsupported_request_field`. Fields PHP ignores are dropped + as PHP drops them. + - A timer delay, retry backoff or wait timeout too large for a timestamp + is refused on native. PHP accepts it, and wraps a timer's fire time into + the past, so the timer fires at once. - **Search attribute limits.** Over 100 attributes or 64 KB per run, PHP raises an uncaught `InvalidArgumentException` while applying the command. Native refuses the completion with `422 search_attribute_limit_exceeded` @@ -4199,6 +4251,11 @@ MariaDB qualification exposed decimal promotion of a nullable integer through Admission requires a quiescent wait and bounds each run to 64 updates, with 64 KiB argument blobs, 256 KiB results and at most 30 seconds per completion wait. Waiting releases the transaction and database connection. +(Since October 23 native enforces PHP's structural limits instead of the +64-update and 64 KiB bounds, which PHP does not have; see the +[structural limits](rust-server-port-checkpoints/2026-10-23-structural-limits.md) +and [update lifetime cap](rust-server-port-checkpoints/2026-10-23-update-lifetime-cap.md) +checkpoints.) Queued/busy control routing, validators, failure tasks and full update semantics remain open. The qualified tenth child-workflow fixture calls two children in sequence, each executing diff --git a/rust/src/config_audit.rs b/rust/src/config_audit.rs index 6b960f49b..3eb4572f0 100644 --- a/rust/src/config_audit.rs +++ b/rust/src/config_audit.rs @@ -447,7 +447,7 @@ const NEXUS: &str = "native cannot apply a changed Nexus operation limit"; const PROVENANCE: &str = "native does not publish package provenance"; const BOUNDARY: &str = "native has no service boundary policy"; const V1: &str = "native does not run Workflow 1 workflows"; -const COMPATIBILITY: &str = "native does not route work by worker compatibility markers"; +const COMPATIBILITY: &str = "native pins runs to registered worker build ids only; it has no server-wide compatibility markers"; const PINNING: &str = "native always pins runs to their recorded workflow definition"; const STRUCTURAL: &str = "native cannot apply a changed structural limit"; const REPAIR: &str = "native cannot apply changed task repair timing"; @@ -591,12 +591,12 @@ pub const SETTINGS: &[Setting] = &[ Setting { name: "DW_V2_LIMIT_COMMAND_BATCH_SIZE", legacy: Some("WORKFLOW_V2_LIMIT_COMMAND_BATCH_SIZE"), disposition: Refused(Int(1000), STRUCTURAL) }, Setting { name: "DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", legacy: Some("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_SIZE"), disposition: Ignored }, Setting { name: "DW_V2_LIMIT_MEMO_SIZE_BYTES", legacy: Some("WORKFLOW_V2_LIMIT_MEMO_SIZE_BYTES"), disposition: Honored }, - Setting { name: "DW_V2_LIMIT_PAYLOAD_SIZE_BYTES", legacy: Some("WORKFLOW_V2_LIMIT_PAYLOAD_SIZE_BYTES"), disposition: Refused(Int(2097152), STRUCTURAL) }, + Setting { name: "DW_V2_LIMIT_PAYLOAD_SIZE_BYTES", legacy: Some("WORKFLOW_V2_LIMIT_PAYLOAD_SIZE_BYTES"), disposition: Honored }, Setting { name: "DW_V2_LIMIT_PENDING_ACTIVITIES", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_ACTIVITIES"), disposition: Refused(Int(2000), STRUCTURAL) }, Setting { name: "DW_V2_LIMIT_PENDING_CHILDREN", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_CHILDREN"), disposition: Ignored }, - Setting { name: "DW_V2_LIMIT_PENDING_SIGNALS", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_SIGNALS"), disposition: Refused(Int(5000), STRUCTURAL) }, + Setting { name: "DW_V2_LIMIT_PENDING_SIGNALS", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_SIGNALS"), disposition: Honored }, Setting { name: "DW_V2_LIMIT_PENDING_TIMERS", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_TIMERS"), disposition: Ignored }, - Setting { name: "DW_V2_LIMIT_PENDING_UPDATES", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_UPDATES"), disposition: Refused(Int(500), STRUCTURAL) }, + Setting { name: "DW_V2_LIMIT_PENDING_UPDATES", legacy: Some("WORKFLOW_V2_LIMIT_PENDING_UPDATES"), disposition: Honored }, Setting { name: "DW_V2_LIMIT_SEARCH_ATTRIBUTE_SIZE_BYTES", legacy: Some("WORKFLOW_V2_LIMIT_SEARCH_ATTRIBUTE_SIZE_BYTES"), disposition: Ignored }, Setting { name: "DW_V2_LIMIT_WARNING_THRESHOLD_PERCENT", legacy: Some("WORKFLOW_V2_LIMIT_WARNING_THRESHOLD_PERCENT"), disposition: Ignored }, Setting { name: "DW_V2_MATCHING_ROLE_QUEUE_WAKE", legacy: Some("WORKFLOW_V2_MATCHING_ROLE_QUEUE_WAKE"), disposition: Ignored }, diff --git a/rust/src/runtime/activity_completion_uploads.rs b/rust/src/runtime/activity_completion_uploads.rs index c68b7e4ad..04341c1e4 100644 --- a/rust/src/runtime/activity_completion_uploads.rs +++ b/rust/src/runtime/activity_completion_uploads.rs @@ -129,6 +129,10 @@ impl Context { value, }) } + /// The task, attempt and lease owner this context names. + pub(super) fn lease_identity(&self) -> (&str, &str, &str) { + (&self.task, &self.attempt, &self.owner) + } fn scope(&self, namespace: &str) -> String { identity(&json!([ namespace, diff --git a/rust/src/runtime/activity_deadlines.rs b/rust/src/runtime/activity_deadlines.rs index 336ecb7d8..9d9995d1f 100644 --- a/rust/src/runtime/activity_deadlines.rs +++ b/rust/src/runtime/activity_deadlines.rs @@ -7,6 +7,9 @@ use serde_json::{Value, json}; use super::transaction::Transaction; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +/// The failure PHP records when an attempt loses its task lease. +const LEASE_EXCEPTION: &str = "Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException"; + pub(super) const TIMEOUT_FIELDS: [&str; 4] = [ "start_to_close_timeout", "schedule_to_start_timeout", @@ -121,6 +124,10 @@ fn timeout_message(activity: &DB::Row, kind: &str) -> Result, { + if kind == "lease" { + let label = DB::string(activity, "activity_type")?; + return Ok(format!("Activity {label} lost its original task lease before the attempt settled.")); + } let deadline_field = match kind { "schedule_to_close" => "schedule_to_close_deadline_at", "schedule_to_start" => "schedule_deadline_at", @@ -138,6 +145,15 @@ where Ok(format!("Activity {label} heartbeat deadline expired at {expired_at} (last heartbeat: {last}).")) } +/// Why an activity attempt lost its task lease. +#[derive(Clone, Copy)] +pub(super) enum LeaseLoss<'a> { + /// The lease expired. + Expired, + /// The worker that holds it deregistered (`revokeWorkerLease`). + Revoked { namespace: &'a str, owner: &'a str }, +} + /// What enforcing one execution's deadline did. pub(super) enum Enforcement { Enforced { retried: bool }, @@ -243,12 +259,162 @@ where }) => return Ok(Skipped("run_already_terminal")), Err(error) => return Err(error), }; - let retried = Self::time_out_activity(&mut tx, &run, &activity, kind).await?; + let retried = Self::time_out_activity(&mut tx, &run, &activity, kind, None).await?; tx.commit().await?; self.wake.notify_waiters(); Ok(Enforcement::Enforced { retried }) } + /// Published PHP's maintenance runs the repair pass every second, and the + /// pass settles each expired activity lease. Native's scheduler settles + /// them on its own ticks instead. + pub(super) async fn expire_activity_leases(&self) -> Result<()> { + const DUE: &str = "SELECT id FROM workflow_tasks WHERE status='leased' AND task_type='activity' AND lease_expires_at IS NOT NULL AND lease_expires_at<=$1 ORDER BY lease_expires_at,id LIMIT 50"; + let candidates = Self::query(DUE).bind(DB::bind_time(now())).fetch_all(&self.pool).await?; + for candidate in candidates { + self.enforce_lease_expiry(&DB::string(&candidate, "id")?).await?; + } + Ok(()) + } + + /// `ActivityTimeoutEnforcer::enforceLeaseExpiry`: an activity attempt + /// whose task lease expired before it reported an outcome closes as + /// expired, and the activity's retry policy decides between a retry and + /// a terminal `ActivityFailed`. An expired business deadline is left to + /// the deadline sweep, which owns that outcome. + pub(super) async fn enforce_lease_expiry(&self, task_id: &str) -> Result { + let mut tx = self.begin().await?; + let outcome = Self::settle_lost_lease(&mut tx, task_id, LeaseLoss::Expired).await?; + if let Enforcement::Enforced { .. } = outcome { + tx.commit().await?; + self.wake.notify_waiters(); + } + Ok(outcome) + } + + /// `ActivityTimeoutEnforcer::enforceExpiredAttempt` for one leased + /// activity task, inside the caller's transaction. It writes nothing when + /// it skips. A revocation proceeds even past an expired deadline, which + /// then settles as that timeout; a lease the deregistered worker no longer + /// holds is left alone. + pub(super) async fn settle_lost_lease( + tx: &mut Transaction<'_, DB>, + task_id: &str, + loss: LeaseLoss<'_>, + ) -> Result { + use Enforcement::Skipped; + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_optional(&mut **tx).await?; + let execution_id = match &task { + Some(task) => DB::optional_document_row(task, "payload")? + .and_then(|payload| payload["activity_execution_id"].as_str().map(str::to_owned)), + None => None, + }; + let (Some(task), Some(execution_id)) = (task, execution_id) else { + return Ok(Skipped("activity_execution_missing")); + }; + let Some(activity) = Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(&execution_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(Skipped("execution_not_found")); + }; + if !matches!(DB::string(&activity, "status")?.as_str(), "pending" | "running") { + return Ok(Skipped("execution_already_terminal")); + } + let at = now(); + let deadline = expired_kind::(&activity, at)?; + if deadline.is_some() && matches!(loss, LeaseLoss::Expired) { + return Ok(Skipped("activity_deadline_expired")); + } + if let Some(reason) = Self::attempt_state_reason(tx, &activity).await? { + return Ok(Skipped(reason)); + } + let run = match Self::active_run(tx, &DB::string(&activity, "workflow_run_id")?).await { + Ok(run) => run, + Err(super::RuntimeError::Refused { + reason: "run_closed" | "cancellation_deadline_expired", + .. + }) => return Ok(Skipped("run_already_terminal")), + Err(error) => return Err(error), + }; + // `originalLeaseMatches`: the task still carries the current + // attempt's lease. + let attempt = match DB::optional_string(&activity, "current_attempt_id")? { + Some(attempt_id) => Self::query("SELECT * FROM activity_attempts WHERE id=$1") + .bind(attempt_id) + .fetch_optional(&mut **tx) + .await?, + None => None, + }; + let expires = DB::optional_instant(&task, "lease_expires_at")?; + let owner = DB::optional_string(&task, "lease_owner")?; + let original = match &attempt { + Some(attempt) => { + DB::optional_string(attempt, "workflow_task_id")?.as_deref() == Some(task_id) + && DB::string(&task, "status")? == "leased" + && DB::string(&task, "task_type")? == "activity" + && DB::number(&task, "attempt_count")? == DB::number(attempt, "attempt_number")? + && owner == DB::optional_string(attempt, "lease_owner")? + } + None => false, + }; + // `leaseLossMetadata`: an expiry needs the lease to have expired; a + // revocation needs the departing owner, in its namespace, with the + // attempt still running. + let authorized = match loss { + LeaseLoss::Expired => expires.is_some_and(|expires| at >= expires), + LeaseLoss::Revoked { namespace, owner: departing } => { + DB::optional_string(&run, "namespace")?.as_deref() == Some(namespace) + && DB::optional_string(&task, "namespace")?.as_deref() == Some(namespace) + && owner.as_deref() == Some(departing) + && match &attempt { + Some(attempt) => DB::string(attempt, "status")? == "running", + None => false, + } + } + }; + let Some(expires) = expires.filter(|_| original && authorized) else { + return Ok(Skipped(match loss { + LeaseLoss::Expired => "original_lease_not_expired", + LeaseLoss::Revoked { .. } => "lease_owner_mismatch", + })); + }; + let micros = |at: DateTime| at.to_rfc3339_opts(chrono::SecondsFormat::Micros, true); + let revoked = matches!(loss, LeaseLoss::Revoked { .. }) && at < expires; + let mut lease = json!({"failure_reason": if revoked { "worker_deregistered" } else { "lease_expired" }, + "lease_expires_at": micros(expires), "lease_owner": owner}); + if revoked { + lease["lease_revoked_at"] = json!(micros(at)); + } + // An expired deadline settles as that timeout, without lease metadata. + let retried = match deadline { + Some(kind) => Self::time_out_activity(tx, &run, &activity, kind, None).await?, + None => Self::time_out_activity(tx, &run, &activity, "lease", Some(&lease)).await?, + }; + Ok(Enforcement::Enforced { retried }) + } + + /// `ActivityTaskLeaseRecovery::recoverForStoppedWorker`: a deregistered + /// worker's activity leases are settled at once, through each activity's + /// retry policy, inside the deregistration's transaction. + pub(super) async fn revoke_worker_activity_leases( + tx: &mut Transaction<'_, DB>, + namespace: &str, + worker_id: &str, + ) -> Result<()> { + let tasks = Self::query("SELECT id FROM workflow_tasks WHERE namespace=$1 AND task_type='activity' AND status='leased' AND lease_owner=$2 ORDER BY id") + .bind(namespace) + .bind(worker_id) + .fetch_all(&mut **tx) + .await?; + for task in tasks { + let task_id = DB::string(&task, "id")?; + Self::settle_lost_lease(tx, &task_id, LeaseLoss::Revoked { namespace, owner: worker_id }).await?; + } + Ok(()) + } + /// `currentAttemptStateReason`: a running execution's current attempt /// must still be the one it records, and running. async fn attempt_state_reason( @@ -281,12 +447,15 @@ where } /// Fails the expired attempt, then schedules a retry or records the - /// timeout. Returns whether it retried. + /// timeout. A lost lease (`lease` carries its metadata) closes the + /// attempt as expired and settles as PHP's `ActivityLeaseExpiredException` + /// instead. Returns whether it retried. async fn time_out_activity( tx: &mut Transaction<'_, DB>, run: &DB::Row, activity: &DB::Row, kind: &'static str, + lease: Option<&Value>, ) -> Result { let activity_id = &DB::string(activity, "id")?; let run_id = DB::string(activity, "workflow_run_id")?; @@ -295,7 +464,13 @@ where let count = DB::number(activity, "attempt_count")?.max(1); let max = policy["max_attempts"].as_i64().unwrap_or(1); let current = DB::optional_string(activity, "current_attempt_id")?; - let retried = count < max && kind != "schedule_to_close"; + // `isNonRetryableFailure` for the lease exception: its class is the + // only class or type it reports. + let non_retryable = lease.is_some() + && policy["non_retryable_error_types"] + .as_array() + .is_some_and(|types| types.contains(&json!(LEASE_EXCEPTION))); + let retried = count < max && kind != "schedule_to_close" && !non_retryable; let task = Self::query(DB::ACTIVITY_OPEN_TASK_SQL) .bind(&run_id) .bind(activity_id) @@ -307,14 +482,30 @@ where .transpose()?; let at = now(); if let Some(attempt_id) = current.as_deref() { - Self::query("UPDATE activity_attempts SET status='failed',lease_expires_at=NULL,closed_at=$1 WHERE id=$2 AND status='running'") - .bind(DB::bind_time(at)).bind(attempt_id).execute(&mut **tx).await?; + let status = if lease.is_some() { "expired" } else { "failed" }; + Self::query("UPDATE activity_attempts SET status=$1,lease_expires_at=NULL,closed_at=COALESCE(closed_at,$2) WHERE id=$3 AND status='running'") + .bind(status).bind(DB::bind_time(at)).bind(attempt_id).execute(&mut **tx).await?; } if let Some(task_id) = task_id.as_deref() { - Self::query("UPDATE workflow_tasks SET status='cancelled',lease_expires_at=NULL,last_error=NULL WHERE id=$1") - .bind(task_id).execute(&mut **tx).await?; + // A lost lease keeps its reason on the closed task, as PHP does. + let error = lease.and_then(|lease| lease["failure_reason"].as_str()); + Self::query("UPDATE workflow_tasks SET status='cancelled',lease_expires_at=NULL,last_error=$1 WHERE id=$2") + .bind(error).bind(task_id).execute(&mut **tx).await?; } let message = timeout_message::(activity, kind)?; + // A lease loss names itself by its metadata and retry reason, not a + // timeout kind. + let lease_fields = |payload: &mut Value, retry_reason: bool| { + if let (Some(fields), Some(Value::Object(lease))) = (payload.as_object_mut(), lease) { + fields.remove("timeout_kind"); + for (key, value) in lease { + fields.insert(key.clone(), value.clone()); + } + if retry_reason { + fields.insert("retry_reason".into(), lease["failure_reason"].clone()); + } + } + }; if retried { let backoff = policy["backoff_seconds"] .as_array() @@ -328,7 +519,9 @@ where })?; Self::query("UPDATE activity_executions SET status='pending',last_heartbeat_at=NULL,close_deadline_at=NULL,heartbeat_deadline_at=NULL,schedule_deadline_at=$1 WHERE id=$2") .bind(DB::optional_time(deadline(&policy,"schedule_to_start_timeout",available)?)).bind(activity_id).execute(&mut **tx).await?; - let retry = Self::create_task(tx,&run_id,&DB::string(&activity,"queue")?,"activity",json!({"activity_execution_id":activity_id,"retry_of_task_id":task_id,"retry_after_attempt_id":current,"retry_after_attempt":count,"retry_backoff_seconds":backoff,"max_attempts":max,"retry_policy":policy,"timeout_kind":kind})).await?; + let mut retry_payload = json!({"activity_execution_id":activity_id,"retry_of_task_id":task_id,"retry_after_attempt_id":current,"retry_after_attempt":count,"retry_backoff_seconds":backoff,"max_attempts":max,"retry_policy":policy,"timeout_kind":kind}); + lease_fields(&mut retry_payload, true); + let retry = Self::create_task(tx,&run_id,&DB::string(&activity,"queue")?,"activity",retry_payload).await?; Self::query( "UPDATE workflow_tasks SET available_at=$1,attempt_count=$2 WHERE id=$3", ) @@ -343,6 +536,7 @@ where .await?; let mut payload = Self::activity_payload(&updated)?; for (key,value) in json!({"activity_attempt_id":current,"retry_task_id":retry,"retry_of_task_id":task_id,"retry_available_at":available,"retry_backoff_seconds":backoff,"retry_after_attempt_id":current,"retry_after_attempt":count,"max_attempts":max,"retry_policy":policy,"timeout_kind":kind,"message":message}).as_object().unwrap() { if !value.is_null() {payload[key]=value.clone();} } + lease_fields(&mut payload, true); Self::append( tx, &run_id, @@ -354,7 +548,13 @@ where .await?; } else { let failure_id = id(); - const CLASS: &str = "Workflow\\V2\\Exceptions\\ActivityTimeoutException"; + // A lost lease is an internal activity failure; a deadline is a + // timeout. + let (class, category, propagation, event) = if lease.is_some() { + (LEASE_EXCEPTION, "internal", "activity", "ActivityFailed") + } else { + ("Workflow\\V2\\Exceptions\\ActivityTimeoutException", "timeout", "timeout", "ActivityTimedOut") + }; Self::query( "UPDATE activity_executions SET status='failed',closed_at=$1 WHERE id=$2", ) @@ -362,29 +562,35 @@ where .bind(activity_id) .execute(&mut **tx) .await?; - Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,exception_class,message,non_retryable,handled,file,line,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,'timeout','timeout',$4,$5,FALSE,FALSE,'',0,$6,$7)") - .bind(&failure_id).bind(&run_id).bind(activity_id).bind(CLASS).bind(&message).bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,exception_class,message,non_retryable,handled,file,line,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,$4,$5,$6,$7,CASE WHEN $8='true' THEN TRUE ELSE FALSE END,FALSE,'',0,$9,$10)") + .bind(&failure_id).bind(&run_id).bind(activity_id).bind(propagation).bind(category).bind(class).bind(&message) + .bind(if non_retryable { "true" } else { "false" }).bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; let updated = Self::query("SELECT * FROM activity_executions WHERE id=$1") .bind(activity_id) .fetch_one(&mut **tx) .await?; let mut payload = Self::activity_payload(&updated)?; - for (key,value) in json!({"activity_attempt_id":current,"attempt_number":count,"failure_id":failure_id,"failure_category":"timeout","timeout_kind":kind,"message":message,"exception_class":CLASS}).as_object().unwrap() {payload[key]=value.clone();} - for field in [ - "schedule_deadline_at", - "close_deadline_at", - "schedule_to_close_deadline_at", - "heartbeat_deadline_at", - ] { - payload[field] = json!( - DB::optional_instant(&activity, field)? - .map(|at| at.to_rfc3339_opts(chrono::SecondsFormat::Secs, false)) - ); + for (key,value) in json!({"activity_attempt_id":current,"attempt_number":count,"failure_id":failure_id,"failure_category":category,"timeout_kind":kind,"message":message,"exception_class":class}).as_object().unwrap() {payload[key]=value.clone();} + if lease.is_some() { + lease_fields(&mut payload, false); + payload["non_retryable"] = json!(non_retryable); + } else { + for field in [ + "schedule_deadline_at", + "close_deadline_at", + "schedule_to_close_deadline_at", + "heartbeat_deadline_at", + ] { + payload[field] = json!( + DB::optional_instant(&activity, field)? + .map(|at| at.to_rfc3339_opts(chrono::SecondsFormat::Secs, false)) + ); + } } Self::append( tx, &run_id, - "ActivityTimedOut", + event, payload, task_id.as_deref(), None, @@ -400,18 +606,26 @@ where &metadata, "activity", activity_id, - "ActivityTimedOut", - ) - .await?; - Self::selection_resume( - tx, - &run_id, - &DB::string(&run, "queue")?, - json!({"activity_execution_id":activity_id}), - &metadata, - false, + event, ) .await?; + // A standalone activity's host run fails with it: as failed for a + // lost lease, otherwise as timed out. + let host = super::standalone_activities::HostClose::Failed { + payload: super::standalone_activities::timed_out_payload(&failure_id, activity_id, category, non_retryable, class, &message), + closed_reason: if lease.is_some() { "failed" } else { "timed_out" }, + }; + if !Self::close_standalone_host(tx, &run_id, host).await? { + Self::selection_resume( + tx, + &run_id, + &DB::string(&run, "queue")?, + json!({"activity_execution_id":activity_id}), + &metadata, + false, + ) + .await?; + } } Ok(retried) } diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 1dca08ace..062f7720a 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -302,6 +302,33 @@ pub(super) fn snapshot_failure(stored: &Value) -> Value { snapshot } +/// `ActivityOutcomeRecorder`: a result over the payload ceiling fails its +/// activity with PHP's structural-limit exception instead. +pub(super) async fn structural_failure( + exceeded: &super::memo_limits::Exceeded, + semaphore: Arc, +) -> Result<(Value, String)> { + let failure = json!({"class":"Workflow\\V2\\Exceptions\\StructuralLimitExceededException", + "message":exceeded.message,"code":0}); + let stored = failure.clone(); + let permit = semaphore + .acquire_owned() + .await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "failure_codec_unavailable"))?; + let blob = tokio::task::spawn_blocking(move || { + let _permit = permit; + super::memos::from_json(stored) + .and_then(|value| { + crate::codec::ValueCodec::new() + .and_then(|codec| codec.encode(&value)) + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "failure_codec_unavailable")) + }) + }) + .await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "failure_codec_unavailable"))??; + Ok((failure, blob)) +} + /// Validates a reported failure like published PHP and returns its /// normalized form with the Avro blob stored on the execution. pub(super) async fn prepare_failure( @@ -374,7 +401,10 @@ where body: Value, failure: Value, blob: String, + limit: Option, ) -> Result { + // An oversized result is still answered as the completion it was. + let reported = if limit.is_some() { "completed" } else { "failed" }; let attempt_id = text(&body, "activity_attempt_id")?; let mut tx = self.begin().await?; let task = self.task_row(&mut tx, task_id).await?; @@ -384,7 +414,7 @@ where Self::cancelled_activity_outcome(&mut tx, &task, &body).await?; if Self::duplicate_receipt(&task, &body)? { return Ok( - json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":"failed","recorded":false,"reason":"already_completed"}), + json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":reported,"recorded":false,"reason":"already_completed"}), ); } let attempt = @@ -399,11 +429,11 @@ where if DB::string(&attempt, "status")? != "running" || count != DB::number(&task, "attempt_count")? { return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, - response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":"failed", + response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":reported, "recorded":false,"reason":"stale_attempt","next_task_id":null}), }); } - Self::closed_run_outcome(&mut tx, &task, &body, "failed").await?; + Self::closed_run_outcome(&mut tx, &task, &body, reported).await?; Self::fence(&task, &body)?; let run_id = DB::string(&task, "workflow_run_id")?; Self::active_run(&mut tx, &run_id).await?; @@ -445,7 +475,8 @@ where // report's flag in its payload; the derived terminal classification // also accounts for the immutable execution policy. // `isNonRetryableFailure` matches the reported class or type. - let non_retryable = failure["non_retryable"] == true + let non_retryable = limit.is_some() + || failure["non_retryable"] == true || policy["non_retryable_error_types"].as_array().is_some_and(|types| { types.contains(&failure["class"]) || types.contains(&failure["type"]) }); @@ -480,9 +511,10 @@ where Self::query("UPDATE activity_executions SET status='failed',exception=$1,last_heartbeat_at=NULL,closed_at=$2 WHERE id=$3") .bind(blob).bind(DB::bind_time(closed_at)).bind(&activity_id).execute(&mut *tx).await?; let failure_id = super::store::id(); - Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,exception_class,message,non_retryable,handled,file,line,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,'activity','activity',$4,$5,CASE WHEN $6='true' THEN TRUE ELSE FALSE END,FALSE,'external-worker',0,$7,$8)") - .bind(&failure_id).bind(&run_id).bind(&activity_id).bind(failure["class"].as_str().unwrap_or("RuntimeException")) - .bind(failure["message"].as_str().unwrap()).bind(if non_retryable {"true"} else {"false"}) + let (category, file) = if limit.is_some() { ("structural_limit", "") } else { ("activity", "external-worker") }; + Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,exception_class,message,non_retryable,handled,file,line,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,'activity',$4,$5,$6,CASE WHEN $7='true' THEN TRUE ELSE FALSE END,FALSE,$8,0,$9,$10)") + .bind(&failure_id).bind(&run_id).bind(&activity_id).bind(category).bind(failure["class"].as_str().unwrap_or("RuntimeException")) + .bind(failure["message"].as_str().unwrap()).bind(if non_retryable {"true"} else {"false"}).bind(file) .bind(DB::bind_time(closed_at)).bind(DB::bind_time(closed_at)).execute(&mut *tx).await?; let updated = Self::query("SELECT * FROM activity_executions WHERE id=$1") .bind(&activity_id) @@ -492,7 +524,7 @@ where payload["activity_attempt_id"] = json!(attempt_id); payload["attempt_number"] = json!(count); payload["failure_id"] = json!(failure_id); - payload["failure_category"] = json!("activity"); + payload["failure_category"] = json!(category); payload["non_retryable"] = json!(non_retryable); payload["exception_type"] = failure["type"].clone(); payload["exception_class"] = failure["class"].clone(); @@ -500,9 +532,16 @@ where payload["code"] = failure["code"].clone(); payload["exception"] = presented_failure(&failure); payload["activity"]["exception"] = snapshot_failure(&failure); + if let Some(limit) = &limit { + // PHP's own exception payload, with the limit beside it. + payload["exception"] = json!({"message":limit.message,"code":0}); + payload["activity"]["exception"] = payload["exception"].clone(); + payload.as_object_mut().unwrap().extend(limit.fields().as_object().unwrap().clone()); + } payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":count}); + let failed_event = payload.clone(); Self::append( &mut tx, &run_id, @@ -513,19 +552,27 @@ where ) .await?; let run = Self::active_run(&mut tx, &run_id).await?; - let next = Self::selection_resume( - &mut tx, - &run_id, - &DB::string(&run, "queue")?, - json!({"activity_execution_id":activity_id}), - &metadata, - false, - ) - .await?; + let host = super::standalone_activities::HostClose::Failed { + payload: super::standalone_activities::failed_payload(&failed_event, &activity_id), + closed_reason: "failed", + }; + let next = if Self::close_standalone_host(&mut tx, &run_id, host).await? { + None + } else { + Self::selection_resume( + &mut tx, + &run_id, + &DB::string(&run, "queue")?, + json!({"activity_execution_id":activity_id}), + &metadata, + false, + ) + .await? + }; tx.commit().await?; self.wake.notify_waiters(); return Ok( - json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":"failed","recorded":true,"reason":null,"next_task_id":next}), + json!({"task_id":task_id,"activity_attempt_id":attempt_id,"outcome":reported,"recorded":true,"reason":null,"next_task_id":next}), ); } Self::query("UPDATE activity_executions SET status='pending',exception=$1,last_heartbeat_at=NULL,close_deadline_at=NULL,heartbeat_deadline_at=NULL,schedule_deadline_at=$2 WHERE id=$3") diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 9487901a7..51c16efc6 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -192,12 +192,14 @@ where } => refuse(StatusCode::CONFLICT, "stale_attempt"), other => other, })?; - if ordinary && observed["can_continue"] == false && observed["cancel_requested"] == false { - // Published PHP returns a structured refusal for ordinary expired - // or closed authority. This read never records progress or renews - // a lease; a concurrently closed attempt remains fenced. + // Published PHP answers with this structured refusal for ordinary + // expired authority, and for any closed run, whose activity is told + // it was cancelled. This read never records progress or renews a + // lease; a concurrently closed attempt remains fenced. + let closed = !observed["run_closed_reason"].is_null(); + if observed["can_continue"] == false && (closed || ordinary && observed["cancel_requested"] == false) { let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], - "cancel_requested":false,"can_continue":false,"reason":observed["reason"], + "cancel_requested":observed["cancel_requested"],"can_continue":false,"reason":observed["reason"], "run_closed_reason":observed["run_closed_reason"],"run_closed_at":observed["run_closed_at"], "heartbeat_recorded":false,"lease_expires_at":observed["lease_expires_at"], "last_heartbeat_at":observed["last_heartbeat_at"],"worker_session":observed["worker_session"], @@ -298,7 +300,9 @@ where tx.commit().await?; Ok( json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], - "heartbeat_recorded":true,"last_heartbeat_at":at,"lease_expires_at":expires,"cancel_requested":false,"can_continue":true}), + "cancel_requested":false,"can_continue":true,"reason":null,"run_closed_reason":null,"run_closed_at":null, + "heartbeat_recorded":true,"lease_expires_at":expires,"last_heartbeat_at":at,"worker_session":null, + "protocol_version":protocol,"server_capabilities":super::store::capabilities()}), ) } } diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs index 83890c97d..ac1a84f9e 100644 --- a/rust/src/runtime/activity_lease_renewal.rs +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -128,6 +128,35 @@ store_methods! { Ok(status) } + /// `RuntimePayloadCompletionLease::mayResume` for an activity: its task + /// is still leased to the owner and its attempt can continue on that task + /// for that owner. An expired lease alone does not end it, since the + /// activity may still renew before lease recovery. + pub(super) async fn activity_completion_may_resume( + &self, + tx: &mut Transaction<'_, DB>, + task_id: &str, + attempt_id: &str, + owner: &str, + ) -> Result { + let Some(task) = Self::query("SELECT status,lease_owner FROM workflow_tasks WHERE id=$1 AND namespace=$2") + .bind(task_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(false); + }; + if DB::string(&task, "status")? != "leased" || DB::optional_string(&task, "lease_owner")?.as_deref() != Some(owner) { + return Ok(false); + } + let Some(rows) = Self::attempt_rows(tx, attempt_id).await? else { + return Ok(false); + }; + let status = Self::attempt_status(attempt_id, &rows, None)?; + Ok(status["can_continue"] == true && status["workflow_task_id"] == task_id && status["lease_owner"] == owner) + } + async fn attempt_rows(tx: &mut Transaction<'_, DB>, attempt_id: &str) -> Result>> { let Some(attempt) = Self::query("SELECT * FROM activity_attempts WHERE id=$1") .bind(attempt_id) diff --git a/rust/src/runtime/activity_metadata.rs b/rust/src/runtime/activity_metadata.rs index b0ca65adc..5b9a53291 100644 --- a/rust/src/runtime/activity_metadata.rs +++ b/rust/src/runtime/activity_metadata.rs @@ -639,7 +639,7 @@ pub(super) fn project(events: &[Value], executions: &[Value]) -> Value { let mut presented: Vec<_> = attempts .get(&id) .into_iter() - .flat_map(|a| a.values()) + .flat_map(BTreeMap::values) .map(|a| present_attempt(a, execution)) .collect(); if let Some(synthetic) = synthetic_attempt(&state, row) diff --git a/rust/src/runtime/archive.rs b/rust/src/runtime/archive.rs new file mode 100644 index 000000000..2ee1147f4 --- /dev/null +++ b/rust/src/runtime/archive.rs @@ -0,0 +1,125 @@ +//! Archiving a closed run, as published PHP's `WorkflowStub::attemptArchive` +//! does: an archive command, `ArchiveRequested`, and on the first archive the +//! run's archive columns and `WorkflowArchived`. Nothing is deleted. +use super::laravel_rules::{self, Rule}; +use super::{Result, RuntimeError, backend::Backend, refuse, store::{Store, id, now}}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +/// `WorkflowController::archive`'s rules. +const RULES: &laravel_rules::Rules = &[ + ("reason", &[Rule::Nullable, Rule::String, Rule::Max(1000)]), + ("request_id", &[Rule::Nullable, Rule::String, Rule::Max(255)]), +]; + +/// The validated reason, if any. Laravel trims it and turns an empty one +/// into null first. +pub(super) fn reason(body: &Value, workflow_id: &str) -> Result> { + let body = laravel_rules::normalize(body); + if let Err(RuntimeError::Protocol { status, mut response }) = laravel_rules::validate(&body, RULES) { + // PHP names the workflow on this validation failure. + response["workflow_id"] = json!(workflow_id); + return Err(RuntimeError::Protocol { status, response }); + } + Ok(body["reason"].as_str().map(str::to_owned)) +} + +/// `archiveCommandPayloadAttributes`: the command payload `{reason}`. +pub(super) fn payload(reason: Option<&str>) -> Result { + use crate::codec::{Value as Datum, ValueCodec}; + let reason = reason.map_or(Datum::Null, |reason| Datum::String(reason.to_owned())); + ValueCodec::new() + .and_then(|codec| codec.encode(&Datum::Map(std::collections::BTreeMap::from([("reason".into(), reason)])))) + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "archive_codec_unavailable")) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + pub(crate) async fn archive_workflow( + &self, + workflow_id: &str, + selected_run: Option<&str>, + reason: Option, + blob: String, + context: Value, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let run = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1 AND i.namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? + .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; + let run_id = DB::string(&run, "id")?; + if selected_run.is_some_and(|id| id != run_id) { + return Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"message":"Commands cannot target historical runs. Use the instance-level endpoint to command the current run, or omit the run ID.", + "workflow_id":workflow_id,"run_id":selected_run,"reason":"historical_run_command_rejected","target_scope":"run"}), + }); + } + let open = matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting"); + let archived = DB::optional_string(&run, "archive_command_id")?.is_some() + || DB::optional_instant(&run, "archived_at")?.is_some(); + let command_id = id(); + let sequence = DB::number(&run, "last_command_sequence")? + .checked_add(1) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "command_sequence_exhausted"))?; + let at = now(); + let (status, outcome, rejection) = if open { + ("rejected", "rejected_run_not_closed", Some("run_not_closed")) + } else if archived { + ("accepted", "archive_not_needed", None) + } else { + ("accepted", "archived", None) + }; + let decided = |when: bool| DB::optional_time(when.then_some(at)); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,target_scope,source,status,outcome,rejection_reason,workflow_type,payload_codec,payload,command_sequence,accepted_at,applied_at,rejected_at,context) VALUES ($1,$2,$3,$4,'archive','instance','control_plane',$5,$6,$7,$8,'avro',$9,$10,$11,$12,$13,$14)") + .bind(&command_id).bind(workflow_id).bind(&run_id).bind(&run_id).bind(status).bind(outcome).bind(rejection) + .bind(DB::string(&run, "workflow_type")?).bind(&blob).bind(sequence) + .bind(decided(!open)).bind(decided(!open)).bind(decided(open)).bind(DB::document(&context)) + .execute(&mut *tx).await?; + Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") + .bind(sequence) + .bind(&run_id) + .execute(&mut *tx) + .await?; + let mut response = json!({"outcome":outcome,"workflow_id":workflow_id,"run_id":run_id,"requested_run_id":null, + "resolved_run_id":run_id,"command_id":command_id,"command_sequence":sequence,"target_scope":"instance", + "workflow_type":DB::string(&run, "workflow_type")?,"command_status":status,"command_source":"control_plane", + "reason":if open { json!(rejection) } else { json!(reason) },"rejection_reason":rejection,"validation_errors":[]}); + if open { + response["accepted"] = json!(false); + tx.commit().await?; + return Ok((StatusCode::CONFLICT, response)); + } + let mut requested = json!({"workflow_command_id":command_id,"workflow_instance_id":workflow_id, + "workflow_run_id":run_id,"command_type":"archive","outcome":outcome}); + if let Some(reason) = &reason { + requested["reason"] = json!(reason); + } + Self::append(&mut tx, &run_id, "ArchiveRequested", requested, None, Some(&command_id)).await?; + if !archived { + Self::query("UPDATE workflow_runs SET archived_at=$1,archive_command_id=$2,archive_reason=$3,last_progress_at=$4 WHERE id=$5") + .bind(DB::bind_time(at)).bind(&command_id).bind(reason.as_deref()).bind(DB::bind_time(at)).bind(&run_id) + .execute(&mut *tx).await?; + let mut event = json!({"workflow_command_id":command_id,"workflow_instance_id":workflow_id, + "workflow_run_id":run_id,"archive_command_id":command_id}); + if let Some(reason) = &reason { + event["reason"] = json!(reason); + } + Self::append(&mut tx, &run_id, "WorkflowArchived", event, None, Some(&command_id)).await?; + } + tx.summary_changed(&run_id); + tx.commit().await?; + Ok((StatusCode::OK, response)) + } +} diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 481461fc9..722cd8f75 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -14,57 +14,57 @@ pub(super) trait Backend: Database { type EncodedInteger: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; type EncodedFloat: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; type EncodedFlag: for<'q> Encode<'q, Self> + Type + Send + Sync + 'static; - const HISTORY_PAGE: &'static str; + const HISTORY_PAGE: &str; // Multi-statement reads that must observe one committed state. SQLite's // deferred read transaction already keeps one WAL snapshot. - const READ_SNAPSHOT_BEGIN: &'static str = "BEGIN"; - const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT sequence,event_type,CASE WHEN projection_bytes<=8388608 THEN payload ELSE NULL END AS payload,workflow_task_id,recorded_at,projection_count FROM (SELECT sequence,event_type,payload,workflow_task_id,recorded_at,COUNT(*) OVER () AS projection_count,SUM(octet_length(CAST(payload AS TEXT))) OVER (ORDER BY sequence) AS projection_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') ORDER BY sequence LIMIT 10001) AS page ORDER BY sequence"; + const READ_SNAPSHOT_BEGIN: &str = "BEGIN"; + const ACTIVITY_METADATA_HISTORY: &str = "SELECT sequence,event_type,CASE WHEN projection_bytes<=8388608 THEN payload ELSE NULL END AS payload,workflow_task_id,recorded_at,projection_count FROM (SELECT sequence,event_type,payload,workflow_task_id,recorded_at,COUNT(*) OVER () AS projection_count,SUM(octet_length(CAST(payload AS TEXT))) OVER (ORDER BY sequence) AS projection_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') ORDER BY sequence LIMIT 10001) AS page ORDER BY sequence"; const OPEN_WAIT_ORDER: &str = "WITH clocks AS (SELECT id,wait_id,opened_at,EXTRACT(EPOCH FROM opened_at) AS seconds FROM workflow_run_waits WHERE workflow_run_id=$1 AND status='open') SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END,ROUND((CASE WHEN seconds<0 THEN 2*FLOOR(seconds)-seconds ELSE seconds END)*1000),wait_id LIMIT 100 OFFSET $2"; const SUMMARY_HISTORY_SQL: &str = "SELECT id,sequence,event_type,CAST(payload AS TEXT) AS payload_json FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT 16"; const OBSERVATION_FAILURE_EVENT: &str = "SELECT id,sequence,event_type,recorded_at FROM workflow_history_events WHERE workflow_run_id=$1 AND payload->>'failure_id'=$2 AND event_type IN ('ActivityFailed','ActivityTimedOut','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','WorkflowFailed','WorkflowTimedOut','WorkflowCancelled','WorkflowTerminated','UpdateCompleted') ORDER BY sequence DESC LIMIT 1"; const OBSERVATION_ATTRIBUTES: &str = "SELECT \"key\" AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 AND workflow_instance_id=$2 AND \"key\" IN ($3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22) ORDER BY \"key\" LIMIT 20"; - const ACTIVITY_RESULT_EVENT_SQL: &'static str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (payload->>'activity_execution_id'=$2 OR payload->'activity'->>'id'=$3) ORDER BY sequence DESC LIMIT 1"; - const EXTERNAL_PAYLOAD_BYTES: &'static str = "SELECT CAST(COALESCE(SUM(size_bytes),0) AS BIGINT) FROM runtime_external_payloads WHERE namespace=$1"; - const VISIBILITY_PAGE: &'static str = super::visibility::VISIBILITY_PAGE; - const WORKER_REGISTRATION_SQL: &'static str = super::store::WORKER_REGISTRATION_SQL; - const TASK_CANDIDATES_SQL: &'static str = super::store::TASK_CANDIDATES_SQL; - const POLL_WAKE_TASKS_SQL: &'static str = super::cross_kind_poll::TASKS_SQL; - const POLL_WAKE_QUERIES_SQL: &'static str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND CAST(value AS jsonb)->>'namespace'=$2 AND CAST(value AS jsonb)->>'task_queue'=$3 AND CAST(value AS jsonb)->>'query_deadline_at'>$4 AND (CAST(value AS jsonb)->>'status'='pending' OR (CAST(value AS jsonb)->>'status'='leased' AND CAST(value AS jsonb)->>'lease_expires_at'<=$5)) ORDER BY \"key\" LIMIT 65"; - const POLL_RECEIPT_CLEANUP_SQL: &'static str = super::store::POLL_RECEIPT_CLEANUP_SQL; - const TIMER_TASK_SQL: &'static str; + const ACTIVITY_RESULT_EVENT_SQL: &str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (payload->>'activity_execution_id'=$2 OR payload->'activity'->>'id'=$3) ORDER BY sequence DESC LIMIT 1"; + const EXTERNAL_PAYLOAD_BYTES: &str = "SELECT CAST(COALESCE(SUM(size_bytes),0) AS BIGINT) FROM runtime_external_payloads WHERE namespace=$1"; + const VISIBILITY_PAGE: &str = super::visibility::VISIBILITY_PAGE; + const WORKER_REGISTRATION_SQL: &str = super::store::WORKER_REGISTRATION_SQL; + const TASK_CANDIDATES_SQL: &str = super::store::TASK_CANDIDATES_SQL; + const POLL_WAKE_TASKS_SQL: &str = super::cross_kind_poll::TASKS_SQL; + const POLL_WAKE_QUERIES_SQL: &str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND CAST(value AS jsonb)->>'namespace'=$2 AND CAST(value AS jsonb)->>'task_queue'=$3 AND CAST(value AS jsonb)->>'query_deadline_at'>$4 AND (CAST(value AS jsonb)->>'status'='pending' OR (CAST(value AS jsonb)->>'status'='leased' AND CAST(value AS jsonb)->>'lease_expires_at'<=$5)) ORDER BY \"key\" LIMIT 65"; + const POLL_RECEIPT_CLEANUP_SQL: &str = super::store::POLL_RECEIPT_CLEANUP_SQL; + const TIMER_TASK_SQL: &str; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; /// The history sequence of a signal's `SignalReceived` event. const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; - const SELECTION_RESOLUTION_SQL: &'static str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; + const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; - const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND payload->>'replay_blocked'='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; - const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND payload->>'activity_execution_id'=$2 ORDER BY id LIMIT 1"; + const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND payload->>'replay_blocked'='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; + const ACTIVITY_OPEN_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND payload->>'activity_execution_id'=$2 ORDER BY id LIMIT 1"; // Local activities in the namespace, as PHP's operator metrics count them. - const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND activity_options->>'execution_mode'='local'"; - const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND a.activity_options->>'execution_mode'='local'"; - const QUERY_CACHE_SCAN: &'static str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND value::jsonb->>'namespace'=$1 ORDER BY \"key\" LIMIT 65"; - const QUERY_CACHE_PENDING_FOR_RUN: &'static str = "SELECT value::jsonb->>'query_admitted_at' AS query_admitted_at,value::jsonb->>'query_deadline_at' AS query_deadline_at,value::jsonb->>'receipt_expires_at' AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND value::jsonb->>'run_id'=$1 AND value::jsonb->>'status' IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; - const QUERY_CACHE_GET: &'static str = + const LOCAL_ACTIVITY_EXECUTIONS: &str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND activity_options->>'execution_mode'='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND a.activity_options->>'execution_mode'='local'"; + const QUERY_CACHE_SCAN: &str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND value::jsonb->>'namespace'=$1 ORDER BY \"key\" LIMIT 65"; + const QUERY_CACHE_PENDING_FOR_RUN: &str = "SELECT value::jsonb->>'query_admitted_at' AS query_admitted_at,value::jsonb->>'query_deadline_at' AS query_deadline_at,value::jsonb->>'receipt_expires_at' AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND value::jsonb->>'run_id'=$1 AND value::jsonb->>'status' IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; + const QUERY_CACHE_GET: &str = "SELECT value,expiration FROM dw_query_cache WHERE \"key\"=$1"; - const QUERY_CACHE_INSERT: &'static str = + const QUERY_CACHE_INSERT: &str = "INSERT INTO dw_query_cache(\"key\",value,expiration) VALUES ($1,$2,$3)"; - const QUERY_CACHE_UPDATE: &'static str = + const QUERY_CACHE_UPDATE: &str = "UPDATE dw_query_cache SET value=$1,expiration=$2 WHERE \"key\"=$3"; - const QUERY_CACHE_DELETE: &'static str = "DELETE FROM dw_query_cache WHERE \"key\"=$1"; + const QUERY_CACHE_DELETE: &str = "DELETE FROM dw_query_cache WHERE \"key\"=$1"; // A bounded batch of expired per-request query poll rows. - const QUERY_POLL_EXPIRED: &'static str = "SELECT \"key\" FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY \"key\" LIMIT 32"; - const MEMO_INITIAL_INSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,0,false,$5,$6,$7,0)"; - const MEMO_INHERIT_INSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,1,true,$5,$6,$7,1)"; - const MEMO_READ: &'static str = "SELECT \"key\",value,portable_value,portable_value_sequence,upserted_at_sequence FROM workflow_memos WHERE workflow_run_id=$1 ORDER BY \"key\",id LIMIT 101"; - const MEMO_UPSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,$5,false,$6,$7,$8,$9) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET value=excluded.value,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at,portable_value=excluded.portable_value,portable_value_sequence=excluded.portable_value_sequence"; - const MEMO_DELETE: &'static str = + const QUERY_POLL_EXPIRED: &str = "SELECT \"key\" FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY \"key\" LIMIT 32"; + const MEMO_INITIAL_INSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,0,false,$5,$6,$7,0)"; + const MEMO_INHERIT_INSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,1,true,$5,$6,$7,1)"; + const MEMO_READ: &str = "SELECT \"key\",value,portable_value,portable_value_sequence,upserted_at_sequence FROM workflow_memos WHERE workflow_run_id=$1 ORDER BY \"key\",id LIMIT 101"; + const MEMO_UPSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,\"key\",value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,$5,false,$6,$7,$8,$9) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET value=excluded.value,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at,portable_value=excluded.portable_value,portable_value_sequence=excluded.portable_value_sequence"; + const MEMO_DELETE: &str = "DELETE FROM workflow_memos WHERE workflow_run_id=$1 AND \"key\"=$2"; - const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT \"key\" AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY \"key\" LIMIT 1001"; - const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET workflow_instance_id=excluded.workflow_instance_id,type=excluded.type,value_string=excluded.value_string,value_keyword=excluded.value_keyword,value_keyword_list=excluded.value_keyword_list,value_int=excluded.value_int,value_float=excluded.value_float,value_bool=excluded.value_bool,value_datetime=excluded.value_datetime,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at"; - const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; - const SEARCH_ATTRIBUTE_KEY_ROWS: &'static str = "SELECT a.workflow_run_id,a.\"key\" AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.\"key\"=$2"; - const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + const SEARCH_ATTRIBUTE_ROWS: &str = "SELECT \"key\" AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY \"key\" LIMIT 1001"; + const SEARCH_ATTRIBUTE_UPSERT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON CONFLICT(workflow_run_id,\"key\") DO UPDATE SET workflow_instance_id=excluded.workflow_instance_id,type=excluded.type,value_string=excluded.value_string,value_keyword=excluded.value_keyword,value_keyword_list=excluded.value_keyword_list,value_int=excluded.value_int,value_float=excluded.value_float,value_bool=excluded.value_bool,value_datetime=excluded.value_datetime,upserted_at_sequence=excluded.upserted_at_sequence,inherited_from_parent=false,updated_at=excluded.updated_at"; + const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; + const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.\"key\" AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.\"key\"=$2"; + const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; fn statement(sql: &'static str) -> Cow<'static, str> { Cow::Borrowed(sql) } @@ -128,7 +128,7 @@ impl Backend for Sqlite { fn optional_flag(value: Option) -> Option { value } - const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT sequence,event_type,CASE WHEN projection_bytes<=8388608 THEN payload ELSE NULL END AS payload,workflow_task_id,recorded_at,projection_count FROM (SELECT sequence,event_type,payload,workflow_task_id,recorded_at,COUNT(*) OVER () AS projection_count,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS projection_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') ORDER BY sequence LIMIT 10001) AS page ORDER BY sequence"; + const ACTIVITY_METADATA_HISTORY: &str = "SELECT sequence,event_type,CASE WHEN projection_bytes<=8388608 THEN payload ELSE NULL END AS payload,workflow_task_id,recorded_at,projection_count FROM (SELECT sequence,event_type,payload,workflow_task_id,recorded_at,COUNT(*) OVER () AS projection_count,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS projection_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') ORDER BY sequence LIMIT 10001) AS page ORDER BY sequence"; const PHP_CONNECTION: &str = "sqlite"; // Carbon's millisecond comparison rounds at half a millisecond. Parse only // whole UTC seconds with SQLite's clock functions and round integer @@ -150,11 +150,11 @@ SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END, CASE WHEN micros<0 THEN (micros-500)/1000 ELSE (micros+500)/1000 END,wait_id LIMIT 100 OFFSET $2 "#; const OBSERVATION_FAILURE_EVENT: &str = "SELECT id,sequence,event_type,recorded_at FROM workflow_history_events WHERE workflow_run_id=$1 AND json_extract(payload,'$.failure_id') COLLATE BINARY=$2 AND event_type IN ('ActivityFailed','ActivityTimedOut','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','WorkflowFailed','WorkflowTimedOut','WorkflowCancelled','WorkflowTerminated','UpdateCompleted') ORDER BY sequence DESC LIMIT 1"; - const ACTIVITY_RESULT_EVENT_SQL: &'static str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (json_extract(payload,'$.activity_execution_id') COLLATE BINARY=$2 OR json_extract(payload,'$.activity.id') COLLATE BINARY=$3) ORDER BY sequence DESC LIMIT 1"; - const QUERY_CACHE_SCAN: &'static str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND json_extract(value,'$.namespace')=$1 ORDER BY \"key\" LIMIT 65"; - const QUERY_CACHE_PENDING_FOR_RUN: &'static str = "SELECT json_extract(value,'$.query_admitted_at') AS query_admitted_at,json_extract(value,'$.query_deadline_at') AS query_deadline_at,json_extract(value,'$.receipt_expires_at') AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND json_extract(value,'$.run_id')=$1 AND json_extract(value,'$.status') IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; - const POLL_WAKE_QUERIES_SQL: &'static str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND json_extract(value,'$.namespace')=$2 AND json_extract(value,'$.task_queue')=$3 AND json_extract(value,'$.query_deadline_at')>$4 AND (json_extract(value,'$.status')='pending' OR (json_extract(value,'$.status')='leased' AND json_extract(value,'$.lease_expires_at')<=$5)) ORDER BY \"key\" LIMIT 65"; - const POLL_WAKE_TASKS_SQL: &'static str = r#" + const ACTIVITY_RESULT_EVENT_SQL: &str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (json_extract(payload,'$.activity_execution_id') COLLATE BINARY=$2 OR json_extract(payload,'$.activity.id') COLLATE BINARY=$3) ORDER BY sequence DESC LIMIT 1"; + const QUERY_CACHE_SCAN: &str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND json_extract(value,'$.namespace')=$1 ORDER BY \"key\" LIMIT 65"; + const QUERY_CACHE_PENDING_FOR_RUN: &str = "SELECT json_extract(value,'$.query_admitted_at') AS query_admitted_at,json_extract(value,'$.query_deadline_at') AS query_deadline_at,json_extract(value,'$.receipt_expires_at') AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND json_extract(value,'$.run_id')=$1 AND json_extract(value,'$.status') IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; + const POLL_WAKE_QUERIES_SQL: &str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND json_extract(value,'$.namespace')=$2 AND json_extract(value,'$.task_queue')=$3 AND json_extract(value,'$.query_deadline_at')>$4 AND (json_extract(value,'$.status')='pending' OR (json_extract(value,'$.status')='leased' AND json_extract(value,'$.lease_expires_at')<=$5)) ORDER BY \"key\" LIMIT 65"; + const POLL_WAKE_TASKS_SQL: &str = r#" WITH utc_tasks AS ( SELECT t.id,t.namespace,t.queue,t.workflow_run_id,t.status,t.available_at, replace(replace(replace(t.available_at,'T',' '),'Z',''),'+00:00','') AS available_utc, @@ -172,24 +172,24 @@ WHERE r.status IN ('pending','running','waiting') AND (t.available_at IS NULL OR t.available_order<=rtrim(rtrim($5,'0'),'.')) ORDER BY t.available_order,t.id LIMIT 100 "#; - const SELECTION_RESOLUTION_SQL: &'static str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND json_extract(payload,'$.selection_group_id')=$2 ORDER BY sequence LIMIT 1"; + const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND json_extract(payload,'$.selection_group_id')=$2 ORDER BY sequence LIMIT 1"; const REPAIR_COUNTER_MAX: i64 = i64::MAX; - const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; - const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND json_extract(payload,'$.activity_execution_id')=$2 ORDER BY id LIMIT 1"; - const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(activity_options,'$.execution_mode')='local'"; - const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(a.activity_options,'$.execution_mode')='local'"; - const VISIBILITY_PAGE: &'static str = super::visibility::SQLITE_VISIBILITY_PAGE; + const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; + const ACTIVITY_OPEN_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND json_extract(payload,'$.activity_execution_id')=$2 ORDER BY id LIMIT 1"; + const LOCAL_ACTIVITY_EXECUTIONS: &str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(activity_options,'$.execution_mode')='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(a.activity_options,'$.execution_mode')='local'"; + const VISIBILITY_PAGE: &str = super::visibility::SQLITE_VISIBILITY_PAGE; type EncodedTime = String; type EncodedOptionalTime = Option; type EncodedDocument = String; - const TIMER_TASK_SQL: &'static str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; + const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; - const HISTORY_PAGE: &'static str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; + const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier // native precision; SQLite's floating-point date functions are avoided. - const TASK_CANDIDATES_SQL: &'static str = r#" + const TASK_CANDIDATES_SQL: &str = r#" WITH utc_tasks AS ( SELECT t.*, replace(replace(replace(t.available_at,'T',' '),'Z',''),'+00:00','') AS _dw_available_utc, @@ -203,11 +203,12 @@ WITH utc_tasks AS ( ) SELECT t.*,r.workflow_type FROM ordered_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE r.status IN ('pending','running','waiting') - AND (t.status='ready' OR (t.status='leased' AND t._dw_lease_order<=rtrim(rtrim($4,'0'),'.'))) + AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t._dw_lease_order<=rtrim(rtrim($4,'0'),'.'))) AND (t.available_at IS NULL OR t._dw_available_order<=rtrim(rtrim($5,'0'),'.')) + AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t._dw_available_order,t.id LIMIT 100 "#; - const POLL_RECEIPT_CLEANUP_SQL: &'static str = r#" + const POLL_RECEIPT_CLEANUP_SQL: &str = r#" WITH utc_receipts AS ( SELECT p.namespace,p.worker_id,p.kind,p.request_id,t.status, replace(replace(replace(p.expires_at,'T',' '),'Z',''),'+00:00','') AS receipt_utc, @@ -315,7 +316,7 @@ impl Backend for Postgres { fn optional_flag(value: Option) -> Option { value } - const READ_SNAPSHOT_BEGIN: &'static str = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"; + const READ_SNAPSHOT_BEGIN: &str = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"; const PHP_CONNECTION: &str = "pgsql"; fn optional_float(row: &Self::Row, field: &str) -> Result> { Ok(row.try_get(field)?) @@ -323,8 +324,8 @@ impl Backend for Postgres { type EncodedTime = NaiveDateTime; type EncodedOptionalTime = Option; type EncodedDocument = PostgresDocument; - const TIMER_TASK_SQL: &'static str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND payload->>'timer_id'=$2 AND namespace=$3"; - const HISTORY_PAGE: &'static str = "SELECT * FROM (SELECT *,SUM(octet_length(payload::text)) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; + const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND payload->>'timer_id'=$2 AND namespace=$3"; + const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(octet_length(payload::text)) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; fn bind_time(time: DateTime) -> NaiveDateTime { time.naive_utc() } @@ -410,13 +411,13 @@ impl Backend for MySql { fn optional_flag(value: Option) -> Option { value } - const SEARCH_ATTRIBUTE_ROWS: &'static str = "SELECT `key` AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY `key` LIMIT 1001"; - const SEARCH_ATTRIBUTE_UPSERT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON DUPLICATE KEY UPDATE workflow_instance_id=VALUES(workflow_instance_id),type=VALUES(type),value_string=VALUES(value_string),value_keyword=VALUES(value_keyword),value_keyword_list=VALUES(value_keyword_list),value_int=VALUES(value_int),value_float=VALUES(value_float),value_bool=VALUES(value_bool),value_datetime=VALUES(value_datetime),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at)"; - const SEARCH_ATTRIBUTE_DELETE: &'static str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; - const SEARCH_ATTRIBUTE_KEY_ROWS: &'static str = "SELECT a.workflow_run_id,a.`key` AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.`key`=$2"; - const SEARCH_ATTRIBUTE_INHERIT: &'static str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; - const READ_SNAPSHOT_BEGIN: &'static str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; - const ACTIVITY_METADATA_HISTORY: &'static str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; + const SEARCH_ATTRIBUTE_ROWS: &str = "SELECT `key` AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 ORDER BY `key` LIMIT 1001"; + const SEARCH_ATTRIBUTE_UPSERT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,false,$13,$14) ON DUPLICATE KEY UPDATE workflow_instance_id=VALUES(workflow_instance_id),type=VALUES(type),value_string=VALUES(value_string),value_keyword=VALUES(value_keyword),value_keyword_list=VALUES(value_keyword_list),value_int=VALUES(value_int),value_float=VALUES(value_float),value_bool=VALUES(value_bool),value_datetime=VALUES(value_datetime),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at)"; + const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; + const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.`key` AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.`key`=$2"; + const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + const READ_SNAPSHOT_BEGIN: &str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; + const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; const OPEN_WAIT_ORDER: &str = "WITH clocks AS (SELECT id,wait_id,opened_at,TIMESTAMPDIFF(MICROSECOND,'1970-01-01 00:00:00',opened_at) AS micros FROM workflow_run_waits WHERE workflow_run_id=? AND status='open') SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END,ROUND((CASE WHEN micros<0 THEN 2*FLOOR(micros/1000000)*1000000-micros ELSE micros END)/1000),wait_id LIMIT 100 OFFSET ?"; const SUMMARY_HISTORY_SQL: &str = "SELECT id,sequence,event_type,CAST(payload AS CHAR) AS payload_json FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT 16"; @@ -425,42 +426,42 @@ impl Backend for MySql { fn optional_float(row: &Self::Row, field: &str) -> Result> { Ok(row.try_get(field)?) } - const ACTIVITY_RESULT_EVENT_SQL: &'static str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE CAST(workflow_run_id AS BINARY)=CAST(? AS BINARY) AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (CAST(JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity_execution_id')) AS BINARY)=CAST(? AS BINARY) OR CAST(JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity.id')) AS BINARY)=CAST(? AS BINARY)) ORDER BY sequence DESC LIMIT 1"; - const QUERY_CACHE_SCAN: &'static str = "SELECT `key`,value,expiration FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND BINARY JSON_UNQUOTE(JSON_EXTRACT(value,'$.namespace'))=BINARY ? ORDER BY `key` LIMIT 65"; - const POLL_WAKE_QUERIES_SQL: &'static str = "SELECT `key` AS id FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND expiration>? AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.namespace')) AS BINARY)=CAST(? AS BINARY) AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.task_queue')) AS BINARY)=CAST(? AS BINARY) AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_deadline_at')) AS BINARY)>CAST(? AS BINARY) AND (JSON_UNQUOTE(JSON_EXTRACT(value,'$.status'))='pending' OR (JSON_UNQUOTE(JSON_EXTRACT(value,'$.status'))='leased' AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.lease_expires_at')) AS BINARY)<=CAST(? AS BINARY))) ORDER BY `key` LIMIT 65"; - const EXTERNAL_PAYLOAD_BYTES: &'static str = "SELECT CAST(COALESCE(SUM(size_bytes),0) AS SIGNED) FROM runtime_external_payloads WHERE namespace=$1"; - const MEMO_UPSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,$5,false,$6,$7,$8,$9) ON DUPLICATE KEY UPDATE value=VALUES(value),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at),portable_value=VALUES(portable_value),portable_value_sequence=VALUES(portable_value_sequence)"; - const MEMO_DELETE: &'static str = + const ACTIVITY_RESULT_EVENT_SQL: &str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE CAST(workflow_run_id AS BINARY)=CAST(? AS BINARY) AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (CAST(JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity_execution_id')) AS BINARY)=CAST(? AS BINARY) OR CAST(JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity.id')) AS BINARY)=CAST(? AS BINARY)) ORDER BY sequence DESC LIMIT 1"; + const QUERY_CACHE_SCAN: &str = "SELECT `key`,value,expiration FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND BINARY JSON_UNQUOTE(JSON_EXTRACT(value,'$.namespace'))=BINARY ? ORDER BY `key` LIMIT 65"; + const POLL_WAKE_QUERIES_SQL: &str = "SELECT `key` AS id FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND expiration>? AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.namespace')) AS BINARY)=CAST(? AS BINARY) AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.task_queue')) AS BINARY)=CAST(? AS BINARY) AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_deadline_at')) AS BINARY)>CAST(? AS BINARY) AND (JSON_UNQUOTE(JSON_EXTRACT(value,'$.status'))='pending' OR (JSON_UNQUOTE(JSON_EXTRACT(value,'$.status'))='leased' AND CAST(JSON_UNQUOTE(JSON_EXTRACT(value,'$.lease_expires_at')) AS BINARY)<=CAST(? AS BINARY))) ORDER BY `key` LIMIT 65"; + const EXTERNAL_PAYLOAD_BYTES: &str = "SELECT CAST(COALESCE(SUM(size_bytes),0) AS SIGNED) FROM runtime_external_payloads WHERE namespace=$1"; + const MEMO_UPSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,$5,false,$6,$7,$8,$9) ON DUPLICATE KEY UPDATE value=VALUES(value),upserted_at_sequence=VALUES(upserted_at_sequence),inherited_from_parent=false,updated_at=VALUES(updated_at),portable_value=VALUES(portable_value),portable_value_sequence=VALUES(portable_value_sequence)"; + const MEMO_DELETE: &str = "DELETE FROM workflow_memos WHERE workflow_run_id=$1 AND `key`=$2"; - const MEMO_INITIAL_INSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,0,false,$5,$6,$7,0)"; - const MEMO_INHERIT_INSERT: &'static str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,1,true,$5,$6,$7,1)"; - const MEMO_READ: &'static str = "SELECT `key`,value,portable_value,portable_value_sequence,upserted_at_sequence FROM workflow_memos WHERE workflow_run_id=$1 ORDER BY `key`,id LIMIT 101"; - const SELECTION_RESOLUTION_SQL: &'static str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SelectionResolved' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.selection_group_id'))=? ORDER BY sequence LIMIT 1"; + const MEMO_INITIAL_INSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,0,false,$5,$6,$7,0)"; + const MEMO_INHERIT_INSERT: &str = "INSERT INTO workflow_memos(workflow_run_id,workflow_instance_id,`key`,value,upserted_at_sequence,inherited_from_parent,created_at,updated_at,portable_value,portable_value_sequence) VALUES ($1,$2,$3,$4,1,true,$5,$6,$7,1)"; + const MEMO_READ: &str = "SELECT `key`,value,portable_value,portable_value_sequence,upserted_at_sequence FROM workflow_memos WHERE workflow_run_id=$1 ORDER BY `key`,id LIMIT 101"; + const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SelectionResolved' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.selection_group_id'))=? ORDER BY sequence LIMIT 1"; const REPAIR_COUNTER_MAX: i64 = u32::MAX as i64; - const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND namespace=? AND task_type='workflow' AND status='failed' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.replay_blocked'))='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; - const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND task_type='activity' AND status IN ('ready','leased') AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity_execution_id'))=? ORDER BY id LIMIT 1"; - const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(activity_options,'$.execution_mode'))='local'"; - const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(a.activity_options,'$.execution_mode'))='local'"; + const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND namespace=? AND task_type='workflow' AND status='failed' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.replay_blocked'))='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; + const ACTIVITY_OPEN_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND task_type='activity' AND status IN ('ready','leased') AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity_execution_id'))=? ORDER BY id LIMIT 1"; + const LOCAL_ACTIVITY_EXECUTIONS: &str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(activity_options,'$.execution_mode'))='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(a.activity_options,'$.execution_mode'))='local'"; type EncodedTime = DateTime; type EncodedOptionalTime = Option>; type EncodedDocument = Json; - const QUERY_CACHE_PENDING_FOR_RUN: &'static str = "SELECT JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_admitted_at')) AS query_admitted_at,JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_deadline_at')) AS query_deadline_at,JSON_UNQUOTE(JSON_EXTRACT(value,'$.receipt_expires_at')) AS receipt_expires_at FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND JSON_UNQUOTE(JSON_EXTRACT(value,'$.run_id'))=? AND expiration>? AND JSON_UNQUOTE(JSON_EXTRACT(value,'$.status')) IN ('pending','leased') ORDER BY `key` LIMIT 65"; - const QUERY_CACHE_GET: &'static str = + const QUERY_CACHE_PENDING_FOR_RUN: &str = "SELECT JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_admitted_at')) AS query_admitted_at,JSON_UNQUOTE(JSON_EXTRACT(value,'$.query_deadline_at')) AS query_deadline_at,JSON_UNQUOTE(JSON_EXTRACT(value,'$.receipt_expires_at')) AS receipt_expires_at FROM dw_query_cache WHERE `key` LIKE 'dw-native-query:%' AND JSON_UNQUOTE(JSON_EXTRACT(value,'$.run_id'))=? AND expiration>? AND JSON_UNQUOTE(JSON_EXTRACT(value,'$.status')) IN ('pending','leased') ORDER BY `key` LIMIT 65"; + const QUERY_CACHE_GET: &str = "SELECT value,expiration FROM dw_query_cache WHERE `key`=$1"; - const QUERY_CACHE_INSERT: &'static str = + const QUERY_CACHE_INSERT: &str = "INSERT INTO dw_query_cache(`key`,value,expiration) VALUES ($1,$2,$3)"; - const QUERY_CACHE_UPDATE: &'static str = + const QUERY_CACHE_UPDATE: &str = "UPDATE dw_query_cache SET value=$1,expiration=$2 WHERE `key`=$3"; - const QUERY_CACHE_DELETE: &'static str = "DELETE FROM dw_query_cache WHERE `key`=$1"; - const QUERY_POLL_EXPIRED: &'static str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; - const TIMER_TASK_SQL: &'static str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; + const QUERY_CACHE_DELETE: &str = "DELETE FROM dw_query_cache WHERE `key`=$1"; + const QUERY_POLL_EXPIRED: &str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; + const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. - const HISTORY_PAGE: &'static str = "SELECT h.* FROM (SELECT id,sequence,SUM(payload_bytes) OVER (ORDER BY sequence) AS page_bytes FROM (SELECT id,sequence,LENGTH(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=? AND sequence>? ORDER BY sequence LIMIT ?) AS sizes) AS page JOIN workflow_history_events h ON h.id=page.id WHERE page.page_bytes<=8388608 ORDER BY page.sequence"; - const WORKER_REGISTRATION_SQL: &'static str = "INSERT INTO workflow_worker_registrations(namespace,worker_id,task_queue,runtime,sdk_version,build_id,supported_workflow_types,supported_activity_types,capabilities,capability_manifest,last_heartbeat_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE task_queue=VALUES(task_queue),runtime=VALUES(runtime),sdk_version=VALUES(sdk_version),build_id=VALUES(build_id),supported_workflow_types=VALUES(supported_workflow_types),supported_activity_types=VALUES(supported_activity_types),capabilities=VALUES(capabilities),capability_manifest=VALUES(capability_manifest),last_heartbeat_at=VALUES(last_heartbeat_at),updated_at=VALUES(updated_at)"; + const HISTORY_PAGE: &str = "SELECT h.* FROM (SELECT id,sequence,SUM(payload_bytes) OVER (ORDER BY sequence) AS page_bytes FROM (SELECT id,sequence,LENGTH(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=? AND sequence>? ORDER BY sequence LIMIT ?) AS sizes) AS page JOIN workflow_history_events h ON h.id=page.id WHERE page.page_bytes<=8388608 ORDER BY page.sequence"; + const WORKER_REGISTRATION_SQL: &str = "INSERT INTO workflow_worker_registrations(namespace,worker_id,task_queue,runtime,sdk_version,build_id,supported_workflow_types,supported_activity_types,capabilities,capability_manifest,last_heartbeat_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE task_queue=VALUES(task_queue),runtime=VALUES(runtime),sdk_version=VALUES(sdk_version),build_id=VALUES(build_id),supported_workflow_types=VALUES(supported_workflow_types),supported_activity_types=VALUES(supported_activity_types),capabilities=VALUES(capabilities),capability_manifest=VALUES(capability_manifest),last_heartbeat_at=VALUES(last_heartbeat_at),updated_at=VALUES(updated_at)"; fn statement(sql: &'static str) -> Cow<'static, str> { // These driver-owned templates already use '?' bindings and contain // literal '$.' JSON paths, not PostgreSQL bind placeholders. @@ -632,6 +633,7 @@ mod timestamp_tests { .bind("workflow") .bind(Sqlite::bind_time(clock)) .bind(Sqlite::bind_time(clock)) + .bind(None::<&str>) .fetch_all(pool) .await .unwrap(); @@ -649,6 +651,7 @@ mod timestamp_tests { .bind("workflow") .bind(Sqlite::bind_time(clock)) .bind(Sqlite::bind_time(clock)) + .bind(None::<&str>) .fetch_all(pool) .await .unwrap(); diff --git a/rust/src/runtime/cancellation.rs b/rust/src/runtime/cancellation.rs index 4b8e0c0e8..fc0fbcb7d 100644 --- a/rust/src/runtime/cancellation.rs +++ b/rust/src/runtime/cancellation.rs @@ -20,16 +20,7 @@ use tokio::sync::Semaphore; // Pin its reviewed character set rather than a changing Unicode whitespace // predicate. Internal characters and the embedded API's reasons stay intact. pub(super) fn normalize_http_reason(reason: &str) -> &str { - reason.trim_matches(|ch| { - matches!(ch, - '\0' | '\t'..='\r' | ' ' | '\u{85}' | '\u{a0}' | '\u{ad}' | - '\u{34f}' | '\u{61c}' | '\u{115f}' | '\u{1160}' | '\u{1680}' | - '\u{17b4}' | '\u{17b5}' | '\u{180e}' | '\u{2000}'..='\u{200f}' | - '\u{2028}' | '\u{2029}' | '\u{202f}' | '\u{205f}' | - '\u{2060}'..='\u{2065}' | '\u{206a}'..='\u{206f}' | '\u{2800}' | - '\u{3000}' | '\u{3164}' | '\u{feff}' | '\u{ffa0}' | '\u{1d159}' | - '\u{1d173}'..='\u{1d17a}' | '\u{e0020}') - }) + super::laravel_validation::laravel_trim(reason) } /// The two immediate terminal commands. Published PHP runs both through one diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index d9403d1a1..563909abd 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -129,17 +129,8 @@ where return Ok(ChildParentResume::New); } if Self::cancellation_request(parent)?.is_none() { - // An update task does not replay the root workflow, so the - // resolution gets its own resume task, as activity and timer - // resolutions do. Published PHP reuses the open task instead. - for task in &tasks { - if DB::document_row(task, "payload")?["workflow_update_id"] - .as_str() - .is_none() - { - return Err(refuse(StatusCode::CONFLICT, "child_parent_task_conflict")); - } - } + // The resolution gets its own resume task, as activity and timer + // resolutions do. Published PHP reuses an open task instead. return Ok(ChildParentResume::New); } if tasks.len() > 1000 { @@ -200,10 +191,13 @@ where Ok(consistent.then_some((parent_close, cancellation))) } + /// `externalCommandContract`: the type's contract from the most recently + /// heartbeated active worker, in the start's cohort when it has one. pub(super) async fn attach_workflow_contract( tx: &mut Transaction<'_, DB>, queue: &str, workflow_type: &str, + cohort: Option<&super::start_definition::Cohort>, started: &mut Value, ) -> Result<()> { let instance = Self::query("SELECT namespace FROM workflow_instances WHERE id=$1") @@ -211,9 +205,13 @@ where .fetch_one(&mut **tx) .await?; let namespace = DB::string(&instance, "namespace")?; - let workers = Self::query("SELECT supported_workflow_types,workflow_command_contracts FROM workflow_worker_registrations WHERE task_queue=$1 AND status='active' AND workflow_command_contracts IS NOT NULL AND last_heartbeat_at>$2 AND namespace=$3 ORDER BY id DESC LIMIT 100") + let workers = Self::query("SELECT build_id,supported_workflow_types,workflow_command_contracts FROM workflow_worker_registrations WHERE task_queue=$1 AND status='active' AND workflow_command_contracts IS NOT NULL AND last_heartbeat_at>$2 AND namespace=$3 ORDER BY last_heartbeat_at DESC,id DESC LIMIT 100") .bind(queue).bind(DB::bind_time(now() - chrono::Duration::seconds(60))).bind(&namespace).fetch_all(&mut **tx).await?; for worker in workers { + let build = DB::optional_string(&worker, "build_id")?; + if cohort.is_some_and(|cohort| !cohort.admits(build.as_deref())) { + continue; + } if DB::document_row(&worker, "supported_workflow_types")? .as_array() .is_some_and(|types| types.iter().any(|value| value == workflow_type)) @@ -256,28 +254,12 @@ where task_id: &str, command: &Value, sequence: i64, - ) -> Result<()> { + ) -> Result { let parent_run = DB::string(parent, "id")?; let parent_instance = DB::string(parent, "workflow_instance_id")?; - // This first slice permits one outstanding authored child. A replayed - // completion is handled by the enclosing immutable task receipt. - for sql in [ - "SELECT COUNT(*) FROM workflow_child_calls WHERE parent_workflow_run_id=$1 AND status NOT IN ('completed','cancelled','terminated','failed')", - "SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$1 AND status!='completed'", - "SELECT COUNT(*) FROM workflow_run_timers WHERE workflow_run_id=$1 AND status='pending'", - ] { - if Self::scalar(sql) - .bind(&parent_run) - .fetch_one(&mut **tx) - .await? - != 0 - { - return Err(refuse(StatusCode::CONFLICT, "pending_durable_operations")); - } - } - if Self::open_wait(tx, &parent_run).await?.is_some() { - return Err(refuse(StatusCode::CONFLICT, "pending_durable_operations")); - } + // Published PHP starts a child whatever else the parent has open: + // activities, timers, waits and other children. A replayed completion + // is handled by the enclosing immutable task receipt. let workflow_type = text(command, "workflow_type")?; let queue = command .get("queue") @@ -292,12 +274,15 @@ where // A child lives in its parent's namespace (ChildWorkflowNamespaceProjection). let namespace = DB::optional_string(parent, "namespace")?; let started_at = now(); + // `ChildCallService`: a child keeps its parent's build pin. A child of + // an unpinned parent is unpinned too (checked against published 2.11.1). + let compatibility = DB::optional_string(parent, "compatibility")?.filter(|build| !build.trim().is_empty()); // PHP children without explicit timeout options have nullable budgets // and deadlines. Do not invent parent defaults for these runs. Self::query("INSERT INTO workflow_instances(id,namespace,workflow_type,workflow_class,current_run_id,run_count,started_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,1,$6,$7,$8)") .bind(&child_instance).bind(namespace.as_deref()).bind(workflow_type).bind(workflow_type).bind(&child_run).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).execute(&mut **tx).await?; - Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,namespace,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,started_at,created_at,updated_at,last_progress_at) VALUES ($1,$2,$3,$4,$5,1,'pending','avro',$6,$7,$8,$9,$10,$11)") - .bind(&child_run).bind(&child_instance).bind(namespace.as_deref()).bind(workflow_type).bind(workflow_type).bind(&arguments).bind(&queue).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,namespace,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,started_at,created_at,updated_at,last_progress_at,compatibility) VALUES ($1,$2,$3,$4,$5,1,'pending','avro',$6,$7,$8,$9,$10,$11,$12)") + .bind(&child_run).bind(&child_instance).bind(namespace.as_deref()).bind(workflow_type).bind(workflow_type).bind(&arguments).bind(&queue).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(compatibility.as_deref()).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_child_calls(parent_workflow_run_id,parent_workflow_instance_id,sequence,child_workflow_type,child_workflow_class,resolved_child_instance_id,resolved_child_run_id,parent_close_policy,cancellation_propagation,queue,status,scheduled_at,started_at,arguments,metadata,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,($9<>'abandon'),$10,'started',$11,$12,$13,$14,$15,$16)") .bind(&parent_run).bind(&parent_instance).bind(sequence).bind(workflow_type).bind(workflow_type).bind(&child_instance).bind(&child_run).bind(&policies.parent_close).bind(&policies.cancellation).bind(&queue).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)) .bind(DB::document(&json!({"payload":arguments}))).bind(DB::document(&json!({"child_call_id":call,"attempt_count":1,"cancellation_policy":policies.cancellation}))).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).execute(&mut **tx).await?; @@ -333,14 +318,14 @@ where ) .await?; let mut child_started = json!({"workflow_class":workflow_type,"workflow_type":workflow_type,"workflow_instance_id":child_instance,"workflow_run_id":child_run,"parent_workflow_instance_id":parent_instance,"parent_workflow_run_id":parent_run,"parent_sequence":sequence,"workflow_link_id":call,"child_call_id":call,"retry_policy":null,"timeout_policy":null,"execution_timeout_seconds":null,"run_timeout_seconds":null,"execution_deadline_at":null,"run_deadline_at":null}); - Self::attach_workflow_contract(tx, &queue, workflow_type, &mut child_started).await?; + Self::attach_workflow_contract(tx, &queue, workflow_type, None, &mut child_started).await?; Self::append(tx, &child_run, "WorkflowStarted", child_started, None, None).await?; - Self::create_task(tx, &child_run, &queue, "workflow", json!({})).await?; + let child_task = Self::create_task(tx, &child_run, &queue, "workflow", json!({})).await?; Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1") .bind(&parent_run) .execute(&mut **tx) .await?; - Ok(()) + Ok(child_task) } pub(super) async fn complete_child( @@ -363,7 +348,7 @@ where .bind(&child_run).fetch_one(&mut **tx).await?; let original = DB::document_row(&started, "payload")?; let links = Self::query("SELECT * FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow' LIMIT 2").bind(&child_run).fetch_all(&mut **tx).await?; - if links.is_empty() && original.get("parent_workflow_run_id").is_none() { + if links.is_empty() && original["parent_workflow_run_id"].is_null() { return Ok(()); } if links.is_empty() { @@ -378,12 +363,21 @@ where let link = &links[0]; let parent_run = DB::string(link, "parent_workflow_run_id")?; let sequence = DB::number(link, "sequence")?; - let call_id = DB::string(link, "id")?; + let link_id = DB::string(link, "id")?; + // A child that continued as new keeps its first call: its started + // event names the continue-as-new link and the original call, and the + // parent's link to it is a new one (`applyContinueAsNew`). + let continued = original["continued_from_run_id"].is_string(); + let call_id = if continued { + original["child_call_id"].as_str().map(str::to_owned).ok_or_else(|| refuse(StatusCode::CONFLICT, "child_call_mismatch"))? + } else { + link_id.clone() + }; if original["parent_workflow_run_id"] != parent_run || original["parent_workflow_instance_id"] != DB::string(link, "parent_workflow_instance_id")? || original["parent_sequence"] != sequence - || original["workflow_link_id"] != call_id + || (!continued && original["workflow_link_id"] != call_id) || original["child_call_id"] != call_id || original["workflow_instance_id"] != DB::string(child, "workflow_instance_id")? || original["workflow_run_id"] != child_run @@ -400,7 +394,9 @@ where || DB::optional_string(call, "resolved_child_instance_id")?.as_deref() != Some(&DB::string(child, "workflow_instance_id")?) || DB::document_row(call, "metadata")?["child_call_id"] != call_id - || DB::string(call, "child_workflow_type")? != DB::string(child, "workflow_type")? + // A continued child may run a new workflow type; the call keeps + // the type it started with, as PHP's does. + || (!continued && DB::string(call, "child_workflow_type")? != DB::string(child, "workflow_type")?) || DB::string(link, "child_workflow_instance_id")? != DB::string(child, "workflow_instance_id")? || DB::string(call, "parent_workflow_instance_id")? @@ -439,7 +435,7 @@ where return Ok(()); } let parent_resume = Self::child_parent_resume(tx, &parent).await?; - let mut payload = json!({"sequence":sequence,"workflow_link_id":call_id,"child_call_id":call_id,"child_workflow_instance_id":DB::string(child,"workflow_instance_id")?,"child_workflow_run_id":child_run,"child_workflow_class":DB::string(child,"workflow_class")?,"child_workflow_type":DB::string(child,"workflow_type")?,"child_run_number":DB::number(child,"run_number")?,"child_status":status,"closed_reason":status,"closed_at":closed_at}); + let mut payload = json!({"sequence":sequence,"workflow_link_id":link_id,"child_call_id":call_id,"child_workflow_instance_id":DB::string(child,"workflow_instance_id")?,"child_workflow_run_id":child_run,"child_workflow_class":DB::string(child,"workflow_class")?,"child_workflow_type":DB::string(child,"workflow_type")?,"child_run_number":DB::number(child,"run_number")?,"child_status":status,"closed_reason":status,"closed_at":closed_at}); match resolution { ChildResolution::Completed(output) => { payload["output"] = wire(output); diff --git a/rust/src/runtime/command_payloads.rs b/rust/src/runtime/command_payloads.rs new file mode 100644 index 000000000..c4e09df0f --- /dev/null +++ b/rust/src/runtime/command_payloads.rs @@ -0,0 +1,150 @@ +//! The payload checks published PHP 2.11.1 runs on a workflow task +//! completion before it applies any command +//! (`WorkerController::resolveWorkflowTaskCommandPayloadReferences`, then the +//! command normalizer): every payload a command carries must be one complete +//! Avro value, sent as a string or as a `{codec, blob}` envelope with the Avro +//! codec. A refusal is a Laravel validation failure naming the field. +use super::input_envelope::codec_refusal; +use super::laravel_validation::failure_messages; +use super::RuntimeError; +use serde_json::Value; + +/// `WorkflowCommandNormalizer::PAYLOAD_ENVELOPE_FIELDS`. +const FIELDS: &[(&str, &[&str])] = &[ + ("complete_workflow", &["result"]), + ("schedule_activity", &["arguments"]), + ("start_child_workflow", &["arguments"]), + ("continue_as_new", &["arguments"]), + ("complete_update", &["result"]), + ("record_side_effect", &["result"]), + ("record_local_activity", &["arguments", "result"]), + ("start_service_operation", &["request_payload"]), + ("upsert_memo", &["entries"]), +]; + +/// Fields the normalizer accepts only as a string or an envelope. +const STRICT: &[(&str, &str)] = &[("complete_workflow", "result"), ("schedule_activity", "arguments")]; + +/// `AvroPayloadEnvelopeResolver::assertSerializedPayload`. +pub(super) const UNDECODABLE: &str = + "The payload must contain one complete Avro Value encoded with the official codec. JSON is only the HTTP document transport."; + +pub(super) fn invalid(field: &str, message: &str) -> RuntimeError { + failure_messages(vec![(field.to_owned(), vec![message.to_owned()])]).expect("one message") +} + +/// One check, in PHP's order: a blob to decode, or a refusal already known. +pub(super) enum Check { + Decode { field: String, blob: String }, + Refuse(RuntimeError), +} + +/// The checks for each command's payload fields, in command and field order. +/// A refusal ends the list, since PHP stops at the first invalid field. +pub(super) fn checks(body: &Value) -> Vec { + let mut checks = Vec::new(); + let Some(commands) = body["commands"].as_array() else { + return checks; + }; + for (index, command) in commands.iter().enumerate() { + // `PayloadCodecContract::canonicalize` on any command's codec. + if let Some(codec) = command.get("payload_codec") + && let Some(message) = codec_refusal(codec) + { + checks.push(Check::Refuse(invalid(&format!("commands.{index}.payload_codec"), &message))); + return checks; + } + let Some(kind) = command["type"].as_str() else { + continue; + }; + let Some((_, fields)) = FIELDS.iter().find(|(name, _)| *name == kind) else { + continue; + }; + for field in *fields { + let path = format!("commands.{index}.{field}"); + let strict = STRICT.contains(&(kind, *field)); + let not_payload = || { + Check::Refuse(invalid( + &path, + &format!("Workflow task command field [{field}] must be a string or a payload envelope when provided."), + )) + }; + match &command[*field] { + Value::Null => {} + Value::String(blob) => checks.push(Check::Decode { field: path.clone(), blob: blob.clone() }), + Value::Object(envelope) if envelope.contains_key("codec") => { + if let Some(message) = codec_refusal(&envelope["codec"]) { + checks.push(Check::Refuse(invalid(&format!("{path}.codec"), &message))); + return checks; + } + // External references are resolved, and refused, on their own path. + if envelope.contains_key("external_storage") || envelope.contains_key("external_payload") { + continue; + } + let mut keys: Vec<&str> = envelope.keys().map(String::as_str).collect(); + keys.sort_unstable(); + if keys != ["blob", "codec"] { + if strict { + checks.push(not_payload()); + return checks; + } + continue; + } + if !envelope["codec"].as_str().is_some_and(|codec| !codec.is_empty()) { + checks.push(Check::Refuse(invalid(&format!("{path}.codec"), "The payload envelope codec must be a non-empty string."))); + return checks; + } + let Some(blob) = envelope["blob"].as_str() else { + checks.push(Check::Refuse(invalid(&format!("{path}.blob"), "The payload envelope blob must be a string."))); + return checks; + }; + checks.push(Check::Decode { field: format!("{path}.blob"), blob: blob.to_owned() }); + } + _ if strict => { + checks.push(not_payload()); + return checks; + } + _ => {} + } + } + } + checks +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn summary(body: Value) -> Vec { + checks(&body) + .into_iter() + .map(|check| match check { + Check::Decode { field, blob } => format!("decode {field} {blob}"), + Check::Refuse(RuntimeError::Protocol { response, .. }) => format!("refuse {}", response["errors"]), + Check::Refuse(other) => panic!("{other:?}"), + }) + .collect() + } + + #[test] + fn checks_follow_php_s_order_and_messages() { + let commands = |commands: Value| summary(json!({"commands": commands})); + assert_eq!(commands(json!([{"type":"complete_workflow","result":{"codec":"avro","blob":"A"}}])), ["decode commands.0.result.blob A"]); + assert_eq!(commands(json!([{"type":"record_side_effect","result":"B"}])), ["decode commands.0.result B"]); + assert_eq!(commands(json!([{"type":"complete_workflow","result":[1]}])), + [r#"refuse {"commands.0.result":["Workflow task command field [result] must be a string or a payload envelope when provided."]}"#]); + assert_eq!(commands(json!([{"type":"complete_workflow","result":{"codec":"avro"}}])).len(), 1); + assert_eq!(commands(json!([{"type":"complete_workflow","result":{"codec":"","blob":"A"}}])), + [r#"refuse {"commands.0.result.codec":["The payload envelope codec must be a non-empty string."]}"#]); + assert_eq!(commands(json!([{"type":"complete_workflow","result":{"codec":"avro","blob":5}}])), + [r#"refuse {"commands.0.result.blob":["The payload envelope blob must be a string."]}"#]); + assert!(commands(json!([{"type":"complete_workflow","result":{"codec":"json","blob":"[]"}}]))[0].contains("commands.0.result.codec")); + // Plain continue-as-new arguments and memo entries are not payload strings. + assert!(commands(json!([{"type":"continue_as_new","arguments":[1]},{"type":"upsert_memo","entries":{"k":1}}])).is_empty()); + // A later command's blob is checked after the earlier ones, in order. + assert_eq!(commands(json!([{"type":"schedule_activity","arguments":{"codec":"avro","blob":"A"}}, + {"type":"schedule_activity","arguments":5}])).len(), 2); + assert!(commands(json!([{"type":"complete_workflow","result":{"codec":"avro","external_payload":{}}}])).is_empty()); + } +} diff --git a/rust/src/runtime/completion_rules.rs b/rust/src/runtime/completion_rules.rs new file mode 100644 index 000000000..dc887b57b --- /dev/null +++ b/rust/src/runtime/completion_rules.rs @@ -0,0 +1,773 @@ +//! The transport rules published PHP 2.11.1 checks on a workflow task +//! completion before anything else, ownership included +//! (`WorkflowTaskCompletionValidation::rules` with the Workflow package's +//! parallel metadata rules, in `WorkerController::completeWorkflowTask`). +//! +//! PHP validates the top-level fields once and the commands in chunks of 25, +//! keeping each command's index. Laravel reports a chunk's errors rule by +//! rule, each rule's expansions in data order, so native does the same. A +//! wildcard attribute is named as written (`commands.0.type`); a named one is +//! shown with spaces (`sticky cache.workflow id`). +//! +//! Once the rules pass, PHP turns digit strings in its integer fields into +//! integers, so `"7"` is a valid delay. +use super::laravel_validation::{Rule, default_message, integer, passes, present}; +use super::Result; +use serde_json::Value; + +#[derive(Clone, Copy)] +enum Check { + Rule(Rule), + Sometimes, + Prohibited, + /// `required_with`: the other field, relative to the same command, or + /// to the body for a top-level field. + RequiredWith(&'static str), + /// `array:keys`: an array with no other keys. + Keys(&'static [&'static str]), +} + +const REQ: Check = Check::Rule(Rule::Required); +const NUL: Check = Check::Rule(Rule::Nullable); +const STR: Check = Check::Rule(Rule::String); +const ARR: Check = Check::Rule(Rule::Array); +const INT: Check = Check::Rule(Rule::Integer); +const BOOL: Check = Check::Rule(Rule::Boolean); +const SOME: Check = Check::Sometimes; +const NO: Check = Check::Prohibited; +const fn min(least: i64) -> Check { + Check::Rule(Rule::Min(least)) +} +const fn max(most: i64) -> Check { + Check::Rule(Rule::Max(most)) +} +const fn one_of(values: &'static [&'static str]) -> Check { + Check::Rule(Rule::In(values)) +} +const STREAM: Check = Check::RequiredWith("workflow_stream"); +const STICKY: Check = Check::RequiredWith("sticky_cache"); +const CLEANUP: Check = Check::RequiredWith("cancellation_cleanup"); +const OUTCOMES: &[&str] = &["completed", "failed", "timed_out", "cancelled"]; +const TIMEOUT_KINDS: &[&str] = &["start_to_close", "schedule_to_close", "heartbeat"]; + +/// The rules outside `commands.*`, in PHP's order. +const TOP: &[(&str, &[Check])] = &[ + ("lease_owner", &[REQ, STR]), + ("workflow_task_attempt", &[REQ, INT, min(1)]), + ("commands", &[REQ, ARR, min(1)]), + ("sticky_cache", &[NUL, ARR]), + ("sticky_cache.worker_id", &[STICKY, STR, max(255)]), + ("sticky_cache.workflow_id", &[STICKY, STR, max(255)]), + ("sticky_cache.run_id", &[STICKY, STR, max(255)]), + ("sticky_cache.build_id", &[STICKY, STR, max(255)]), + ("sticky_cache.ttl_seconds", &[STICKY, INT, min(1), max(3600)]), + ("sticky_cache.metrics", &[NUL, ARR]), + ("sticky_cache.metrics.hit", &[NUL, INT, min(0)]), + ("sticky_cache.metrics.miss", &[NUL, INT, min(0)]), + ("sticky_cache.metrics.eviction", &[NUL, INT, min(0)]), + ("sticky_cache.metrics.forced_cold_replay", &[NUL, INT, min(0)]), +]; + +/// The `commands.*.` rules, relative to one command, in PHP's order. Rules +/// that are only `nullable` never fail and are left out. +const COMMAND: &[(&str, &[Check])] = &[ + ("type", &[REQ, STR]), + ("selection_group_id", &[SOME, STR]), + ("member_key", &[SOME, STR]), + ("member_index", &[SOME, INT, min(0)]), + ("member_base_sequence", &[SOME, INT, min(1)]), + ("member_size", &[SOME, INT, min(1)]), + ("operation_kind", &[SOME, STR]), + ("operation_identity", &[SOME, STR]), + ("cancellation_scope_id", &[SOME, STR, min(1), max(255)]), + ("cancellation_cleanup", &[SOME, REQ, Check::Keys(&["scope_id", "request_id", "delivery_history_event_id"])]), + ("cancellation_cleanup.scope_id", &[SOME, REQ, STR, max(255)]), + ("cancellation_cleanup.request_id", &[CLEANUP, STR, max(255)]), + ("cancellation_cleanup.delivery_history_event_id", &[CLEANUP, STR, max(255)]), + ("activity_type", &[NUL, STR]), + ("connection", &[NUL, STR]), + ("queue", &[NUL, STR]), + ("retry_policy", &[NUL, ARR]), + ("retry_policy.max_attempts", &[NUL, INT, min(1)]), + ("retry_policy.backoff_seconds", &[NUL, ARR]), + ("retry_policy.backoff_seconds.*", &[INT, min(0)]), + ("retry_policy.non_retryable_error_types", &[NUL, ARR]), + ("retry_policy.non_retryable_error_types.*", &[STR]), + ("start_to_close_timeout", &[NUL, INT, min(1)]), + ("schedule_to_start_timeout", &[NUL, INT, min(1)]), + ("schedule_to_close_timeout", &[NUL, INT, min(1)]), + ("heartbeat_timeout", &[NUL, INT, min(1)]), + ("execution_mode", &[NUL, STR, one_of(&["local"])]), + ("outcome", &[NUL, STR, one_of(OUTCOMES)]), + ("attempts", &[NUL, ARR, max(100)]), + ("attempts.*.attempt_id", &[NUL, STR, max(255)]), + ("attempts.*.attempt_number", &[REQ, INT, min(1)]), + ("attempts.*.outcome", &[REQ, STR, one_of(OUTCOMES)]), + ("attempts.*.duration_ms", &[NUL, INT, min(0)]), + ("attempts.*.message", &[NUL, STR]), + ("attempts.*.exception_type", &[NUL, STR, max(255)]), + ("attempts.*.non_retryable", &[NUL, BOOL]), + ("attempts.*.timeout_kind", &[NUL, STR, one_of(TIMEOUT_KINDS)]), + ("attempts.*.retry_reason", &[NUL, STR, one_of(&["failure", "timeout", "cold_replay"])]), + ("attempts.*.backoff_seconds", &[NUL, INT, min(0)]), + ("attempts.*.heartbeats", &[NUL, ARR, max(1000)]), + ("attempts.*.heartbeats.*.details", &[NUL, ARR]), + ("attempts.*.heartbeats.*.elapsed_ms", &[NUL, INT, min(0)]), + ("worker_session", &[NUL, ARR]), + ("worker_session.session_id", &[NUL, STR, max(255)]), + ("worker_session.connection", &[NUL, STR, max(255)]), + ("worker_session.queue", &[NUL, STR, max(255)]), + ("worker_session.requirements", &[NUL, ARR]), + ("worker_session.requirements.*", &[STR, max(255)]), + ("worker_session.lease_seconds", &[NUL, INT, min(1)]), + ("worker_session.ttl_seconds", &[NUL, INT, min(1)]), + ("worker_session.max_concurrent_activities", &[NUL, INT, min(1)]), + ("worker_session.create_if_missing", &[NUL, BOOL]), + ("worker_session.allow_reacquire_after_failure", &[NUL, BOOL]), + ("execution_timeout_seconds", &[NUL, INT, min(1)]), + ("run_timeout_seconds", &[NUL, INT, min(1)]), + ("workflow_type", &[NUL, STR]), + ("delay_seconds", &[NUL, INT, min(0)]), + ("message", &[NUL, STR]), + ("timeout_kind", &[NUL, STR, one_of(TIMEOUT_KINDS)]), + ("payload_codec", &[NUL, STR]), + ("update_id", &[NUL, STR]), + ("exception_class", &[NUL, STR]), + ("exception_type", &[NUL, STR]), + ("exception", &[NUL, ARR]), + ("failed_step_sequence", &[NUL, INT, min(1)]), + ("failed_activity_execution_id", &[NUL, STR, max(255)]), + ("change_id", &[NUL, STR]), + ("version", &[NUL, INT]), + ("min_supported", &[NUL, INT]), + ("max_supported", &[NUL, INT]), + ("attributes", &[NUL, ARR]), + ("attribute_types", &[NUL, ARR]), + ("attribute_types.*", &[STR]), + ("entries", &[NUL, ARR]), + ("non_retryable", &[NUL, BOOL]), + ("parent_close_policy", &[NUL, STR]), + ("cancellation_policy", &[NUL, STR, one_of(&["try_cancel", "wait_cancellation_completed", "abandon"])]), + ("condition_key", &[NUL, STR]), + ("condition_definition_fingerprint", &[NUL, STR]), + ("condition_wait_occurrence_id", &[NUL, STR]), + ("signal_name", &[NUL, STR]), + ("timeout_seconds", &[NUL, INT, min(0)]), + ("endpoint_name", &[NUL, STR, max(191)]), + ("service_name", &[NUL, STR, max(191)]), + ("operation_name", &[NUL, STR, max(191)]), + ("namespace", &[NUL, STR, max(191)]), + ("caller_namespace", &[NUL, STR, max(191)]), + ("service_call_id", &[NUL, STR, max(191)]), + ("idempotency_key", &[NUL, STR, max(191)]), + ("mode_override", &[NUL, STR, one_of(&["sync", "async"])]), + ("wait_for", &[NUL, STR, one_of(&["accepted", "completed"])]), + ("wait_timeout_seconds", &[NUL, INT, min(0)]), + ("target_workflow_instance_id", &[NUL, STR, max(191)]), + ("target_workflow_run_id", &[NUL, STR, max(191)]), + ("business_key", &[NUL, STR, max(191)]), + ("labels", &[NUL, ARR]), + ("memo", &[NUL, ARR]), + ("search_attributes", &[NUL, ARR]), + ("duplicate_start_policy", &[NUL, STR, one_of(&["reject_duplicate", "return_existing_active"])]), + ("metadata", &[NUL, ARR]), + ("request_payload_reference", &[NUL, STR, max(191)]), + ("principal_subject", &[NO]), + ("principal_method", &[NO]), + ("principal_roles", &[NO]), + ("principal_tenant", &[NO]), + ("principal_claims", &[NO]), + // `WorkflowCommandNormalizer::parallelMetadataValidationRules`. + ("parallel_group_id", &[NUL, STR]), + ("parallel_group_kind", &[NUL, STR]), + ("parallel_group_mode", &[NUL, STR]), + ("parallel_group_base_sequence", &[NUL, INT, min(1)]), + ("parallel_group_size", &[NUL, INT, min(1)]), + ("parallel_group_index", &[NUL, INT, min(0)]), + ("selection_member_index", &[NUL, INT, min(0)]), + ("selection_member_base_sequence", &[NUL, INT, min(1)]), + ("selection_member_size", &[NUL, INT, min(1)]), + ("selection_member_kind", &[NUL, STR]), + ("parallel_group_path", &[NUL, ARR]), + ("parallel_group_path.*.parallel_group_id", &[REQ, STR]), + ("parallel_group_path.*.parallel_group_kind", &[REQ, STR]), + ("parallel_group_path.*.parallel_group_mode", &[NUL, STR]), + ("parallel_group_path.*.parallel_group_base_sequence", &[REQ, INT, min(1)]), + ("parallel_group_path.*.parallel_group_size", &[REQ, INT, min(1)]), + ("parallel_group_path.*.parallel_group_index", &[REQ, INT, min(0)]), + ("parallel_group_path.*.selection_member_index", &[NUL, INT, min(0)]), + ("parallel_group_path.*.selection_member_base_sequence", &[NUL, INT, min(1)]), + ("parallel_group_path.*.selection_member_size", &[NUL, INT, min(1)]), + ("parallel_group_path.*.selection_member_kind", &[NUL, STR]), + ("workflow_stream", &[NUL, ARR]), + ("workflow_stream.operation", &[STREAM, STR, one_of(&["append", "close", "error"])]), + ("workflow_stream.stream_name", &[STREAM, STR, max(191)]), + ("workflow_stream.command_identity", &[STREAM, STR, max(191)]), + ("workflow_stream.command_ordinal", &[STREAM, INT, min(0)]), + // `WorkflowStreamService::DEFAULT_MAX_ITEMS_PER_APPEND`. + ("workflow_stream.items", &[NUL, ARR, max(100)]), + ("workflow_stream.items.*", &[ARR]), + ("workflow_stream.items.*.payload_reference", &[NUL, STR, max(191)]), + ("workflow_stream.items.*.payload_codec", &[NUL, STR, max(64)]), + ("workflow_stream.items.*.idempotency_key", &[NUL, STR, max(191)]), + ("workflow_stream.items.*.item_type", &[NUL, STR, max(64)]), + ("workflow_stream.items.*.content_type", &[NUL, STR, max(191)]), + ("workflow_stream.max_pending_items", &[NUL, INT, min(1)]), + ("workflow_stream.error_reason", &[NUL, STR, max(191)]), + ("workflow_stream.retention_seconds", &[NUL, INT, min(1)]), +]; + +/// `WORKFLOW_TASK_COMMAND_VALIDATION_BATCH_SIZE`. +const CHUNK: usize = 25; + +/// `normalizeWorkflowTaskCommandIntegerFields`. +const INTEGER_FIELDS: &[&str] = &[ + "start_to_close_timeout", + "schedule_to_start_timeout", + "schedule_to_close_timeout", + "heartbeat_timeout", + "execution_timeout_seconds", + "run_timeout_seconds", + "delay_seconds", + "version", + "min_supported", + "max_supported", + "timeout_seconds", + "wait_timeout_seconds", + "failed_step_sequence", +]; +const SESSION_INTEGER_FIELDS: &[&str] = &["lease_seconds", "ttl_seconds", "max_concurrent_activities"]; + +/// Each concrete attribute a rule key names in `value`, with its value when +/// present: `*` takes every element in order; a missing segment ends the +/// expansion unless no `*` follows it, when the attribute is absent. +fn expand<'a>(value: Option<&'a Value>, segments: &[&str], name: String, out: &mut Vec<(String, Option<&'a Value>)>) { + let Some((segment, rest)) = segments.split_first() else { + out.push((name, value)); + return; + }; + let join = |key: &str| if name.is_empty() { key.to_owned() } else { format!("{name}.{key}") }; + if *segment == "*" { + match value { + Some(Value::Array(items)) => { + for (index, item) in items.iter().enumerate() { + expand(Some(item), rest, join(&index.to_string()), out); + } + } + Some(Value::Object(items)) => { + for (key, item) in items { + expand(Some(item), rest, join(key), out); + } + } + _ => {} + } + return; + } + let child = value.and_then(Value::as_object).and_then(|fields| fields.get(*segment)); + if child.is_none() && rest.contains(&"*") { + return; + } + expand(child, rest, join(segment), out); +} + +/// One attribute's messages, as Laravel's `validateAttribute` gives them. +fn messages(display: &str, checks: &[Check], value: Option<&Value>, other: Option<(String, bool)>) -> Vec { + let has = |wanted: fn(&Check) -> bool| checks.iter().any(wanted); + if has(|check| matches!(check, Check::Sometimes)) && value.is_none() { + return Vec::new(); + } + let nullable = has(|check| matches!(check, Check::Rule(Rule::Nullable))); + let numeric = has(|check| matches!(check, Check::Rule(Rule::Integer))); + let array = has(|check| matches!(check, Check::Rule(Rule::Array) | Check::Keys(_))); + let mut found = Vec::new(); + for check in checks { + let implicit = matches!(check, Check::Rule(Rule::Required) | Check::RequiredWith(_)); + if !implicit && (value.is_none() || nullable && value.is_some_and(Value::is_null)) { + continue; + } + let failed = match *check { + Check::Sometimes | Check::Rule(Rule::Nullable) => None, + Check::Rule(rule) => (!passes(rule, value, numeric)).then(|| default_message(display, rule, numeric, array)), + Check::Prohibited => present(value).then(|| format!("The {display} field is prohibited.")), + Check::RequiredWith(_) => match &other { + Some((other, true)) if !present(value) => Some(format!("The {display} field is required when {other} is present.")), + _ => None, + }, + Check::Keys(keys) => { + let fits = match value { + Some(Value::Object(fields)) => fields.keys().all(|key| keys.contains(&key.as_str())), + Some(Value::Array(items)) => items.is_empty(), + _ => false, + }; + (!fits).then(|| default_message(display, Rule::Array, numeric, array)) + } + }; + if let Some(message) = failed { + found.push(message); + if implicit { + break; + } + } + } + found +} + +/// The field a `required_with` names, shown as Laravel shows it, and +/// whether it is filled. +fn other_field(checks: &[Check], scope: Option<&Value>, prefix: &str) -> Option<(String, bool)> { + checks.iter().find_map(|check| match check { + Check::RequiredWith(other) => { + let shown = if prefix.is_empty() { other.replace('_', " ") } else { format!("{prefix}.{other}") }; + Some((shown, present(scope.and_then(|scope| scope.get(*other))))) + } + _ => None, + }) +} + +/// The errors for one set of rules over one value, rule by rule. +fn errors(rules: &[(&str, &[Check])], scope: Option<&Value>, prefix: &str, found: &mut Vec<(String, Vec)>) { + for (key, checks) in rules { + let segments: Vec<&str> = key.split('.').collect(); + let mut attributes = Vec::new(); + expand(scope, &segments, prefix.to_owned(), &mut attributes); + let other = other_field(checks, scope, prefix); + for (name, value) in attributes { + // A wildcard attribute is named as written; a named one with spaces. + let display = if prefix.is_empty() && !key.contains('*') { name.replace('_', " ") } else { name.clone() }; + let messages = messages(&display, checks, value, other.clone()); + if !messages.is_empty() && !found.iter().any(|(seen, _)| *seen == name) { + found.push((name, messages)); + } + } + } +} + +/// PHP's `normalizeValidatedInteger`: a digit string becomes an integer. +fn whole(value: &mut Value) { + if let Value::String(text) = value { + let digits = text.strip_prefix('-').unwrap_or(text); + if !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) && let Ok(number) = text.parse::() { + *value = Value::from(number); + } + } +} + +/// Validates the completion as PHP's transport rules do, then normalizes its +/// integer fields. +pub(super) fn checked(body: Value) -> Result { + // Laravel's TrimStrings and ConvertEmptyStringsToNull run first. + let mut body = super::start_request::normalized(body); + let mut found = Vec::new(); + errors(TOP, Some(&body), "", &mut found); + if let Some(commands) = body["commands"].as_array() { + for (chunk, commands) in commands.chunks(CHUNK).enumerate() { + let mut chunk_errors = Vec::new(); + for (key, checks) in COMMAND { + for (offset, command) in commands.iter().enumerate() { + let index = chunk * CHUNK + offset; + errors(&[(*key, *checks)], Some(command), &format!("commands.{index}"), &mut chunk_errors); + } + } + for (name, messages) in chunk_errors { + if !found.iter().any(|(seen, _)| *seen == name) { + found.push((name, messages)); + } + } + } + } + if let Some(error) = super::laravel_validation::failure_messages(found) { + return Err(error); + } + if let Some(attempt) = body.get("workflow_task_attempt").and_then(integer) { + body["workflow_task_attempt"] = Value::from(attempt); + } + if let Some(commands) = body["commands"].as_array_mut() { + for command in commands.iter_mut().filter_map(Value::as_object_mut) { + for field in INTEGER_FIELDS { + if let Some(value) = command.get_mut(*field) { + whole(value); + } + } + if let Some(session) = command.get_mut("worker_session").and_then(Value::as_object_mut) { + for field in SESSION_INTEGER_FIELDS { + if let Some(value) = session.get_mut(*field) { + whole(value); + } + } + } + if let Some(policy) = command.get_mut("retry_policy").and_then(Value::as_object_mut) { + if let Some(value) = policy.get_mut("max_attempts") { + whole(value); + } + if let Some(Value::Array(backoff)) = policy.get_mut("backoff_seconds") { + backoff.iter_mut().for_each(whole); + } + } + } + } + if let Some(commands) = body["commands"].as_array() + && let Some(error) = super::laravel_validation::failure_messages(controller_scopes(commands)) + { + return Err(error); + } + Ok(body) +} + +fn has_value(command: &Value, field: &str) -> bool { + command.get(field).is_some_and(|value| !value.is_null()) +} + +/// `WorkerController::validateWorkflowTaskCommandScopes`, which PHP runs +/// after the integer normalization and before any guard. +fn controller_scopes(commands: &[Value]) -> Vec<(String, Vec)> { + let mut found: Vec<(String, Vec)> = Vec::new(); + let mut add = |field: String, message: String| match found.iter_mut().find(|(seen, _)| *seen == field) { + Some((_, messages)) => messages.push(message), + None => found.push((field, vec![message])), + }; + let int = |command: &Value, field: &str| command.get(field).and_then(Value::as_i64); + for (index, command) in commands.iter().enumerate() { + let Some(kind) = command.get("type").and_then(Value::as_str) else { + continue; + }; + let at = |field: &str| format!("commands.{index}.{field}"); + if has_value(command, "retry_policy") + && !matches!(kind, "schedule_activity" | "record_local_activity" | "prepare_local_activity" | "start_child_workflow") + { + add(at("retry_policy"), "retry_policy is only supported for schedule_activity and start_child_workflow commands.".into()); + } + for field in ["start_to_close_timeout", "schedule_to_start_timeout", "schedule_to_close_timeout", "heartbeat_timeout"] { + if has_value(command, field) && !matches!(kind, "schedule_activity" | "record_local_activity" | "prepare_local_activity") { + add(at(field), format!("{field} is only supported for schedule_activity commands.")); + } + } + if has_value(command, "worker_session") && kind != "schedule_activity" { + add(at("worker_session"), "worker_session is only supported for schedule_activity commands.".into()); + } + for field in ["execution_timeout_seconds", "run_timeout_seconds"] { + if has_value(command, field) && kind != "start_child_workflow" { + add(at(field), format!("{field} is only supported for start_child_workflow commands.")); + } + } + if has_value(command, "cancellation_policy") + && !matches!(kind, "start_child_workflow" | "schedule_activity" | "prepare_local_activity") + { + add(at("cancellation_policy"), + "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands.".into()); + } + if has_value(command, "non_retryable") && !matches!(kind, "fail_workflow" | "fail_update" | "record_local_activity") { + add(at("non_retryable"), "non_retryable is only supported for fail_workflow and fail_update commands.".into()); + } + if has_value(command, "exception") && kind != "fail_workflow" { + add(at("exception"), "exception is only supported for fail_workflow commands.".into()); + } + for field in ["failed_step_sequence", "failed_activity_execution_id"] { + if has_value(command, field) && kind != "fail_workflow" { + add(at(field), format!("{field} is only supported for fail_workflow commands.")); + } + } + if kind == "fail_workflow" && has_value(command, "failed_step_sequence") != has_value(command, "failed_activity_execution_id") { + add(at("failed_step_sequence"), + "Uncaught activity failure identity requires both failed_step_sequence and failed_activity_execution_id.".into()); + } + if matches!(kind, "schedule_activity" | "record_local_activity" | "prepare_local_activity") { + // `validateActivityTimeoutEnvelope`. + let (start, queue, close, heartbeat) = (int(command, "start_to_close_timeout"), int(command, "schedule_to_start_timeout"), + int(command, "schedule_to_close_timeout"), int(command, "heartbeat_timeout")); + if let (Some(heartbeat), Some(start)) = (heartbeat, start) && heartbeat > start { + add(at("heartbeat_timeout"), "heartbeat_timeout cannot exceed start_to_close_timeout.".into()); + } + if let (Some(start), Some(close)) = (start, close) && start > close { + add(at("start_to_close_timeout"), "start_to_close_timeout cannot exceed schedule_to_close_timeout.".into()); + } + if let (Some(queue), Some(close)) = (queue, close) && queue > close { + add(at("schedule_to_start_timeout"), "schedule_to_start_timeout cannot exceed schedule_to_close_timeout.".into()); + } + } + if kind == "start_child_workflow" + && let (Some(execution), Some(run)) = (int(command, "execution_timeout_seconds"), int(command, "run_timeout_seconds")) + && run > execution + { + add(at("run_timeout_seconds"), "run_timeout_seconds cannot exceed execution_timeout_seconds.".into()); + } + } + found +} + +/// `WorkflowCommandNormalizer::FIELD_SCOPES`: each field, the commands it +/// applies to, and the guidance PHP gives elsewhere. +const FIELD_SCOPES: &[(&str, &[&str], &str)] = &[ + ("retry_policy", &["schedule_activity", "record_local_activity", "start_child_workflow"], + "Configure activity retries on a schedule_activity command, or child workflow retries on a start_child_workflow command. Workflow failure itself is non-retryable, and SDK HTTP transport retry is a client concern that does not appear in the workflow task command stream."), + ("start_to_close_timeout", &["schedule_activity", "record_local_activity"], + "start_to_close_timeout limits one activity attempt and only applies to a schedule_activity command."), + ("schedule_to_start_timeout", &["schedule_activity"], + "schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command."), + ("schedule_to_close_timeout", &["schedule_activity", "record_local_activity"], + "schedule_to_close_timeout limits the entire activity execution including retries and only applies to a schedule_activity command."), + ("heartbeat_timeout", &["schedule_activity", "record_local_activity"], + "heartbeat_timeout limits the gap between activity heartbeats and only applies to a schedule_activity command."), + ("worker_session", &["schedule_activity"], + "worker_session pins a sequence of activity attempts to one worker and only applies to a schedule_activity command."), + ("execution_timeout_seconds", &["start_child_workflow"], + "execution_timeout_seconds limits the entire child workflow execution and only applies to a start_child_workflow command."), + ("run_timeout_seconds", &["start_child_workflow"], + "run_timeout_seconds limits one child workflow run and only applies to a start_child_workflow command."), + ("non_retryable", &["fail_workflow", "fail_update", "record_local_activity"], + "non_retryable marks a workflow, update, or recorded local-activity failure as non-retryable. Durable activity non-retryable error types belong inside the schedule_activity retry_policy.non_retryable_error_types list."), + ("timeout_kind", &["record_local_activity"], + "timeout_kind identifies the bounded timeout that ended a recorded local activity and only applies to a record_local_activity command."), + ("exception", &["fail_workflow"], + "exception carries the structured terminal workflow failure payload and only applies to a fail_workflow command."), + ("failed_step_sequence", &["fail_workflow"], + "failed_step_sequence identifies an uncaught activity failure and only applies to a fail_workflow command."), + ("failed_activity_execution_id", &["fail_workflow"], + "failed_activity_execution_id identifies an uncaught activity failure and only applies to a fail_workflow command."), + ("parent_close_policy", &["start_child_workflow"], + "parent_close_policy declares how a child workflow reacts when its parent closes and only applies to a start_child_workflow command."), + ("cancellation_policy", &["start_child_workflow", "schedule_activity"], + "cancellation_policy declares how an awaiting workflow handles child or remote activity cancellation. Explicit activity policies require protocol 1.20 and remote abandon requires a finite schedule_to_close_timeout."), + ("cancellation_scope_id", &["schedule_activity", "start_timer", "start_child_workflow", "open_signal_wait", "open_condition_wait"], + "cancellation_scope_id records activity, timer, child or wait membership and requires candidate protocol 1.20."), + ("cancellation_cleanup", &["start_timer"], + "Shielded scoped timers require the original scope, request and delivery identities under candidate protocol 1.20."), + ("delay_seconds", &["start_timer"], "delay_seconds is the timer delay and only applies to a start_timer command."), + ("timeout_seconds", &["open_condition_wait", "open_signal_wait"], + "timeout_seconds only applies to open_condition_wait or open_signal_wait. For activities use start_to_close_timeout / schedule_to_start_timeout / schedule_to_close_timeout / heartbeat_timeout; for child workflows use execution_timeout_seconds / run_timeout_seconds."), + ("payload_codec", &["complete_workflow", "schedule_activity", "start_child_workflow", "continue_as_new", "complete_update", + "record_side_effect", "record_local_activity", "start_service_operation"], + "payload_codec identifies the codec used for command payload bytes and only applies to commands that carry result, arguments, or request_payload bytes."), + ("entries", &["upsert_memo"], "entries contains the non-indexed memo patch and only applies to an upsert_memo command."), +]; + +/// The command normalizer's per-command checks native shares with PHP: +/// `assertCommandFieldScope`, then the timer, version marker and wait rules. +/// PHP collects these across the batch; native stops at the first command. +pub(super) fn normalizer_checks(index: usize, command: &Value) -> Result<()> { + let Some(kind) = command.get("type").and_then(Value::as_str) else { + return Ok(()); + }; + let mut found: Vec<(String, Vec)> = Vec::new(); + for (field, allowed, guidance) in FIELD_SCOPES { + // A marker's wire shape is frozen; PHP drops its payload codec. + if !has_value(command, field) || allowed.contains(&kind) || kind == "record_version_marker" && *field == "payload_codec" { + continue; + } + found.push((format!("commands.{index}.{field}"), + vec![format!("Workflow task command field [{field}] is not valid on a {kind} command. {guidance}")])); + } + let whole = |field: &str| command.get(field).and_then(Value::as_i64); + match kind { + "start_timer" if !whole("delay_seconds").is_some_and(|delay| delay >= 0) => found.push(( + format!("commands.{index}.delay_seconds"), + vec!["Start timer commands require a non-negative integer delay_seconds.".into()], + )), + "open_signal_wait" | "open_condition_wait" + if has_value(command, "timeout_seconds") && !whole("timeout_seconds").is_some_and(|seconds| seconds >= 0) => + { + let wait = if kind == "open_signal_wait" { "signal" } else { "condition" }; + found.push((format!("commands.{index}.timeout_seconds"), + vec![format!("Open {wait} wait timeout_seconds must be a non-negative integer when provided.")])); + } + "record_version_marker" => { + let mut messages = Vec::new(); + if !command.get("change_id").and_then(Value::as_str).is_some_and(|id| !id.trim().is_empty()) { + messages.push("change_id is required".to_owned()); + } + for field in ["version", "min_supported", "max_supported"] { + if whole(field).is_none() { + messages.push(format!("{field} must be an integer")); + } + } + if !messages.is_empty() { + found.push((format!("commands.{index}"), messages)); + } + } + _ => {} + } + match super::laravel_validation::failure_messages(found) { + Some(error) => Err(error), + None => Ok(()), + } +} + +/// The payload fields whose bare string PHP's normalizer takes as the blob +/// itself, in the run's codec (`resolveCommandPayloadWithCodec`). +const BLOB_FIELDS: &[(&str, &str)] = &[ + ("complete_workflow", "result"), + ("schedule_activity", "arguments"), + ("start_child_workflow", "arguments"), + ("continue_as_new", "arguments"), + ("complete_update", "result"), +]; + +/// The fields `WorkflowCommandNormalizer::normalize` reads for each command +/// type native runs. PHP ignores any other field, apart from the ones its +/// scope checks refuse on the wrong command. +const READS: &[(&str, &[&str])] = &[ + ("complete_workflow", &["result", "payload_codec"]), + ("fail_workflow", &["message", "exception_class", "exception_type", "non_retryable", "exception", + "failed_step_sequence", "failed_activity_execution_id"]), + ("schedule_activity", &["activity_type", "arguments", "payload_codec", "connection", "queue", "retry_policy", + "start_to_close_timeout", "schedule_to_start_timeout", "schedule_to_close_timeout", "heartbeat_timeout", + "cancellation_policy", "worker_session"]), + ("start_timer", &["delay_seconds", "cancellation_cleanup"]), + ("start_child_workflow", &["workflow_type", "arguments", "payload_codec", "connection", "queue", "retry_policy", + "execution_timeout_seconds", "run_timeout_seconds", "parent_close_policy", "cancellation_policy"]), + ("continue_as_new", &["arguments", "payload_codec", "workflow_type", "queue"]), + ("complete_update", &["update_id", "result", "payload_codec"]), + ("fail_update", &["update_id", "message", "exception_class", "exception_type", "non_retryable"]), + ("record_side_effect", &["result", "payload_codec"]), + ("record_version_marker", &["change_id", "version", "min_supported", "max_supported"]), + ("upsert_search_attributes", &["attributes", "attribute_types"]), + ("upsert_memo", &["entries"]), + ("open_condition_wait", &["condition_key", "condition_definition_fingerprint", "condition_wait_occurrence_id", "timeout_seconds"]), + ("open_signal_wait", &["signal_name", "timeout_seconds"]), +]; + +/// `parallelMetadataValidationRules`' fields, read for the leaf commands. +const PARALLEL_FIELDS: &[&str] = &["parallel_group_id", "parallel_group_kind", "parallel_group_mode", + "parallel_group_base_sequence", "parallel_group_size", "parallel_group_index", "selection_member_key", + "selection_member_index", "selection_member_base_sequence", "selection_member_size", "selection_member_kind", + "parallel_group_path"]; +const PARALLEL_LEAVES: &[&str] = &["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait"]; + +/// Fields a scope check refuses on the wrong command, so they must reach it. +const SCOPED_FIELDS: &[&str] = &["retry_policy", "start_to_close_timeout", "schedule_to_start_timeout", "schedule_to_close_timeout", + "heartbeat_timeout", "worker_session", "execution_timeout_seconds", "run_timeout_seconds", "cancellation_policy", + "non_retryable", "exception", "failed_step_sequence", "failed_activity_execution_id", "timeout_kind", + "parent_close_policy", "cancellation_scope_id", "cancellation_cleanup", "delay_seconds", "timeout_seconds", + "payload_codec", "entries", "principal_subject", "principal_method", "principal_roles", "principal_tenant", + "principal_claims"]; + +/// `optionalRetryPolicy` reads these keys and ignores the rest. +const RETRY_POLICY_FIELDS: &[&str] = &["max_attempts", "backoff_seconds", "non_retryable_error_types"]; + +/// Drops what PHP ignores before native's own field checks: on a command +/// type native runs, a field PHP's normalizer neither reads nor scope-checks, +/// and an unknown retry policy key. A field PHP reads stays, so native still +/// refuses one it cannot run. +fn ignored_fields(command: &mut Value) { + let Some(kind) = command.get("type").and_then(Value::as_str).map(str::to_owned) else { + return; + }; + let Some((_, reads)) = READS.iter().find(|(owner, _)| *owner == kind) else { + return; + }; + let Some(fields) = command.as_object_mut() else { + return; + }; + let parallel = PARALLEL_LEAVES.contains(&kind.as_str()); + fields.retain(|field, value| { + field == "type" + || reads.contains(&field.as_str()) + || parallel && PARALLEL_FIELDS.contains(&field.as_str()) + || SCOPED_FIELDS.contains(&field.as_str()) && !value.is_null() + }); + if let Some(Value::Object(policy)) = fields.get_mut("retry_policy") { + policy.retain(|key, _| RETRY_POLICY_FIELDS.contains(&key.as_str())); + } +} + +/// Gives each payload the envelope form native applies, as PHP's normalizer +/// reads it: a bare string is an Avro blob, an empty one is no payload, and +/// continue-as-new arguments that are plain values are no arguments, so the +/// run continues with its own. Runs after the payload checks, which name the +/// field as the worker sent it. +pub(super) fn canonical_payloads(mut body: Value) -> Result { + let Some(commands) = body["commands"].as_array_mut() else { + return Ok(body); + }; + for (index, command) in commands.iter_mut().enumerate() { + ignored_fields(command); + let Some(kind) = command["type"].as_str().map(str::to_owned) else { + continue; + }; + let avro = command.get("payload_codec").is_none_or(|codec| codec.is_null() || codec == "avro"); + for (_, field) in BLOB_FIELDS.iter().filter(|(owner, _)| *owner == kind) { + match command.get(*field) { + Some(Value::String(blob)) if blob.is_empty() => command[*field] = Value::Null, + Some(Value::String(blob)) if avro => command[*field] = serde_json::json!({"codec":"avro","blob":blob}), + _ => {} + } + } + if matches!(kind.as_str(), "continue_as_new" | "start_child_workflow") { + match command.get("arguments") { + Some(Value::Array(_)) => command["arguments"] = Value::Null, + Some(Value::Object(fields)) if !fields.contains_key("codec") => command["arguments"] = Value::Null, + Some(Value::Number(_) | Value::Bool(_)) => { + return Err(super::command_payloads::invalid( + &format!("commands.{index}.arguments"), + "Workflow task command field [arguments] must be a string or a payload envelope when provided.", + )); + } + _ => {} + } + } + } + Ok(body) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn refusal(body: Value) -> Value { + match checked(body) { + Err(super::super::RuntimeError::Protocol { response, .. }) => response, + other => panic!("{other:?}"), + } + } + + fn base(commands: Value) -> Value { + json!({"lease_owner":"w","workflow_task_attempt":1,"commands":commands}) + } + + #[test] + fn errors_follow_php_s_rule_order_names_and_chunks() { + let response = refusal(base(json!([{"type":"schedule_activity","attempts":[{}, {"attempt_number":0,"outcome":"x", + "heartbeats":[{"elapsed_ms":-1,"details":"x"}]}]}]))); + assert_eq!(response["message"], "The commands.0.attempts.0.attempt_number field is required. (and 5 more errors)"); + assert_eq!(response["errors"]["commands.0.attempts.1.outcome"], json!(["The selected commands.0.attempts.1.outcome is invalid."])); + let response = refusal(json!({"lease_owner":"w","workflow_task_attempt":1,"commands":[{"type":"start_timer","delay_seconds":1}], + "sticky_cache":{"worker_id":"w"}})); + assert_eq!(response["message"], "The sticky cache.workflow id field is required when sticky cache is present. (and 3 more errors)"); + let response = refusal(base(json!([{"type":"start_timer","delay_seconds":1,"cancellation_cleanup":{"scope_id":"a"}}]))); + assert_eq!(response["errors"]["commands.0.cancellation_cleanup.request_id"], + json!(["The commands.0.cancellation_cleanup.request_id field is required when commands.0.cancellation_cleanup is present."])); + let response = refusal(base(json!([{"type":"start_timer","delay_seconds":1,"cancellation_cleanup": + {"scope_id":"a","request_id":"b","delivery_history_event_id":"c","extra":1}}]))); + assert_eq!(response["message"], "The commands.0.cancellation_cleanup field must be an array."); + let response = refusal(base(json!([{"type":"start_timer","delay_seconds":1,"cancellation_scope_id":null}]))); + assert_eq!(response["errors"]["commands.0.cancellation_scope_id"], json!(["The commands.0.cancellation_scope_id field must be a string.", + "The commands.0.cancellation_scope_id field must be at least 1 characters."])); + let response = refusal(base(json!([{"type":"complete_workflow","principal_subject":"x","principal_roles":null}]))); + assert_eq!(response["message"], "The commands.0.principal_subject field is prohibited."); + // A later chunk's errors follow the earlier chunk's, whatever the rule. + let mut commands = vec![json!({"type":"record_side_effect"}); 30]; + commands[3]["member_index"] = json!(-1); + commands[26] = json!({"type":5}); + commands[29] = json!({"type":"start_timer","delay_seconds":-1}); + let response = refusal(base(Value::Array(commands))); + assert_eq!(response["message"], "The commands.3.member_index field must be at least 0. (and 2 more errors)"); + assert_eq!(refusal(base(json!([])))["message"], "The commands field is required."); + assert_eq!(refusal(base(json!(["x"])))["message"], "The commands.0.type field is required."); + assert_eq!(refusal(json!({"workflow_task_attempt":"x","commands":[{"type":"a"}]}))["message"], + "The lease owner field is required. (and 1 more error)"); + } + + #[test] + fn digit_strings_become_integers_once_the_rules_pass() { + let body = checked(json!({"lease_owner":"w","workflow_task_attempt":"2","commands":[{"type":"start_timer","delay_seconds":"7"}, + {"type":"schedule_activity","heartbeat_timeout":"10","retry_policy":{"max_attempts":"3","backoff_seconds":["1",2]}, + "worker_session":{"ttl_seconds":"30"}},{"type":"start_timer","delay_seconds":5.0}],"unknown":1})).unwrap(); + assert_eq!(body["workflow_task_attempt"], 2); + assert_eq!(body["commands"][0]["delay_seconds"], 7); + assert_eq!(body["commands"][1]["heartbeat_timeout"], 10); + assert_eq!(body["commands"][1]["retry_policy"], json!({"max_attempts":3,"backoff_seconds":[1,2]})); + assert_eq!(body["commands"][1]["worker_session"]["ttl_seconds"], 30); + // A float stays a float, as PHP leaves it. + assert_eq!(body["commands"][2]["delay_seconds"], 5.0); + } +} diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 9586ffb40..7a5d4d62c 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -87,19 +87,9 @@ where task: &DB::Row, command: &Value, sequence: i64, - ) -> Result<()> { + ) -> Result { let old = DB::string(run, "id")?; let workflow = DB::string(run, "workflow_instance_id")?; - for field in ["workflow_type", "queue"] { - if command.get(field).is_some_and(|value| !value.is_null()) - && super::text(command, field)? != DB::string(run, field)? - { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "continue_as_new_routing_unqualified", - )); - } - } let instance = Self::query( "SELECT current_run_id FROM workflow_instances WHERE id=$1 AND namespace=$2", ) @@ -113,23 +103,11 @@ where "continue_as_new_run_not_current", )); } + // Published PHP continues a run whatever it has open. Its activities, + // timers, children and waits stay with the closed predecessor, and no + // parent-close policy applies. Streams, service calls and live + // queries are not transferred by native yet. for (sql, selected) in [ - ( - "SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$1 AND status NOT IN ('completed','failed','cancelled')", - old.as_str(), - ), - ( - "SELECT COUNT(*) FROM workflow_run_timers WHERE workflow_run_id=$1 AND status='pending'", - old.as_str(), - ), - ( - "SELECT COUNT(*) FROM workflow_child_calls WHERE parent_workflow_run_id=$1", - old.as_str(), - ), - ( - "SELECT COUNT(*) FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow'", - old.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), @@ -159,12 +137,6 @@ where )); } } - if Self::open_wait(tx, &old).await?.is_some() { - return Err(refuse( - StatusCode::CONFLICT, - "continue_as_new_transfer_unqualified", - )); - } // Select this run before applying the bound. Receipt retention does // not extend original query execution, and unrelated cache prefixes // cannot hide a live claim. Read timing only, never history snapshots. @@ -246,11 +218,19 @@ where } else { DB::string(run, "arguments")? }; - let kind = DB::string(run, "workflow_type")?; - let class = DB::string(run, "workflow_class")?; - let queue = DB::string(run, "queue")?; - Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,connection,queue,business_key,visibility_labels,compatibility,run_timeout_seconds,execution_deadline_at,run_deadline_at,started_at,created_at,updated_at,namespace,last_progress_at) SELECT $1,workflow_instance_id,workflow_type,workflow_class,$2,'pending',payload_codec,$3,connection,queue,business_key,visibility_labels,compatibility,run_timeout_seconds,execution_deadline_at,$4,$5,$6,$7,namespace,$8 FROM workflow_runs WHERE id=$9") - .bind(&next).bind(number).bind(&arguments).bind(DB::optional_time(deadline)) + // `applyContinueAsNew`: the command may name a new workflow type and + // queue. The type also becomes the run's and the instance's class. + let routed = |field: &str| -> Result { + match command.get(field) { + Some(value) if !value.is_null() => Ok(super::text(command, field)?.to_owned()), + _ => DB::string(run, field), + } + }; + let kind = routed("workflow_type")?; + let class = kind.clone(); + let queue = routed("queue")?; + Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,connection,queue,business_key,visibility_labels,compatibility,run_timeout_seconds,execution_deadline_at,run_deadline_at,started_at,created_at,updated_at,namespace,last_progress_at) SELECT $1,workflow_instance_id,$2,$3,$4,'pending',payload_codec,$5,connection,$6,business_key,visibility_labels,compatibility,run_timeout_seconds,execution_deadline_at,$7,$8,$9,$10,namespace,$11 FROM workflow_runs WHERE id=$12") + .bind(&next).bind(&kind).bind(&class).bind(number).bind(&arguments).bind(&queue).bind(DB::optional_time(deadline)) .bind(DB::bind_time(started)).bind(DB::bind_time(started)) .bind(DB::bind_time(started)).bind(DB::bind_time(started)).bind(&old) .execute(&mut **tx).await?; @@ -275,13 +255,16 @@ where .bind(&old) .execute(&mut **tx) .await?; - Self::query("UPDATE workflow_instances SET current_run_id=$1,run_count=$2,updated_at=$3 WHERE id=$4") - .bind(&next).bind(number).bind(DB::bind_time(started)).bind(&workflow).execute(&mut **tx).await?; + Self::query("UPDATE workflow_instances SET current_run_id=$1,run_count=$2,workflow_class=$3,updated_at=$4 WHERE id=$5") + .bind(&next).bind(number).bind(&class).bind(DB::bind_time(started)).bind(&workflow).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at) VALUES ($1,'continue_as_new',$2,$3,$4,$5,$6,TRUE,'abandon',$7,$8)") .bind(&link).bind(sequence).bind(&workflow).bind(&old).bind(&workflow).bind(&next) .bind(DB::bind_time(started)).bind(DB::bind_time(started)).execute(&mut **tx).await?; + let parent = Self::continue_child_links(tx, &old, &next, number, &class, &kind, started).await?; Self::append(tx,&next,"WorkflowStarted",json!({"workflow_class":class,"workflow_type":kind, - "workflow_instance_id":workflow,"workflow_run_id":next,"continued_from_run_id":old,"workflow_link_id":link}),None,None).await?; + "workflow_instance_id":workflow,"workflow_run_id":next,"continued_from_run_id":old,"workflow_link_id":link, + "child_call_id":parent["child_call_id"],"parent_workflow_instance_id":parent["parent_workflow_instance_id"], + "parent_workflow_run_id":parent["parent_workflow_run_id"],"parent_sequence":parent["parent_sequence"]}),None,None).await?; Self::transfer_messages(tx, &old, &next, &workflow).await?; Self::query("UPDATE workflow_runs SET status='completed',closed_reason='continued',closed_at=$1,last_progress_at=$2 WHERE id=$3") .bind(DB::bind_time(started)).bind(DB::bind_time(started)).bind(&old).execute(&mut **tx).await?; @@ -291,8 +274,85 @@ where // `PendingUpdateCloser::closeForTerminalRun`: updates already applied // but not completed stay with the predecessor and close there. Self::close_pending_updates(tx, &old, Some(task)).await?; - Self::create_task(tx, &next, &queue, "workflow", json!({})).await?; - Ok(()) + Self::create_task(tx, &next, &queue, "workflow", json!({})).await + } + + /// `applyContinueAsNew` for a child run: the continued run stays its + /// parent's child. Each parent link gets a link to the continued run. An + /// open parent's child call follows it (`markChildCallContinued`), and + /// the parent records `ChildRunStarted` for it. Returns the parent + /// reference for the continued run's `WorkflowStarted`, all null for a + /// run without a parent. + #[allow(clippy::too_many_arguments)] + async fn continue_child_links( + tx: &mut Transaction<'_, DB>, + old: &str, + next: &str, + number: i64, + class: &str, + kind: &str, + started: chrono::DateTime, + ) -> Result { + let original = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") + .bind(old).fetch_optional(&mut **tx).await? + .map(|row| DB::document_row(&row, "payload")) + .transpose()? + .unwrap_or(Value::Null); + let call_id = original["child_call_id"].as_str().filter(|id| !id.is_empty()).map(str::to_owned); + let reference = match original["parent_workflow_run_id"].as_str().filter(|id| !id.is_empty()) { + None => json!({"child_call_id":call_id,"parent_workflow_instance_id":null,"parent_workflow_run_id":null,"parent_sequence":null}), + Some(parent_run) => json!({"child_call_id":call_id, + "parent_workflow_instance_id":original["parent_workflow_instance_id"].as_str().unwrap_or(&original["workflow_instance_id"].as_str().unwrap_or_default()), + "parent_workflow_run_id":parent_run,"parent_sequence":original["parent_sequence"].as_i64()}), + }; + let links = Self::query("SELECT id,sequence,parent_workflow_run_id,child_workflow_instance_id,parent_close_policy FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow' ORDER BY id") + .bind(old).fetch_all(&mut **tx).await?; + for parent_link in links { + let link = id(); + Self::query("INSERT INTO workflow_links(id,link_type,sequence,parallel_group_path,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at) SELECT $1,link_type,sequence,parallel_group_path,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,$2,is_primary_parent,parent_close_policy,$3,$4 FROM workflow_links WHERE id=$5") + .bind(&link).bind(next).bind(DB::bind_time(started)).bind(DB::bind_time(started)) + .bind(DB::string(&parent_link, "id")?).execute(&mut **tx).await?; + let parent_run = DB::optional_string(&parent_link, "parent_workflow_run_id")?.filter(|id| !id.is_empty()); + let (Some(parent_run), Some(sequence)) = (parent_run, DB::optional_number(&parent_link, "sequence")?) else { + continue; + }; + let open = Self::query("SELECT status FROM workflow_runs WHERE id=$1") + .bind(&parent_run).fetch_optional(&mut **tx).await? + .map(|row| DB::string(&row, "status")) + .transpose()? + .is_some_and(|status| matches!(status.as_str(), "pending" | "running" | "waiting")); + if !open { + continue; + } + let child_instance = DB::string(&parent_link, "child_workflow_instance_id")?; + if let Some(call) = Self::query("SELECT id,metadata FROM workflow_child_calls WHERE parent_workflow_run_id=$1 AND sequence=$2 AND status NOT IN ('completed','cancelled','terminated','failed') ORDER BY id DESC LIMIT 1") + .bind(&parent_run).bind(sequence).fetch_optional(&mut **tx).await? + { + let mut metadata = DB::optional_document_row(&call, "metadata")? + .filter(Value::is_object) + .unwrap_or_else(|| json!({})); + let continued = metadata["continuation_count"].as_i64().unwrap_or(0) + 1; + if let Some(call_id) = &call_id { + metadata["child_call_id"] = json!(call_id); + } + metadata["continuation_count"] = json!(continued); + metadata["last_continued_from_child_workflow_run_id"] = json!(old); + Self::query("UPDATE workflow_child_calls SET resolved_child_instance_id=$1,resolved_child_run_id=$2,status='started',started_at=$3,metadata=$4,updated_at=$5 WHERE id=$6") + .bind(&child_instance).bind(next).bind(DB::bind_time(started)).bind(DB::document(&metadata)) + .bind(DB::bind_time(started)).bind(DB::number(&call, "id")?).execute(&mut **tx).await?; + } + let mut payload = json!({"sequence":sequence,"workflow_link_id":link,"child_call_id":call_id, + "child_workflow_instance_id":child_instance,"child_workflow_run_id":next, + "child_workflow_class":class,"child_workflow_type":kind,"child_run_number":number, + "parent_close_policy":DB::string(&parent_link, "parent_close_policy")?}); + if call_id.is_none() { + payload.as_object_mut().expect("an object").remove("child_call_id"); + } + // PHP projects the parent again (`projectRun`). + tx.summary_changed(&parent_run); + Self::append(tx, &parent_run, "ChildRunStarted", payload, None, None).await?; + } + Ok(reference) } /// `ContinuedRunMessageHandoff::transferInstanceScoped`: received signals diff --git a/rust/src/runtime/control_plane.rs b/rust/src/runtime/control_plane.rs index 56d5a37b4..7d0bfdb6d 100644 --- a/rust/src/runtime/control_plane.rs +++ b/rust/src/runtime/control_plane.rs @@ -13,6 +13,8 @@ pub(super) enum Operation<'a> { History, Cancel, Terminate, + Archive, + Redrive, Repair, Debug, Update(&'a str), @@ -28,6 +30,13 @@ impl Operation<'_> { run_id: Option<&str>, ) -> super::RuntimeError { let (status, mut body) = match error { + // A typed external payload error gets the route's envelope once, + // from the guard. + super::RuntimeError::Protocol { ref response, .. } + if response["schema"] == super::external_transport::TYPED_SCHEMA => + { + return error; + } super::RuntimeError::Refused { status, reason } if status == axum::http::StatusCode::NOT_FOUND && matches!(reason, "instance_not_found" | "run_not_found") => @@ -108,6 +117,8 @@ impl Operation<'_> { Self::History => ("history", &["workflow_id", "run_id"], &["next_page_token"]), Self::Cancel => ("cancel", &["workflow_id"], &["outcome"]), Self::Terminate => ("terminate", &["workflow_id"], &["outcome"]), + Self::Archive => ("archive", &["workflow_id"], &["outcome"]), + Self::Redrive => ("redrive", &["workflow_id", "continued_from_run_id"], &["run_id", "resume_step_sequence", "outcome"]), Self::Repair => ("repair", &["workflow_id"], &["outcome"]), Self::Debug => ("debug_workflow", &["workflow_id"], &["run_id", "diagnostic_status"]), Self::Update(_) => ( @@ -133,7 +144,10 @@ impl Operation<'_> { "required_fields":required, "success_fields":success}}); if let Some((fields, reasons)) = self.rejection_contract() { metadata["contract"]["rejection_fields"] = words(fields); - metadata["contract"]["rejection_reasons"] = words(reasons); + // PHP's redrive contract names no rejection reasons. + if !reasons.is_empty() { + metadata["contract"]["rejection_reasons"] = words(reasons); + } } match self { Self::Update(name) => { @@ -253,7 +267,8 @@ impl Operation<'_> { "workflow_id run_id reason message retryable error_id exception", "control_plane_internal_error", ), - Self::Cancel | Self::Terminate | Self::Repair => { + Self::Redrive => ("workflow_id continued_from_run_id run_id reason message", ""), + Self::Cancel | Self::Terminate | Self::Archive | Self::Repair => { ("workflow_id run_id reason message remediation", "v1_projection_read_only") } Self::Update(_) => ( @@ -297,13 +312,16 @@ fn words(list: &str) -> Value { } pub(super) async fn admission_response(request: Request, mut body: Value) -> Value { - let segments: Vec<_> = request - .uri() - .path() - .trim_start_matches('/') - .split('/') - .collect(); + let path = request.uri().path().to_owned(); + let segments: Vec<_> = path.trim_start_matches('/').split('/').collect(); let operation = match (request.method().as_str(), segments.as_slice()) { + ("POST", ["api", "workflows"]) => Operation::Start, + ("POST", ["api", "workflows", _, "signal", name]) + | ("POST", ["api", "workflows", _, "runs", _, "signal", name]) => Operation::Signal(name), + ("POST", ["api", "workflows", _, "update", name]) + | ("POST", ["api", "workflows", _, "runs", _, "update", name]) => Operation::Update(name), + ("POST", ["api", "workflows", _, "query", name]) + | ("POST", ["api", "workflows", _, "runs", _, "query", name]) => Operation::Query(name), ("GET", ["api", "workflows"]) => Operation::List, ("GET", ["api", "workflows", _]) => Operation::Describe, ("GET", ["api", "workflows", _, "runs"]) => Operation::ListRuns, @@ -315,6 +333,9 @@ pub(super) async fn admission_response(request: Request, mut body: Value) -> Val | ("POST", ["api", "workflows", _, "runs", _, "cancel"]) => Operation::Cancel, ("POST", ["api", "workflows", _, "terminate"]) | ("POST", ["api", "workflows", _, "runs", _, "terminate"]) => Operation::Terminate, + ("POST", ["api", "workflows", _, "runs", _, "redrive"]) => Operation::Redrive, + ("POST", ["api", "workflows", _, "archive"]) + | ("POST", ["api", "workflows", _, "runs", _, "archive"]) => Operation::Archive, ("POST", ["api", "workflows", _, "repair"]) | ("POST", ["api", "workflows", _, "runs", _, "repair"]) => Operation::Repair, ("GET", ["api", "workflows", _, "debug"]) diff --git a/rust/src/runtime/cooperative.rs b/rust/src/runtime/cooperative.rs index f39aebb6c..df633552d 100644 --- a/rust/src/runtime/cooperative.rs +++ b/rust/src/runtime/cooperative.rs @@ -801,16 +801,17 @@ where ) -> Result<(StatusCode, Value)> { let mut tx = self.begin().await?; let instance = Self::query( - "SELECT current_run_id FROM workflow_instances WHERE id=$1 AND namespace='default'", + "SELECT current_run_id FROM workflow_instances WHERE id=$1 AND namespace=$2", ) .bind(workflow_id) + .bind(self.namespace.as_ref()) .fetch_optional(&mut *tx) .await? .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; let current = DB::string(&instance, "current_run_id")?; let run_id = selected_run.unwrap_or(¤t); - let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1 AND workflow_instance_id=$2 AND namespace='default'") - .bind(run_id).bind(workflow_id).fetch_optional(&mut *tx).await? + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1 AND workflow_instance_id=$2 AND namespace=$3") + .bind(run_id).bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? .ok_or_else(|| refuse(StatusCode::NOT_FOUND,"run_not_found"))?; if run_id != current { return Err(refuse(StatusCode::CONFLICT, "selected_run_not_current")); diff --git a/rust/src/runtime/cross_kind_poll.rs b/rust/src/runtime/cross_kind_poll.rs index bb8bd7155..3396c3e92 100644 --- a/rust/src/runtime/cross_kind_poll.rs +++ b/rust/src/runtime/cross_kind_poll.rs @@ -8,7 +8,7 @@ pub(super) struct PollWakeSnapshot { ready: Vec, } -pub(super) const TASKS_SQL: &str = "SELECT t.id,t.namespace,t.queue FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type!=$3 AND t.task_type IN ('workflow','activity') AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) ORDER BY t.available_at,t.id LIMIT 100"; +pub(super) const TASKS_SQL: &str = "SELECT t.id,t.namespace,t.queue FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type!=$3 AND t.task_type IN ('workflow','activity') AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) ORDER BY t.available_at,t.id LIMIT 100"; impl Store where diff --git a/rust/src/runtime/deployments.rs b/rust/src/runtime/deployments.rs new file mode 100644 index 000000000..7f2e1a9ba --- /dev/null +++ b/rust/src/runtime/deployments.rs @@ -0,0 +1,827 @@ +//! Worker build-id rollouts, as published PHP 2.11.1 keeps them: +//! `DeploymentLifecycleService`, `DeploymentLifecyclePlan`, +//! `WorkerBuildIdNewStartSelector`, `TaskQueueRoutingGate` and the +//! task-queue build-id and `/api/deployments` routes. +//! +//! One rollout row per namespace, task queue and build id records what the +//! operator asked for. A promoted build is where new starts on the queue are +//! pinned. A draining build's workers are refused polls, and a queue whose +//! live workers all drain refuses new starts. +use super::laravel_rules::{self, Rule}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend, start_definition::Cohort, store::{Store, now}}; +use axum::http::StatusCode; +use chrono::{DateTime, Timelike, Utc}; +use serde_json::{Map, Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +pub(super) const DRAINING: &str = "draining"; +const ACTIVE: &str = "active"; + +/// `WorkerBuildIdRollout::buildIdKey`: the trimmed build id, empty when +/// unversioned. +pub(super) fn key(build: Option<&str>) -> String { + build.map(str::trim).unwrap_or_default().to_owned() +} + +/// `buildIdFromKey`. +fn public(key: &str) -> Option { + (!key.is_empty()).then(|| key.to_owned()) +} + +/// Carbon's `toIso8601String`. +fn iso(at: Option>) -> Value { + json!(at.map(|at| at.format("%Y-%m-%dT%H:%M:%S+00:00").to_string())) +} + +/// `ControlPlaneTimestamp::zuluSecond`. +pub(super) fn zulu(at: Option>) -> Value { + json!(at.map(|at| at.format("%Y-%m-%dT%H:%M:%SZ").to_string())) +} + +/// Laravel stores these timestamps to the second. +fn second() -> DateTime { + let at = now(); + at.with_nanosecond(0).unwrap_or(at) +} + +/// PHP's `trim`, with an empty result as none. +fn trimmed(value: Option) -> Option { + value.map(|value| value.trim().to_owned()).filter(|value| !value.is_empty()) +} + +/// One rollout row. +pub(super) struct Rollout { + id: i64, + pub(super) key: String, + pub(super) drain_intent: String, + pub(super) drained_at: Option>, + pub(super) promoted_at: Option>, + pub(super) rolled_back_at: Option>, + required: Option, + fingerprint: Option, + policy: Option, + workflow_types: Option, +} + +impl Rollout { + fn read(row: &DB::Row) -> Result { + Ok(Self { + id: DB::number(row, "id")?, + key: DB::optional_string(row, "build_id")?.unwrap_or_default(), + drain_intent: DB::optional_string(row, "drain_intent")?.unwrap_or_else(|| ACTIVE.to_owned()), + drained_at: DB::optional_instant(row, "drained_at")?, + promoted_at: DB::optional_instant(row, "promoted_at")?, + rolled_back_at: DB::optional_instant(row, "rolled_back_at")?, + required: DB::optional_string(row, "required_compatibility")?, + fingerprint: DB::optional_string(row, "recorded_fingerprint")?, + policy: DB::optional_string(row, "compatibility_policy")?, + workflow_types: DB::optional_document_row(row, "workflow_types")?, + }) + } + + fn new(key: String) -> Self { + Self { id: 0, key, drain_intent: ACTIVE.to_owned(), drained_at: None, promoted_at: None, rolled_back_at: None, + required: None, fingerprint: None, policy: None, workflow_types: None } + } + + /// `WorkerBuildIdNewStartSelector::isSelectable`. + fn selectable(&self) -> bool { + self.drain_intent == ACTIVE && self.promoted_at.is_some() && self.rolled_back_at.is_none() + } +} + +/// `selectedKeyFromRollouts`: the most recently promoted selectable rollout, +/// the later row on a tie. +pub(super) fn selected_key(rollouts: &[Rollout]) -> Option<&str> { + rollouts + .iter() + .filter(|rollout| rollout.selectable()) + .max_by(|left, right| left.promoted_at.cmp(&right.promoted_at).then(left.id.cmp(&right.id))) + .map(|rollout| rollout.key.as_str()) +} + +/// `DeploymentLifecycleState`, without `pending`, which no row derives. +#[derive(Clone, Copy, PartialEq, Debug)] +enum State { + Active, + Promoted, + Draining, + Drained, + RolledBack, +} + +impl State { + fn name(self) -> &'static str { + match self { + Self::Active => "active", + Self::Promoted => "promoted", + Self::Draining => "draining", + Self::Drained => "drained", + Self::RolledBack => "rolled_back", + } + } + + /// `WorkerDeployment::deriveState`. + fn of(rollout: &Rollout) -> Self { + let intent = rollout.drain_intent.trim(); + if rollout.rolled_back_at.is_some() { + Self::RolledBack + } else if intent == DRAINING || rollout.drained_at.is_some() { + if intent == "drained" { Self::Drained } else { Self::Draining } + } else if intent == "drained" { + Self::Drained + } else if rollout.promoted_at.is_some() { + Self::Promoted + } else { + Self::Active + } + } +} + +/// `WorkerDeployment`. +pub(super) struct Deployment { + namespace: String, + task_queue: String, + build: Option, + state: State, + policy: &'static str, + required: Option, + fingerprint: Option, + workflow_types: Vec, + promoted_at: Option>, + drained_at: Option>, + rolled_back_at: Option>, +} + +impl Deployment { + /// `forActiveBuild`, and `find`'s answer for a build only workers know. + fn active(namespace: &str, task_queue: &str, build: Option, required: Option) -> Self { + Self { namespace: namespace.to_owned(), task_queue: task_queue.to_owned(), build, state: State::Active, + policy: "pinned", required, fingerprint: None, workflow_types: Vec::new(), promoted_at: None, + drained_at: None, rolled_back_at: None } + } + + /// `deploymentFromRollout`. + fn of(namespace: &str, task_queue: &str, rollout: &Rollout) -> Self { + let build = public(&rollout.key); + let workflow_types = match &rollout.workflow_types { + Some(Value::String(types)) => types.split(',').map(str::to_owned).collect(), + Some(Value::Array(types)) => types.iter().filter_map(Value::as_str).map(str::to_owned).collect(), + _ => Vec::new(), + }; + let mut workflow_types: Vec = + workflow_types.into_iter().map(|kind| kind.trim().to_owned()).filter(|kind| !kind.is_empty()).collect(); + workflow_types.sort(); + workflow_types.dedup(); + Self { + namespace: namespace.to_owned(), + task_queue: task_queue.to_owned(), + required: trimmed(rollout.required.clone()).or_else(|| build.clone()), + build, + state: State::of(rollout), + policy: if rollout.policy.as_deref() == Some("auto_upgrade") { "auto_upgrade" } else { "pinned" }, + fingerprint: trimmed(rollout.fingerprint.clone()), + workflow_types, + promoted_at: rollout.promoted_at, + drained_at: rollout.drained_at, + rolled_back_at: rollout.rolled_back_at, + } + } + + fn name(&self) -> String { + format!("{}/{}@{}", self.namespace, self.task_queue, self.build.as_deref().unwrap_or("unversioned")) + } + + /// `deploymentPayload`, with the route's timestamp format. + fn view(&self, selected: Option<&str>, time: fn(Option>) -> Value) -> Value { + json!({"name":self.name(),"namespace":self.namespace,"task_queue":self.task_queue,"build_id":self.build, + "state":self.state.name(),"accepts_new_work":matches!(self.state, State::Active | State::Promoted), + "compatibility_policy":self.policy,"required_compatibility":self.required, + "recorded_fingerprint":self.fingerprint,"workflow_types":self.workflow_types, + "promoted_at":time(self.promoted_at),"drained_at":time(self.drained_at),"rolled_back_at":time(self.rolled_back_at), + "new_start_selected":selected == Some(key(self.build.as_deref()).as_str())}) + } + + /// `DeploymentLifecyclePlan::scopeOf`, or the service's `blockageScope`, + /// which leaves out the workflow types. + fn scope(&self, plan: bool) -> Value { + let mut scope = json!({"namespace":self.namespace,"task_queue":self.task_queue,"build_id":self.build, + "state":self.state.name()}); + if let Some(required) = &self.required { + scope["required_compatibility"] = json!(required); + } + if plan && !self.workflow_types.is_empty() { + scope["workflow_types"] = json!(self.workflow_types.join(",")); + } + scope + } +} + +/// `DeploymentBlockage`. +struct Blockage { + reason: &'static str, + message: String, + scope: Value, + expected: Option, +} + +impl Blockage { + fn view(&self) -> Value { + json!({"reason":self.reason,"message":self.message,"scope":self.scope,"expected_resolution":self.expected}) + } + + /// `orderBlockages`' rank. + fn rank(&self) -> u8 { + match self.reason { + "unknown_deployment" | "incompatible_policy" | "fleet_is_draining" => 0, + "no_compatible_workers" | "missing_worker_heartbeat" | "fingerprint_mismatch" => 1, + "replay_safety_failed" => 2, + _ => 3, + } + } + + fn unknown(namespace: &str, task_queue: &str, build: Option<&str>, rollback: bool) -> Self { + let name = format!("{namespace}/{task_queue}@{}", build.unwrap_or("unversioned")); + Self { + reason: "unknown_deployment", + message: if rollback { + format!("No deployment is registered for {name}; nothing to roll back.") + } else { + format!("No deployment is registered for {name}.") + }, + scope: json!({"namespace":namespace,"task_queue":task_queue,"build_id":build}), + expected: (!rollback).then(|| "Roll a worker that heartbeats this build id, then retry promotion.".to_owned()), + } + } + + fn policy(deployment: &Deployment, message: String, expected: &str) -> Self { + Self { reason: "incompatible_policy", message, scope: deployment.scope(true), expected: Some(expected.to_owned()) } + } +} + +/// `fleetSnapshot`: the queue's live workers by id, and the stale ones +/// that support the required build. +struct Fleet { + active: std::collections::BTreeSet, + supporting: std::collections::BTreeSet, + advertised: Vec, + stale_supporting: usize, + last_stale: Option>, +} + +/// A lifecycle action on a deployment. +#[derive(Clone, Copy)] +pub(crate) enum Action { + Promote, + Drain, + Resume, + Rollback, +} + +impl Action { + pub(crate) fn named(name: &str) -> Option { + Some(match name { + "promote" => Self::Promote, + "drain" => Self::Drain, + "resume" => Self::Resume, + "rollback" => Self::Rollback, + _ => return None, + }) + } +} + +/// `DeploymentLifecycleService::parseName`. +fn parse_name(name: &str) -> std::result::Result<(String, String, Option), &'static str> { + let name = name.trim(); + if name.is_empty() { + return Err("Deployment name must not be empty."); + } + let Some((scope, build)) = name.rsplit_once('@') else { + return Err("Deployment name must be of the form namespace/task_queue@build_id (use \"@unversioned\" for the unversioned cohort)."); + }; + let Some((namespace, task_queue)) = scope.split_once('/') else { + return Err("Deployment name must include a namespace and task queue separated by \"/\"."); + }; + let (namespace, task_queue, build) = (namespace.trim(), task_queue.trim(), build.trim()); + if namespace.is_empty() || task_queue.is_empty() || build.is_empty() { + return Err("Deployment name must include namespace, task queue, and build id (or \"unversioned\")."); + } + Ok((namespace.to_owned(), task_queue.to_owned(), (build != "unversioned").then(|| build.to_owned()))) +} + +/// `TaskQueueController`'s `build_id` rule: present, nullable, a string of +/// at most 255 characters. Returns the trimmed build id, none when blank. +fn build_id(body: &Value) -> Result> { + const RULES: &laravel_rules::Rules = &[("build_id", &[Rule::Present, Rule::Nullable, Rule::String, Rule::Max(255)])]; + let body = laravel_rules::normalize(body); + laravel_rules::validate(&body, RULES)?; + Ok(body["build_id"].as_str().map(str::trim).filter(|build| !build.is_empty()).map(str::to_owned)) +} + +fn bad_request(response: Value) -> RuntimeError { + RuntimeError::Protocol { status: StatusCode::BAD_REQUEST, response } +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + pub(super) async fn rollouts(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str) -> Result> { + let rows = Self::query("SELECT * FROM workflow_worker_build_id_rollouts WHERE namespace=$1 AND task_queue=$2 ORDER BY id") + .bind(namespace).bind(task_queue).fetch_all(&mut **tx).await?; + rows.iter().map(Rollout::read::).collect() + } + + async fn rollout(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, key: &str) -> Result> { + Self::query("SELECT * FROM workflow_worker_build_id_rollouts WHERE namespace=$1 AND task_queue=$2 AND build_id=$3") + .bind(namespace).bind(task_queue).bind(key).fetch_optional(&mut **tx).await? + .map(|row| Rollout::read::(&row)).transpose() + } + + /// `WorkflowStartVersionPin::promotedBuildId`: the cohort new starts on + /// the queue are pinned to, when a build was promoted there. + pub(super) async fn promoted_cohort(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str) -> Result> { + let rollouts = Self::rollouts(tx, namespace, task_queue).await?; + Ok(selected_key(&rollouts).map(|key| Cohort::of_run(Some(key)))) + } + + /// `workerRegistrationStatus`: a registration on a draining build stays + /// draining across registrations and heartbeats. + pub(super) async fn registration_status(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, build: Option<&str>) -> Result<&'static str> { + let draining = Self::rollout(tx, namespace, task_queue, &key(build)).await? + .is_some_and(|rollout| rollout.drain_intent == DRAINING); + Ok(if draining { DRAINING } else { ACTIVE }) + } + + /// `resolveRegisteredWorker`'s last check: a draining worker claims + /// nothing until its build resumes. + pub(super) async fn refuse_draining_worker(&self, worker_id: &str, task_queue: &str) -> Result<()> { + let Some(worker) = Self::query("SELECT task_queue,build_id,status FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + .bind(worker_id).bind(self.namespace.as_ref()).fetch_optional(&self.pool).await? + else { + return Ok(()); + }; + if DB::string(&worker, "task_queue")? != task_queue || DB::optional_string(&worker, "status")?.as_deref() != Some(DRAINING) { + return Ok(()); + } + Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task":null,"poll_status":"draining", + "error":format!("Worker [{worker_id}] is marked draining for task queue [{task_queue}] and cannot claim new tasks until the cohort is resumed."), + "reason":"worker_draining","worker_id":worker_id,"task_queue":task_queue, + "registered_build_id":DB::optional_string(&worker, "build_id")?.filter(|build| !build.is_empty()), + "worker_status":DRAINING,"drain_intent":DRAINING}), + }) + } + + /// `TaskQueueRoutingGate::workflowStartBlock`: why a start on the queue + /// is refused, when every live worker there drains. + pub(super) async fn routing_block(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, stale_after: i64) -> Result>> { + let cutoff = now() - chrono::Duration::seconds(stale_after.max(1)); + let (mut active, mut draining, mut stale) = (0, 0, 0); + for worker in Self::query("SELECT status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2") + .bind(namespace).bind(task_queue).fetch_all(&mut **tx).await? + { + if DB::optional_instant(&worker, "last_heartbeat_at")?.is_some_and(|heartbeat| heartbeat < cutoff) { + stale += 1; + } else if DB::optional_string(&worker, "status")?.as_deref() == Some(DRAINING) { + draining += 1; + } else { + active += 1; + } + } + let mut builds: Vec = Self::rollouts(tx, namespace, task_queue).await?.into_iter() + .filter(|rollout| rollout.drain_intent == DRAINING).map(|rollout| rollout.key).collect(); + builds.sort(); + if active > 0 || (draining == 0 && builds.is_empty()) { + return Ok(None); + } + let Value::Object(block) = json!({"routing_status":"draining","active_worker_count":active, + "draining_worker_count":draining,"stale_worker_count":stale, + "draining_build_ids":builds.iter().map(|key| public(key)).collect::>(),"drain_intent":DRAINING}) + else { + unreachable!() + }; + Ok(Some(block)) + } + + /// `find`: the rollout's deployment, else one only a registration knows. + async fn find(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, build: Option<&str>) -> Result> { + if let Some(rollout) = Self::rollout(tx, namespace, task_queue, &key(build)).await? { + return Ok(Some(Deployment::of(namespace, task_queue, &rollout))); + } + let worker = Self::query("SELECT build_id FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2 AND (($3 IS NULL AND (build_id IS NULL OR build_id='')) OR build_id=$4) ORDER BY last_heartbeat_at DESC LIMIT 1") + .bind(namespace).bind(task_queue).bind(build).bind(build).fetch_optional(&mut **tx).await?; + Ok(match worker { + Some(worker) => { + let required = trimmed(DB::optional_string(&worker, "build_id")?); + Some(Deployment::active(namespace, task_queue, build.map(str::to_owned), required)) + } + None => None, + }) + } + + /// `fleetSnapshot`. PHP joins two views of the queue's workers. Its + /// compatibility heartbeats, recorded at each registration and heartbeat + /// and kept 30 seconds, count draining workers too; native reads them + /// from the registration's heartbeat time. The registrations count live, + /// undrained workers, and the stale ones that support the required build. + async fn fleet(&self, tx: &mut Transaction<'_, DB>, task_queue: &str, required: Option<&str>) -> Result { + let at = now(); + let cutoff = at - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1)); + let recorded = at - chrono::Duration::seconds(30); + let mut fleet = Fleet { active: Default::default(), supporting: Default::default(), advertised: Vec::new(), + stale_supporting: 0, last_stale: None }; + for worker in Self::query("SELECT worker_id,build_id,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2 ORDER BY worker_id") + .bind(self.namespace.as_ref()).bind(task_queue).fetch_all(&mut **tx).await? + { + let id = DB::string(&worker, "worker_id")?.trim().to_owned(); + let build = trimmed(DB::optional_string(&worker, "build_id")?); + let supports = required.is_none() || build.as_deref() == required; + let heartbeat = DB::optional_instant(&worker, "last_heartbeat_at")?; + if heartbeat.is_some_and(|heartbeat| heartbeat >= recorded) { + fleet.active.insert(id.clone()); + if supports { + fleet.supporting.insert(id.clone()); + } + if let Some(build) = &build && !fleet.advertised.contains(build) { + fleet.advertised.push(build.clone()); + } + } + if heartbeat.is_some_and(|heartbeat| heartbeat < cutoff) { + if supports && required.is_some() { + fleet.stale_supporting += 1; + fleet.last_stale = fleet.last_stale.max(heartbeat); + } + continue; + } + if trimmed(DB::optional_string(&worker, "status")?).as_deref() == Some(DRAINING) { + continue; + } + fleet.active.insert(id.clone()); + if supports { + fleet.supporting.insert(id); + } + if let Some(build) = build && !fleet.advertised.contains(&build) { + fleet.advertised.push(build); + } + } + fleet.advertised.sort(); + Ok(fleet) + } + + /// `evaluatePromote` with `normalizePromotionBlockages`. + fn promote_blockages(deployment: &Deployment, fleet: &Fleet) -> Vec { + let name = deployment.name(); + if matches!(deployment.state, State::Drained | State::RolledBack) { + return vec![Blockage::policy(deployment, + format!("Deployment {name} is {} and cannot be promoted; recreate the deployment first.", deployment.state.name()), + "Create a new active deployment for this build id, then promote it.")]; + } + let mut blockages = Vec::new(); + if deployment.state == State::Draining { + blockages.push(Blockage { reason: "fleet_is_draining", message: format!("Deployment {name} is draining; resume it before promoting."), + scope: deployment.scope(true), expected: Some("Resume the deployment, then retry promotion.".to_owned()) }); + } + let Some(required) = &deployment.required else { + return blockages; + }; + let (namespace, queue) = (&deployment.namespace, &deployment.task_queue); + let start = format!("Start at least one worker that advertises compatibility [{required}] for this task queue, then retry promotion."); + let stale = || Blockage { + reason: "missing_worker_heartbeat", + message: format!("{} worker(s) for {namespace}/{queue} advertise required compatibility [{required}], but none have a fresh heartbeat.{}", + fleet.stale_supporting, + fleet.last_stale.map(|at| format!(" Last matching heartbeat was {}.", at.format("%Y-%m-%dT%H:%M:%S%.6fZ"))).unwrap_or_default()), + scope: deployment.scope(false), + expected: Some(format!("Restart or heartbeat a worker that advertises compatibility [{required}] for this task queue, then retry promotion.")), + }; + if fleet.active.is_empty() { + blockages.push(if fleet.stale_supporting > 0 { stale() } else { + Blockage { reason: "no_compatible_workers", + message: format!("No workers are heartbeating against {namespace}/{queue}; promotion would route fresh tasks to a fleet that cannot claim them."), + scope: deployment.scope(true), expected: Some(start) } + }); + } else if fleet.supporting.is_empty() { + blockages.push(if fleet.stale_supporting > 0 { stale() } else { + let advertised = if fleet.advertised.is_empty() { "none".to_owned() } else { fleet.advertised.join(", ") }; + Blockage { reason: "no_compatible_workers", + message: format!("No active worker on {namespace}/{queue} advertises required compatibility [{required}]; active workers advertise [{advertised}]."), + scope: deployment.scope(false), expected: Some(start) } + }); + } + blockages + } + + /// `upsertRollout`: the row for the build, created active when missing. + async fn upsert(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, build: Option<&str>, apply: impl FnOnce(&mut Rollout)) -> Result { + let key = key(build); + let existing = Self::rollout(tx, namespace, task_queue, &key).await?; + let exists = existing.is_some(); + let mut rollout = existing.unwrap_or_else(|| Rollout::new(key.clone())); + apply(&mut rollout); + let at = second(); + if exists { + Self::query("UPDATE workflow_worker_build_id_rollouts SET drain_intent=$1,drained_at=$2,promoted_at=$3,rolled_back_at=$4,updated_at=$5 WHERE id=$6") + .bind(rollout.drain_intent.as_str()).bind(DB::optional_time(rollout.drained_at)).bind(DB::optional_time(rollout.promoted_at)) + .bind(DB::optional_time(rollout.rolled_back_at)).bind(DB::bind_time(at)).bind(rollout.id).execute(&mut **tx).await?; + } else { + Self::query("INSERT INTO workflow_worker_build_id_rollouts(namespace,task_queue,build_id,drain_intent,drained_at,promoted_at,rolled_back_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)") + .bind(namespace).bind(task_queue).bind(key.as_str()).bind(rollout.drain_intent.as_str()).bind(DB::optional_time(rollout.drained_at)) + .bind(DB::optional_time(rollout.promoted_at)).bind(DB::optional_time(rollout.rolled_back_at)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut **tx).await?; + } + Self::rollout(tx, namespace, task_queue, &key).await?.ok_or_else(|| super::refuse(StatusCode::CONFLICT, "build_id_rollout_lost")) + } + + /// `stampWorkerDrainStatus`. + async fn stamp_workers(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str, build: Option<&str>, status: &str, only_draining: bool) -> Result<()> { + Self::query("UPDATE workflow_worker_registrations SET status=$1 WHERE namespace=$2 AND task_queue=$3 AND (($4 IS NULL AND (build_id IS NULL OR build_id='')) OR build_id=$5) AND ($6='all' OR status='draining')") + .bind(status).bind(namespace).bind(task_queue).bind(build).bind(build).bind(if only_draining { "draining" } else { "all" }) + .execute(&mut **tx).await?; + Ok(()) + } + + /// One lifecycle action: the resulting deployment, or why it was refused. + async fn lifecycle(&self, tx: &mut Transaction<'_, DB>, action: Action, task_queue: &str, build: Option<&str>) -> Result, Vec)>> { + let namespace = self.namespace.as_ref(); + let found = Self::find(tx, namespace, task_queue, build).await?; + match action { + Action::Promote => { + let Some(deployment) = found else { + return Ok(Err((None, vec![Blockage::unknown(namespace, task_queue, build, false)]))); + }; + let fleet = self.fleet(tx, task_queue, deployment.required.as_deref()).await?; + let blockages = Self::promote_blockages(&deployment, &fleet); + if !blockages.is_empty() { + return Ok(Err((Some(deployment), blockages))); + } + let at = second(); + let rollout = Self::upsert(tx, namespace, task_queue, build, |row| { + row.drain_intent = ACTIVE.to_owned(); + row.drained_at = None; + row.promoted_at = Some(at); + row.rolled_back_at = None; + }).await?; + Ok(Ok(Deployment::of(namespace, task_queue, &rollout))) + } + Action::Drain => { + let deployment = found.unwrap_or_else(|| Deployment::active(namespace, task_queue, build.map(str::to_owned), None)); + if deployment.state == State::RolledBack { + let message = format!("Deployment {} is rolled back; drain is unnecessary because the deployment no longer accepts work.", deployment.name()); + let blockage = Blockage::policy(&deployment, message, "No action required; the deployment is already terminal."); + return Ok(Err((Some(deployment), vec![blockage]))); + } + let at = second(); + let rollout = Self::upsert(tx, namespace, task_queue, build, |row| { + if row.drain_intent != DRAINING { + row.drained_at = Some(at); + } + row.drain_intent = DRAINING.to_owned(); + }).await?; + Self::stamp_workers(tx, namespace, task_queue, build, DRAINING, false).await?; + Ok(Ok(Deployment::of(namespace, task_queue, &rollout))) + } + Action::Resume => { + let deployment = found.unwrap_or_else(|| Deployment::active(namespace, task_queue, build.map(str::to_owned), None)); + let refused = match deployment.state { + State::Drained => Some((format!("Deployment {} is fully drained; resume is not allowed once a deployment has reached the drained terminal state — recreate it instead.", deployment.name()), + "Create a new active deployment for this build id.")), + State::RolledBack => Some((format!("Deployment {} is rolled back; promote a different deployment rather than resuming this one.", deployment.name()), + "Promote the prior healthy deployment, or create a new active deployment for this build id.")), + _ => None, + }; + if let Some((message, expected)) = refused { + let blockage = Blockage::policy(&deployment, message, expected); + return Ok(Err((Some(deployment), vec![blockage]))); + } + let rollout = Self::upsert(tx, namespace, task_queue, build, |row| { + row.drain_intent = ACTIVE.to_owned(); + row.drained_at = None; + }).await?; + Self::stamp_workers(tx, namespace, task_queue, build, ACTIVE, true).await?; + Ok(Ok(Deployment::of(namespace, task_queue, &rollout))) + } + Action::Rollback => { + let Some(deployment) = found else { + return Ok(Err((None, vec![Blockage::unknown(namespace, task_queue, build, true)]))); + }; + if deployment.state == State::RolledBack { + let message = format!("Deployment {} is already rolled back.", deployment.name()); + let blockage = Blockage::policy(&deployment, message, "No action required; the deployment is already terminal."); + return Ok(Err((Some(deployment), vec![blockage]))); + } + let at = second(); + let rollout = Self::upsert(tx, namespace, task_queue, build, |row| { + row.rolled_back_at = Some(at); + row.promoted_at = None; + }).await?; + // `withState(RolledBack)`: rolled back now, and no drain time. + let mut deployment = Deployment::of(namespace, task_queue, &rollout); + deployment.state = State::RolledBack; + deployment.drained_at = None; + Ok(Ok(deployment)) + } + } + } + + /// The queue's selected rollout key after a change. + async fn selected(tx: &mut Transaction<'_, DB>, namespace: &str, task_queue: &str) -> Result> { + let rollouts = Self::rollouts(tx, namespace, task_queue).await?; + Ok(selected_key(&rollouts).map(str::to_owned)) + } + + /// `lifecycleRejectionPayload`, with blockages in PHP's order. + fn rejection(deployment: Option, mut blockages: Vec, scope: Option) -> Value { + blockages.sort_by_key(Blockage::rank); + let primary = &blockages[0]; + let outcome = format!("rejected_{}", primary.reason); + let mut response = json!({"message":primary.message,"reason":primary.reason,"rejection_reason":primary.reason, + "expected_resolution":primary.expected,"outcome":outcome,"control_plane_outcome":outcome, + "command_status":"rejected","command_source":"control_plane"}); + if let Some(Value::Object(scope)) = scope { + response.as_object_mut().expect("object").extend(scope); + } + response["deployment"] = deployment.unwrap_or(Value::Null); + response["blockages"] = Value::Array(blockages.iter().map(Blockage::view).collect()); + response + } + + /// `POST /api/deployments/{name}/{action}`. + pub(crate) async fn deployment_action(&self, name: &str, action: Action) -> Result<(StatusCode, Value)> { + let (task_queue, build) = self.deployment_scope(name)?; + let mut tx = self.begin().await?; + let result = self.lifecycle(&mut tx, action, &task_queue, build.as_deref()).await?; + let selected = Self::selected(&mut tx, self.namespace.as_ref(), &task_queue).await?; + tx.commit().await?; + Ok(match result { + Ok(deployment) => (StatusCode::OK, deployment.view(selected.as_deref(), iso)), + Err((deployment, blockages)) => (StatusCode::CONFLICT, + Self::rejection(deployment.map(|deployment| deployment.view(selected.as_deref(), iso)), blockages, None)), + }) + } + + /// `GET /api/deployments/{name}`. + pub(crate) async fn deployment(&self, name: &str) -> Result<(StatusCode, Value)> { + let (task_queue, build) = self.deployment_scope(name)?; + let mut tx = self.begin().await?; + let found = Self::find(&mut tx, self.namespace.as_ref(), &task_queue, build.as_deref()).await?; + let selected = Self::selected(&mut tx, self.namespace.as_ref(), &task_queue).await?; + tx.commit().await?; + Ok(match found { + Some(deployment) => (StatusCode::OK, deployment.view(selected.as_deref(), iso)), + None => (StatusCode::NOT_FOUND, json!({"error":"deployment_not_found"})), + }) + } + + /// The task queue and build a deployment name in this namespace names. + fn deployment_scope(&self, name: &str) -> Result<(String, Option)> { + let (namespace, task_queue, build) = parse_name(name).map_err(|error| bad_request(json!({"error":error})))?; + if namespace != self.namespace.as_ref() { + return Err(bad_request(json!({"error":"namespace_mismatch", + "message":format!("Deployment name {name} is in namespace {namespace} but the request targets {}.", self.namespace)}))); + } + Ok((task_queue, build)) + } + + /// `GET /api/deployments`: `listForNamespace`. + pub(crate) async fn deployments(&self) -> Result { + let namespace = self.namespace.as_ref(); + let mut tx = self.begin().await?; + let rollouts = Self::query("SELECT * FROM workflow_worker_build_id_rollouts WHERE namespace=$1 ORDER BY task_queue,build_id") + .bind(namespace).fetch_all(&mut *tx).await?; + let mut rollouts: Vec<(String, Rollout)> = rollouts.iter() + .map(|row| Ok((DB::string(row, "task_queue")?, Rollout::read::(row)?))).collect::>()?; + let workers = Self::query("SELECT task_queue,build_id FROM workflow_worker_registrations WHERE namespace=$1 ORDER BY task_queue,COALESCE(build_id,'')") + .bind(namespace).fetch_all(&mut *tx).await?; + let mut seen: Vec<(String, String)> = Vec::new(); + let mut deployments = Vec::new(); + for worker in &workers { + let Some(queue) = trimmed(DB::optional_string(worker, "task_queue")?) else { + continue; + }; + let build = trimmed(DB::optional_string(worker, "build_id")?); + let scope = (queue.clone(), key(build.as_deref())); + if seen.contains(&scope) { + continue; + } + seen.push(scope.clone()); + deployments.push(match rollouts.iter().position(|(q, rollout)| *q == scope.0 && rollout.key == scope.1) { + Some(index) => { + let (queue, rollout) = rollouts.remove(index); + (queue.clone(), Deployment::of(namespace, &queue, &rollout)) + } + None => (queue.clone(), Deployment::active(namespace, &queue, build.clone(), build)), + }); + } + deployments.extend(rollouts.iter().map(|(queue, rollout)| (queue.clone(), Deployment::of(namespace, queue, rollout)))); + let mut views = Vec::with_capacity(deployments.len()); + for (queue, deployment) in &deployments { + let selected = Self::selected(&mut tx, namespace, queue).await?; + views.push(deployment.view(selected.as_deref(), iso)); + } + tx.commit().await?; + Ok(json!({"namespace":namespace,"deployments":views})) + } + + /// `POST /api/task-queues/{queue}/build-ids/{promote|drain|resume}`. + pub(crate) async fn build_id_action(&self, task_queue: &str, body: &Value, action: Action) -> Result<(StatusCode, Value)> { + let build = build_id(body)?; + let namespace = self.namespace.as_ref(); + let mut tx = self.begin().await?; + let scope = json!({"namespace":namespace,"task_queue":task_queue,"build_id":build}); + if let Action::Promote = action { + let result = self.lifecycle(&mut tx, action, task_queue, build.as_deref()).await?; + let selected = Self::selected(&mut tx, namespace, task_queue).await?; + tx.commit().await?; + return Ok(match result { + Ok(deployment) => { + let view = deployment.view(selected.as_deref(), zulu); + let mut response = scope; + response["drain_intent"] = json!(if deployment.state == State::Draining { DRAINING } else { ACTIVE }); + for field in ["drained_at", "promoted_at", "rolled_back_at"] { + response[field] = view[field].clone(); + } + response["new_start_selected"] = json!(selected.as_deref() == Some(key(build.as_deref()).as_str())); + response["deployment"] = view; + (StatusCode::OK, response) + } + Err((deployment, blockages)) => (StatusCode::CONFLICT, + Self::rejection(deployment.map(|deployment| deployment.view(selected.as_deref(), zulu)), blockages, Some(scope))), + }); + } + // `setBuildIdDrainIntent`: the drain intent alone, without the plan. + let draining = matches!(action, Action::Drain); + let at = second(); + let rollout = Self::upsert(&mut tx, namespace, task_queue, build.as_deref(), |row| { + row.drained_at = if draining { if row.drain_intent == DRAINING { row.drained_at } else { Some(at) } } else { None }; + row.drain_intent = if draining { DRAINING } else { ACTIVE }.to_owned(); + }).await?; + Self::stamp_workers(&mut tx, namespace, task_queue, build.as_deref(), if draining { DRAINING } else { ACTIVE }, !draining).await?; + let selected = Self::selected(&mut tx, namespace, task_queue).await?; + tx.commit().await?; + let mut response = scope; + response["drain_intent"] = json!(rollout.drain_intent); + response["drained_at"] = zulu(rollout.drained_at); + response["promoted_at"] = zulu(rollout.promoted_at); + response["rolled_back_at"] = zulu(rollout.rolled_back_at); + response["new_start_selected"] = json!(rollout.selectable() && selected.as_deref() == Some(rollout.key.as_str())); + Ok((StatusCode::OK, response)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rollout(key: &str, intent: &str, promoted: Option, rolled_back: bool, id: i64) -> Rollout { + let at = |seconds: i64| DateTime::from_timestamp(seconds, 0).unwrap(); + Rollout { id, drain_intent: intent.to_owned(), promoted_at: promoted.map(at), rolled_back_at: rolled_back.then(|| at(9)), + ..Rollout::new(key.to_owned()) } + } + + #[test] + fn names_parse_like_php() { + assert_eq!(parse_name(" default/q@v1 "), Ok(("default".into(), "q".into(), Some("v1".into())))); + assert_eq!(parse_name("default/a/b@unversioned"), Ok(("default".into(), "a/b".into(), None))); + assert_eq!(parse_name("default/q@x@v2"), Ok(("default".into(), "q@x".into(), Some("v2".into())))); + assert!(parse_name("nonsense").unwrap_err().starts_with("Deployment name must be of the form")); + assert!(parse_name("q@v1").unwrap_err().contains("separated by")); + assert!(parse_name("default/@v1").unwrap_err().contains("(or \"unversioned\")")); + assert_eq!(parse_name(" "), Err("Deployment name must not be empty.")); + } + + #[test] + fn the_latest_selectable_promotion_is_selected() { + let rollouts = [rollout("v1", "active", Some(5), false, 1), rollout("v2", "active", Some(6), true, 2), + rollout("", "active", Some(5), false, 3), rollout("v3", "draining", Some(8), false, 4)]; + // v2 is rolled back and v3 drains; v1 and unversioned tie, and the later row wins. + assert_eq!(selected_key(&rollouts), Some("")); + assert_eq!(selected_key(&rollouts[..1]), Some("v1")); + assert_eq!(selected_key(&[]), None); + } + + #[test] + fn states_derive_like_worker_deployment() { + let state = |intent: &str, drained: bool, promoted: bool, rolled_back: bool| { + let mut row = rollout("v1", intent, promoted.then_some(1), rolled_back, 1); + row.drained_at = drained.then(|| DateTime::from_timestamp(2, 0).unwrap()); + State::of(&row) + }; + assert_eq!(state("active", false, false, false), State::Active); + assert_eq!(state("active", false, true, false), State::Promoted); + assert_eq!(state("draining", true, true, false), State::Draining); + assert_eq!(state("active", true, false, false), State::Draining); + assert_eq!(state("drained", true, false, false), State::Drained); + assert_eq!(state("draining", true, true, true), State::RolledBack); + } +} diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index a658f84bd..fa6a57a4f 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -44,6 +44,8 @@ pub struct Runtime { transition_codec: Arc, pub(super) external: Arc, pub(super) queries: Arc, + /// The package's signal, update and payload ceilings (DW_V2_LIMIT_*). + pub(super) structural: super::StructuralLimits, } struct TimerScheduler { @@ -142,6 +144,10 @@ impl Runtime { self.queries = Arc::new(config); self } + pub fn with_structural_limits(mut self, limits: super::StructuralLimits) -> Self { + self.structural = limits; + self + } /// The configured history page sizes (DW_HISTORY_PAGE_SIZE_DEFAULT and _MAX). pub(super) fn history_pages(&self) -> super::history_pages::HistoryPages { super::history_pages::HistoryPages { @@ -187,6 +193,7 @@ impl Runtime { let external = super::ExternalPayloadConfig::from_env()?; let queries = super::QueryTaskConfig::from_env()?; let limit = super::memo_limits::configured_limit()?; + let structural = super::StructuralLimits::from_env()?; let history_budget = super::history_budget::Config::from_env()?; let search_attribute_limit = super::search_attributes::configured_limit()?; let mut store = Store::new(sqlite::open(database).await?); @@ -197,7 +204,7 @@ impl Runtime { store.activity_task_lease_seconds = queries.activity_task_lease_seconds; store.update_wait_seconds = queries.update_wait_completion_timeout_seconds; store.worker_stale_after_seconds = queries.worker_stale_after_seconds; - Self::from_storage(Storage::Sqlite(store), token, external, queries).await + Self::from_storage(Storage::Sqlite(store), token, external, queries, structural).await } pub async fn open_postgres(options: PgConnectOptions, token: String) -> Result { @@ -205,6 +212,7 @@ impl Runtime { let external = super::ExternalPayloadConfig::from_env()?; let queries = super::QueryTaskConfig::from_env()?; let limit = super::memo_limits::configured_limit()?; + let structural = super::StructuralLimits::from_env()?; let history_budget = super::history_budget::Config::from_env()?; let search_attribute_limit = super::search_attributes::configured_limit()?; let mut store = Store::new(PostgresStorage::open(options).await?.into_pool()); @@ -215,7 +223,7 @@ impl Runtime { store.activity_task_lease_seconds = queries.activity_task_lease_seconds; store.update_wait_seconds = queries.update_wait_completion_timeout_seconds; store.worker_stale_after_seconds = queries.worker_stale_after_seconds; - Self::from_storage(Storage::Postgres(store), token, external, queries).await + Self::from_storage(Storage::Postgres(store), token, external, queries, structural).await } pub async fn open_mysql(options: MySqlConnectOptions, token: String) -> Result { @@ -223,6 +231,7 @@ impl Runtime { let external = super::ExternalPayloadConfig::from_env()?; let queries = super::QueryTaskConfig::from_env()?; let limit = super::memo_limits::configured_limit()?; + let structural = super::StructuralLimits::from_env()?; let history_budget = super::history_budget::Config::from_env()?; let search_attribute_limit = super::search_attributes::configured_limit()?; let mut store = Store::new(MySqlStorage::open(options).await?.into_pool()); @@ -233,7 +242,7 @@ impl Runtime { store.activity_task_lease_seconds = queries.activity_task_lease_seconds; store.update_wait_seconds = queries.update_wait_completion_timeout_seconds; store.worker_stale_after_seconds = queries.worker_stale_after_seconds; - Self::from_storage(Storage::MySql(store), token, external, queries).await + Self::from_storage(Storage::MySql(store), token, external, queries, structural).await } async fn from_storage( @@ -241,6 +250,7 @@ impl Runtime { token: String, external: super::ExternalPayloadConfig, queries: super::QueryTaskConfig, + structural: super::StructuralLimits, ) -> Result { let runtime = Self { storage: Arc::new(storage), @@ -252,6 +262,7 @@ impl Runtime { transition_codec: Arc::new(Semaphore::new(1)), external: Arc::new(external), queries: Arc::new(queries), + structural, }; runtime.verify_ready().await?; if let Err(error) = runtime.initialize_namespace_registry().await { @@ -322,6 +333,7 @@ impl Runtime { transition_codec: self.transition_codec.clone(), external: self.external.clone(), queries: self.queries.clone(), + structural: self.structural, } } pub(super) fn namespace(&self) -> &str { @@ -339,6 +351,21 @@ impl Runtime { delegate!(system_health(config: &super::operator_metrics::MetricsConfig) -> Result<(bool, Value)>); delegate!(operator_dashboard(config: &super::operator_metrics::MetricsConfig, bounded: bool) -> Result); delegate!(repair_status() -> Result); + delegate!(repair_pass(request: &super::repair_pass::PassRequest) -> Result); + delegate!(system_metrics() -> Result); + delegate!(prometheus_metrics() -> Result); + delegate!(payload_cleanup_status(config: &super::operator_metrics::MetricsConfig) -> Result); + /// The namespace's payload cleanup pass, with its local storage when an + /// enabled local policy names it. + pub(crate) async fn payload_cleanup_pass(&self, limit: i64) -> Result { + let local = super::external_transport::local(self).await.ok(); + let retention = self.external.abandoned_seconds; + match &*self.storage { + Storage::Sqlite(store) => store.payload_cleanup_pass(limit, retention, local.as_ref()).await, + Storage::Postgres(store) => store.payload_cleanup_pass(limit, retention, local.as_ref()).await, + Storage::MySql(store) => store.payload_cleanup_pass(limit, retention, local.as_ref()).await, + } + } delegate!(retention_status(limit: i64) -> Result); delegate!(retention_pass(limit: i64, run_ids: Vec) -> Result<(Value, bool)>); delegate!(activity_timeout_status(limit: i64) -> Result); @@ -426,6 +453,35 @@ impl Runtime { } } + /// `resolveWorkflowTaskCommandPayloadReferences`: each command payload + /// must decode as one Avro value, checked in command and field order. + async fn check_command_payloads(&self, body: &Value) -> Result<()> { + use super::command_payloads::{Check, UNDECODABLE, invalid}; + let checks = super::command_payloads::checks(body); + if checks.is_empty() { + return Ok(()); + } + // Completion work uses the transition codec, never read capacity. + // The permit is released before the completion takes its own. + let permit = self.transition_codec.clone().acquire_owned().await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let codec = crate::codec::ValueCodec::new() + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))?; + for check in checks { + match check { + Check::Decode { field, blob } if codec.decode(&blob).is_err() => return Err(invalid(&field, UNDECODABLE)), + Check::Decode { .. } => {} + Check::Refuse(error) => return Err(error), + } + } + Ok(()) + }) + .await + .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "signal_codec_unavailable"))? + } + /// Refuses an envelope blob the codec cannot decode, with PHP's message. async fn decodable(&self, blob: &Value) -> Result<()> { let Some(blob) = blob.as_str().map(str::to_owned) else { @@ -468,6 +524,114 @@ impl Runtime { Storage::MySql(store) => store.start(body, memo, external, context).await, } } + /// `ActivityController::start`: validates like PHP, then starts the + /// activity in its own host run. + pub(crate) async fn start_standalone_activity(&self, body: Value, context: Value) -> Result { + let mut body = super::laravel_rules::normalize(&body); + let request = super::standalone_activities::start_request(&body)?; + // `AvroPayloadEnvelopeResolver::resolve`: no input stores none; plain + // values are encoded; an external reference is retained. + let input = body.get("input").is_some_and(|input| !input.is_null()); + let mut external = None; + if input { + super::input_envelope::transport(&body)?; + match self.plain_arguments(&body).await? { + Some(blob) => body["input"] = json!({"codec": "avro", "blob": blob}), + None => { + external = self.prepare_external(&body["input"]).await?; + if external.is_none() { + self.decodable(&body["input"]["blob"]).await?; + } + } + } + } + let arguments = match &external { + Some(prepared) => Some(prepared.stored()), + None if input => Some(super::envelope(&body, "input")?), + None => None, + }; + match &*self.storage { + Storage::Sqlite(store) => store.start_standalone_activity(request, arguments, external, context).await, + Storage::Postgres(store) => store.start_standalone_activity(request, arguments, external, context).await, + Storage::MySql(store) => store.start_standalone_activity(request, arguments, external, context).await, + } + } + + /// `ActivityController::show`. + pub(crate) async fn standalone_activity(&self, activity_id: &str) -> Result { + let (host, execution) = match &*self.storage { + Storage::Sqlite(store) => store.standalone_host(activity_id).await?, + Storage::Postgres(store) => store.standalone_host(activity_id).await?, + Storage::MySql(store) => store.standalone_host(activity_id).await?, + }; + let attempts = self.standalone_attempts(&host, &execution).await?; + let mut view = json!({"activity_id":host["workflow_instance_id"],"workflow_id":host["workflow_instance_id"], + "workflow_run_id":host["id"],"workflow_type":host["workflow_type"],"namespace":self.namespace(), + "activity_type":execution["activity_type"],"activity_class":execution["activity_class"], + "activity_execution_id":execution["id"],"activity_status":execution["status"]}); + Self::extend_attempts(&mut view, &attempts); + for (key, value) in [("task_queue", &host["queue"]), ("business_key", &host["business_key"]), ("status", &host["status"]), + ("closed_reason", &host["closed_reason"]), ("started_at", &host["started_at"]), ("closed_at", &host["closed_at"]), + ("last_progress_at", &host["last_progress_at"]), ("compatibility", &host["compatibility"]), + ("payload_codec", &host["payload_codec"]), ("schedule_to_start_deadline_at", &execution["schedule_deadline_at"]), + ("schedule_to_close_deadline_at", &execution["schedule_to_close_deadline_at"]), + ("last_heartbeat_at", &execution["last_heartbeat_at"])] { + view[key] = value.clone(); + } + view["result"] = match execution["result"].as_str() { + Some(result) if execution["status"] == "completed" => super::wire(result), + _ => Value::Null, + }; + Ok(view) + } + + /// `ActivityController::index`. + pub(crate) async fn standalone_activities(&self, query: Value) -> Result { + let request = super::standalone_activities::list_request(&query)?; + let (page, more) = match &*self.storage { + Storage::Sqlite(store) => store.standalone_hosts(&request).await?, + Storage::Postgres(store) => store.standalone_hosts(&request).await?, + Storage::MySql(store) => store.standalone_hosts(&request).await?, + }; + let mut activities = Vec::with_capacity(page.len()); + for (host, execution) in &page { + let attempts = self.standalone_attempts(host, execution).await?; + let mut entry = json!({"activity_id":host["workflow_instance_id"],"workflow_id":host["workflow_instance_id"], + "workflow_run_id":host["id"],"activity_execution_id":execution["id"],"activity_type":execution["activity_type"], + "activity_class":execution["activity_class"],"activity_status":execution["status"]}); + Self::extend_attempts(&mut entry, &attempts); + for (key, value) in [("task_queue", &host["queue"]), ("status", &host["status"]), + ("status_bucket", &host["status_bucket"]), ("business_key", &host["business_key"]), + ("started_at", &host["started_at"]), ("closed_at", &host["closed_at"])] { + entry[key] = value.clone(); + } + activities.push(entry); + } + Ok(json!({"activity_count":activities.len(),"activities":activities, + "next_page_token":more.then(|| super::standalone_activities::page_token(request.offset + request.page_size))})) + } + + /// The host run's activity view, as `RunActivityView` reads it. + async fn standalone_attempts(&self, host: &Value, execution: &Value) -> Result { + let activities = if execution.is_null() { + json!([]) + } else { + let workflow_id = host["workflow_instance_id"].as_str().unwrap_or_default(); + let run_id = host["id"].as_str().unwrap_or_default(); + self.activities(workflow_id, run_id).await?["activities"].clone() + }; + Ok(super::standalone_activities::attempt_fields(execution, &activities)) + } + + fn extend_attempts(view: &mut Value, attempts: &Value) { + for key in ["attempt_count", "current_attempt_id", "current_attempt_status", "current_attempt", "attempts"] { + view[key] = attempts[key].clone(); + } + view["attempt_state"] = json!({"activity_execution_id":attempts["activity_execution_id"], + "attempt_count":attempts["attempt_count"],"current_attempt_id":attempts["current_attempt_id"], + "current_attempt_status":attempts["current_attempt_status"],"attempts":attempts["attempts"]}); + } + pub(crate) async fn create_schedule( &self, body: Value, @@ -521,8 +685,14 @@ impl Runtime { super::input_envelope::transport(&body)?; super::laravel_validation::operation_name(name, "signal")?; super::command_requests::signal(&body)?; - let arguments = self.command_arguments(&body).await?; - let prepared = super::store::prepare_control_arguments( + // `AvroPayloadEnvelopeResolver::resolve` with the namespace's storage: + // a runtime external reference is checked, then kept by reference. + let mut external = self.prepare_external(&body["input"]).await?; + let arguments = match external.as_mut() { + Some(prepared) => prepared.blob()?, + None => self.command_arguments(&body).await?, + }; + let mut prepared = super::store::prepare_control_arguments( self.signal_codec.clone(), name.to_owned(), arguments, @@ -530,6 +700,9 @@ impl Runtime { true, ) .await?; + if let Some(external) = &external { + prepared.arguments = external.stored(); + } match &*self.storage { Storage::Sqlite(store) => { store @@ -541,6 +714,8 @@ impl Runtime { prepared, self.transition_codec.clone(), context, + self.structural, + external.as_ref(), ) .await } @@ -554,6 +729,8 @@ impl Runtime { prepared, self.transition_codec.clone(), context, + self.structural, + external.as_ref(), ) .await } @@ -567,6 +744,8 @@ impl Runtime { prepared, self.transition_codec.clone(), context, + self.structural, + external.as_ref(), ) .await } @@ -638,13 +817,15 @@ impl Runtime { super::input_envelope::transport(&body)?; super::laravel_validation::operation_name(name, "update")?; super::command_requests::update(&body)?; - let arguments = self.command_arguments(&body).await?; - if arguments.len() > super::queries::ARGUMENT_BYTES { - return Err(refuse( - StatusCode::PAYLOAD_TOO_LARGE, - "update_arguments_exceed_budget", - )); - } + // PHP bounds update arguments by its structural payload ceiling, + // which rejects them durably, not by a request budget. A runtime + // external reference is resolved to its bytes, which the update + // keeps inline (`resolveToArray`). + let mut external = self.prepare_external(&body["input"]).await?; + let arguments = match external.as_mut() { + Some(prepared) => prepared.blob()?, + None => self.command_arguments(&body).await?, + }; let prepared = super::store::prepare_control_arguments( self.signal_codec.clone(), name.to_owned(), @@ -656,17 +837,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone()) + .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone(), self.structural, external.as_ref()) .await } Storage::Postgres(store) => { store - .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone()) + .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone(), self.structural, external.as_ref()) .await } Storage::MySql(store) => { store - .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone()) + .update_workflow(workflow_id, run_id, name, body, prepared, context, self.signal_codec.clone(), self.structural, external.as_ref()) .await } } @@ -699,11 +880,24 @@ impl Runtime { return Err(invalid("input", "The input field must be an array.")); } // Like PHP's AvroPayloadEnvelopeResolver, plain values are encoded - // as the query's arguments. - let arguments = match input { - None => super::queries::empty_arguments(self.signal_codec.clone()).await?, - Some(_) => self.command_arguments(&body).await?, + // as the query's arguments. A runtime external reference is claimed + // and resolved to its bytes, which the query carries inline. + let mut external = match input { + Some(_) => self.prepare_external(&body["input"]).await?, + None => None, + }; + let arguments = match (input, external.as_mut()) { + (None, _) => super::queries::empty_arguments(self.signal_codec.clone()).await?, + (Some(_), Some(prepared)) => prepared.blob()?, + (Some(_), None) => self.command_arguments(&body).await?, }; + if let Some(prepared) = &external { + match &*self.storage { + Storage::Sqlite(store) => store.claim_external(prepared).await?, + Storage::Postgres(store) => store.claim_external(prepared).await?, + Storage::MySql(store) => store.claim_external(prepared).await?, + } + } if arguments.len() > super::queries::ARGUMENT_BYTES { return Err(refuse( StatusCode::PAYLOAD_TOO_LARGE, @@ -932,6 +1126,41 @@ impl Runtime { Storage::MySql(store) => store.describe_task_queue(queue, &self.queries).await, } } + pub(crate) async fn build_id_action(&self, queue: &str, body: Value, action: super::deployments::Action) -> Result<(StatusCode, Value)> { + match &*self.storage { + Storage::Sqlite(store) => store.build_id_action(queue, &body, action).await, + Storage::Postgres(store) => store.build_id_action(queue, &body, action).await, + Storage::MySql(store) => store.build_id_action(queue, &body, action).await, + } + } + pub(crate) async fn priority_fairness(&self, queue: &str) -> Result { + match &*self.storage { + Storage::Sqlite(store) => store.priority_fairness(queue).await, + Storage::Postgres(store) => store.priority_fairness(queue).await, + Storage::MySql(store) => store.priority_fairness(queue).await, + } + } + pub(crate) async fn deployments(&self) -> Result { + match &*self.storage { + Storage::Sqlite(store) => store.deployments().await, + Storage::Postgres(store) => store.deployments().await, + Storage::MySql(store) => store.deployments().await, + } + } + pub(crate) async fn deployment(&self, name: &str) -> Result<(StatusCode, Value)> { + match &*self.storage { + Storage::Sqlite(store) => store.deployment(name).await, + Storage::Postgres(store) => store.deployment(name).await, + Storage::MySql(store) => store.deployment(name).await, + } + } + pub(crate) async fn deployment_action(&self, name: &str, action: super::deployments::Action) -> Result<(StatusCode, Value)> { + match &*self.storage { + Storage::Sqlite(store) => store.deployment_action(name, action).await, + Storage::Postgres(store) => store.deployment_action(name, action).await, + Storage::MySql(store) => store.deployment_action(name, action).await, + } + } pub(crate) async fn task_queue_build_ids(&self, queue: &str) -> Result { match &*self.storage { Storage::Sqlite(store) => store.task_queue_build_ids(queue, &self.queries).await, @@ -983,7 +1212,9 @@ impl Runtime { worker: Option, protocol: Option<&str>, ) -> Result { + self.check_command_payloads(&body).await?; let external = self.prepare_command_payloads(&body).await?; + let body = super::completion_rules::canonical_payloads(body)?; if let Some(commands) = body["commands"].as_array() { for command in commands .iter() @@ -1006,12 +1237,6 @@ impl Runtime { .iter() .filter(|command| command["type"] == "complete_update") { - if commands.len() != 1 { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_update_task_commands", - )); - } let result = super::envelope(command, "result")?; if result.len() > super::queries::RESULT_BYTES { return Err(refuse( @@ -1074,6 +1299,35 @@ impl Runtime { } } } + /// `WorkflowController::redriveRun`: the run must exist before its + /// request is validated. + pub(crate) async fn redrive_run(&self, workflow_id: &str, run_id: &str, body: Value, context: Value) -> Result<(StatusCode, Value)> { + if !self.run_in_namespace(workflow_id, run_id).await? { + return Err(super::redrive::run_not_found(workflow_id, run_id)); + } + let request_id = super::redrive::request_id(&body)?; + match &*self.storage { + Storage::Sqlite(store) => store.redrive_run(workflow_id, run_id, request_id, context).await, + Storage::Postgres(store) => store.redrive_run(workflow_id, run_id, request_id, context).await, + Storage::MySql(store) => store.redrive_run(workflow_id, run_id, request_id, context).await, + } + } + /// `WorkflowStub::attemptArchive` for the current run. + pub(crate) async fn archive_workflow( + &self, + workflow_id: &str, + run_id: Option<&str>, + body: Value, + context: Value, + ) -> Result<(StatusCode, Value)> { + let reason = super::archive::reason(&body, workflow_id)?; + let blob = super::archive::payload(reason.as_deref())?; + match &*self.storage { + Storage::Sqlite(store) => store.archive_workflow(workflow_id, run_id, reason, blob, context).await, + Storage::Postgres(store) => store.archive_workflow(workflow_id, run_id, reason, blob, context).await, + Storage::MySql(store) => store.archive_workflow(workflow_id, run_id, reason, blob, context).await, + } + } pub(crate) async fn request_cancellation( &self, workflow_id: &str, @@ -1123,6 +1377,21 @@ impl Runtime { } pub(crate) async fn complete_activity(&self, task_id: &str, body: Value) -> Result { let external = self.prepare_external(&body["result"]).await?; + // `ActivityOutcomeRecorder` measures the result it would store. + let stored = match &external { + Some(prepared) => prepared.stored(), + None => super::envelope(&body, "result")?, + }; + if let Some(exceeded) = super::memo_limits::Exceeded::payload(self.structural.payload_bytes, &stored) { + let (failure, blob) = + super::activity_failures::structural_failure(&exceeded, self.signal_codec.clone()).await?; + let limit = Some(exceeded); + return match &*self.storage { + Storage::Sqlite(store) => store.fail_activity(task_id, body, failure, blob, limit).await, + Storage::Postgres(store) => store.fail_activity(task_id, body, failure, blob, limit).await, + Storage::MySql(store) => store.fail_activity(task_id, body, failure, blob, limit).await, + }; + } match &*self.storage { Storage::Sqlite(store) => store.complete_activity(task_id, body, external).await, Storage::Postgres(store) => store.complete_activity(task_id, body, external).await, @@ -1136,9 +1405,9 @@ impl Runtime { let (failure, blob) = super::activity_failures::prepare_failure(&body, self.signal_codec.clone()).await?; match &*self.storage { - Storage::Sqlite(store) => store.fail_activity(task_id, body, failure, blob).await, - Storage::Postgres(store) => store.fail_activity(task_id, body, failure, blob).await, - Storage::MySql(store) => store.fail_activity(task_id, body, failure, blob).await, + Storage::Sqlite(store) => store.fail_activity(task_id, body, failure, blob, None).await, + Storage::Postgres(store) => store.fail_activity(task_id, body, failure, blob, None).await, + Storage::MySql(store) => store.fail_activity(task_id, body, failure, blob, None).await, } } } diff --git a/rust/src/runtime/external_registry.rs b/rust/src/runtime/external_registry.rs index 484dab8d0..7710bc65a 100644 --- a/rust/src/runtime/external_registry.rs +++ b/rust/src/runtime/external_registry.rs @@ -386,6 +386,15 @@ where Ok(()) } + /// Claims the reference in a transaction of its own, for a request + /// that records nothing else. + pub(super) async fn claim_external(&self, prepared: &super::external_resolution::PreparedExternal) -> Result<()> { + let mut tx = self.begin().await?; + self.claim_external_in(&mut tx, &prepared.reference, &prepared.uri).await?; + tx.commit().await?; + Ok(()) + } + pub(super) async fn mark_external_fetched(&self, expected: &Reference) -> Result<()> { let mut tx = self.begin().await?; let timestamp = now(); diff --git a/rust/src/runtime/external_resolution.rs b/rust/src/runtime/external_resolution.rs index 622812864..d847fc2ef 100644 --- a/rust/src/runtime/external_resolution.rs +++ b/rust/src/runtime/external_resolution.rs @@ -17,6 +17,18 @@ pub(crate) struct PreparedExternal { pub(super) type PreparedCommands = BTreeMap; impl PreparedExternal { + /// The verified payload: the Avro blob the reference stands for, read + /// once to validate it before the reference alone is stored. + pub(super) fn blob(&mut self) -> Result { + use std::io::Read; + let mut blob = String::new(); + self._snapshot + .read_to_string(&mut blob) + .map_err(|_| refuse(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported"))?; + self._snapshot.rewind().map_err(super::external_transport::snapshot_error)?; + Ok(blob.trim().to_owned()) + } + pub(super) fn stored(&self) -> String { format!( "{STORED_PREFIX}{}", @@ -98,32 +110,21 @@ impl Runtime { permit: Option>, ) -> Result> { if value.get("external_storage").is_some() { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "external_payload_unsupported", - )); + return Err(super::external_transport::provider_reference()); } let Some(reference) = value.get("external_payload") else { return Ok(None); }; - if value.as_object().is_none_or(|map| map.len() != 2) || value["codec"] != "avro" { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "external_payload_unsupported", - )); + // `incomingEnvelope`: exactly codec and external_payload; the codec + // is compared with the registered reference after its lookup. + if value.as_object().is_none_or(|map| map.len() != 2 || !map.contains_key("codec")) { + return Err(super::external_transport::typed_error(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported", + "External payload envelopes must contain exactly codec and external_payload.")); } - let reference = Reference::validate(reference).map_err(|_| { - refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "external_payload_unsupported", - ) + // `RuntimeExternalPayloadReference::validate`'s message. + let reference = Reference::validate(reference).map_err(|error| { + super::external_transport::typed_error(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported", error.to_string()) })?; - if value["codec"] != reference.codec { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "external_payload_integrity_mismatch", - )); - } if reference.size_bytes as u64 > self.external.max_bytes.max(1) { return Err(refuse( StatusCode::PAYLOAD_TOO_LARGE, @@ -132,6 +133,12 @@ impl Runtime { } let prepare = async { let registered = self.external_payload_for_fetch(&reference).await?; + if value["codec"] != reference.codec { + return Err(refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "external_payload_integrity_mismatch", + )); + } let storage = super::external_transport::local(self).await?; let snapshot = match permit { Some(permit) => { diff --git a/rust/src/runtime/external_transport.rs b/rust/src/runtime/external_transport.rs index db6f910c9..6da3fe16d 100644 --- a/rust/src/runtime/external_transport.rs +++ b/rust/src/runtime/external_transport.rs @@ -25,11 +25,54 @@ const CODEC: &str = "x-durable-workflow-payload-codec"; const SIZE: &str = "x-durable-workflow-payload-size"; const DIGEST: &str = "x-durable-workflow-payload-sha256"; +/// PHP's `RuntimeExternalPayloadException` reasons, which its transport +/// middleware renders as the typed external payload error on every route. +pub(super) const TYPED_REASONS: &[&str] = &[ + "external_payload_not_found", + "external_payload_expired", + "external_payload_unauthorized", + "external_payload_unavailable", + "external_payload_oversized", + "external_payload_unsupported", + "external_payload_integrity_mismatch", + "external_payload_namespace_bytes_exhausted", + "external_payload_namespace_objects_exhausted", + "external_payload_namespace_quota_unavailable", + "external_payload_completion_invalid", + "external_payload_completion_lease_rejected", + "external_payload_completion_conflict", + "external_payload_completion_budget_exhausted", +]; + +/// PHP's message for a reference refused as a request is resolved. PHP +/// words a few reasons differently on other paths; the reason, status and +/// retryability are the contract. +fn message(reason: &str) -> &str { + match reason { + "external_payload_not_found" => "External payload reference was not found in this namespace.", + "external_payload_expired" => "External payload reference has expired.", + "external_payload_oversized" => "External payload exceeds the runtime transport size limit.", + "external_payload_unsupported" => "External payload envelopes must contain exactly codec and external_payload.", + "external_payload_integrity_mismatch" => "External payload envelope codec does not match the runtime reference.", + "external_payload_unavailable" => "External payload storage is temporarily unavailable.", + "external_payload_completion_invalid" => "A canonical worker completion context is required.", + "external_payload_completion_lease_rejected" => "The payload upload does not belong to a current worker completion lease.", + "external_payload_completion_conflict" => "A completion payload slot is already bound to different bytes.", + "external_payload_completion_budget_exhausted" => "The bounded payload allowance for this completion lease is exhausted.", + other => other, + } +} + +/// Marks a response as PHP's typed external payload error, so the route's +/// envelope can be added to it. +#[derive(Clone, Copy)] +pub(super) struct TypedRefusal; + pub(super) fn error_body(status: StatusCode, reason: &str) -> Value { let retryable = status == StatusCode::SERVICE_UNAVAILABLE || status == StatusCode::TOO_MANY_REQUESTS; let mut body = json!({"schema":"durable-workflow.v2.runtime-external-payload-error.v1", - "reason":reason,"message":reason,"retryable":retryable,"status":status.as_u16()}); + "reason":reason,"message":message(reason),"retryable":retryable,"status":status.as_u16()}); if reason.starts_with("external_payload_namespace_") { body["retry_after_seconds"] = json!(60); } else if reason == "external_payload_completion_budget_exhausted" { @@ -38,6 +81,38 @@ pub(super) fn error_body(status: StatusCode, reason: &str) -> Value { body } +pub(super) const TYPED_SCHEMA: &str = "durable-workflow.v2.runtime-external-payload-error.v1"; + +/// A typed refusal with PHP's message for this case. +pub(super) fn typed_error(status: StatusCode, reason: &'static str, message: impl Into) -> RuntimeError { + let mut response = error_body(status, reason); + response["message"] = json!(message.into()); + RuntimeError::Protocol { status, response } +} + +/// The provider reference PHP's transport never accepts. +pub(super) fn provider_reference() -> RuntimeError { + typed_error(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported", + "Provider-specific external_storage references are not accepted by the runtime transport.") +} + +/// A typed refusal from a route's handler, with PHP's body and retry header. +pub(super) fn typed_refusal(status: StatusCode, reason: &str) -> Response { + typed_response(status, error_body(status, reason)) +} + +pub(super) fn typed_response(status: StatusCode, body: Value) -> Response { + let reason = body["reason"].as_str().unwrap_or_default().to_owned(); + let mut response = (status, Json(body)).into_response(); + if reason.starts_with("external_payload_namespace_") { + response.headers_mut().insert("retry-after", "60".parse().expect("static header")); + } else if reason == "external_payload_completion_budget_exhausted" { + response.headers_mut().insert("retry-after", "5".parse().expect("static header")); + } + response.extensions_mut().insert(TypedRefusal); + response +} + fn response_error(error: RuntimeError) -> Response { let (status, reason) = match &error { RuntimeError::Refused { status, reason } => (*status, *reason), diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 8511adf76..98febdb4c 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -67,6 +67,8 @@ pub fn router(runtime: Runtime) -> Router { delete(delete_search_attribute), ) .route("/api/workflows", get(list_workflows).post(start)) + .route("/api/activities", get(list_activities).post(start_activity)) + .route("/api/activities/{activity_id}", get(show_activity)) .route( "/api/schedules", get(super::schedule_list::list).post(super::schedule_http::create), @@ -150,6 +152,9 @@ pub fn router(runtime: Runtime) -> Router { "/api/workflows/{workflow_id}/runs/{run_id}/terminate", post(terminate_run), ) + .route("/api/workflows/{workflow_id}/archive", post(archive_current)) + .route("/api/workflows/{workflow_id}/runs/{run_id}/archive", post(archive_run)) + .route("/api/workflows/{workflow_id}/runs/{run_id}/redrive", post(redrive_run)) .route( "/api/workflows/{workflow_id}/query/{query_name}", post(query_current), @@ -203,6 +208,10 @@ pub fn router(runtime: Runtime) -> Router { ) .route("/api/system/operator-metrics", get(operator_metrics)) .route("/api/system/health", get(system_health)) + .route("/api/system/metrics", get(system_metrics).post(system_metrics)) + .route("/api/system/prometheus-metrics", get(prometheus_metrics)) + .route("/api/system/external-payload-cleanup", get(payload_cleanup_status)) + .route("/api/system/external-payload-cleanup/pass", post(payload_cleanup_pass)) .route("/api/system/operator-dashboard", get(operator_dashboard)) .route("/api/system/operator-dashboard/bounded", get(bounded_operator_dashboard)) .route( @@ -210,6 +219,7 @@ pub fn router(runtime: Runtime) -> Router { get(workflow_type_operator_dashboard), ) .route("/api/system/repair", get(repair_status)) + .route("/api/system/repair/pass", post(repair_pass)) .route("/api/system/retention", get(retention_status)) .route("/api/system/retention/pass", post(retention_pass)) .route("/api/system/activity-timeouts", get(activity_timeout_status)) @@ -220,18 +230,12 @@ pub fn router(runtime: Runtime) -> Router { .route("/api/service-catalog/{catalog}/{id}", get(service_catalog_show)) .route("/api/task-queues/{task_queue}", get(describe_task_queue)) .route("/api/task-queues/{task_queue}/build-ids", get(task_queue_build_ids)) - .route( - "/api/task-queues/{task_queue}/build-ids/promote", - post(build_id_rollout_unqualified), - ) - .route( - "/api/task-queues/{task_queue}/build-ids/drain", - post(build_id_rollout_unqualified), - ) - .route( - "/api/task-queues/{task_queue}/build-ids/resume", - post(build_id_rollout_unqualified), - ) + .route("/api/task-queues/{task_queue}/priority-fairness", get(priority_fairness)) + .route("/api/task-queues/{task_queue}/build-ids/promote", post(build_id_promote)) + .route("/api/task-queues/{task_queue}/build-ids/drain", post(build_id_drain)) + .route("/api/task-queues/{task_queue}/build-ids/resume", post(build_id_resume)) + .route("/api/deployments", get(list_deployments)) + .route("/api/deployments/{*name}", get(show_deployment).post(deployment_action)) .route( "/api/workers/{worker_id}", get(describe_worker).delete(deregister), @@ -288,7 +292,8 @@ pub fn router(runtime: Runtime) -> Router { post(complete_query), ) .route("/api/worker/query-tasks/{task_id}/fail", post(fail_query)) - .fallback(unavailable) + .fallback(route_fallback) + .method_not_allowed_fallback(route_fallback) .layer(middleware::from_fn(empty_list_body)) // `payload_limits` reads and bounds every body first. .layer(DefaultBodyLimit::disable()) @@ -314,8 +319,14 @@ async fn empty_list_body(request: Request, next: Next) -> Response { let Ok(bytes) = axum::body::to_bytes(body, 16).await else { return StatusCode::BAD_REQUEST.into_response(); }; - let body = if bytes.trim_ascii() == b"[]" { + // PHP reads an empty body, like `[]`, as empty input. + let empty = bytes.trim_ascii().is_empty() + && matches!(parts.method, axum::http::Method::POST | axum::http::Method::PUT | axum::http::Method::PATCH); + let body = if bytes.trim_ascii() == b"[]" || empty { parts.headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("2")); + if empty && !parts.headers.contains_key(header::CONTENT_TYPE) { + parts.headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + } Body::from("{}") } else { Body::from(bytes) @@ -558,161 +569,192 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) request.extensions_mut().insert(AuthenticatedPrincipal { principal, }); + // The workflow route a typed refusal came from, to name the workflow + // in its envelope. + let route = (!worker && path.starts_with("/api/workflows")).then(|| { + let mut route = Request::new(Body::empty()); + *route.method_mut() = request.method().clone(); + *route.uri_mut() = request.uri().clone(); + *route.extensions_mut() = request.extensions().clone(); + route + }); let mut response = next.run(request).await; + if response.extensions().get::().is_some() && (worker || route.is_some()) { + response = typed_refusal_envelope(response, route, worker).await; + } response .headers_mut() .insert(header, HeaderValue::from_static(version)); response } -fn named_namespace_operation(method: &str, path: &str) -> bool { - // Runtime credentials are server-wide; the namespace only scopes a tenant. - if path == "/api/runtime-credentials" || path.starts_with("/api/runtime-credentials/") { - return true; - } - if path == "/api/external-payloads/v1" { - return method == "POST"; - } - if path.starts_with("/api/external-payloads/v1/") { - return method == "GET"; +/// PHP renders a typed external payload error through `WorkerProtocol` or +/// `ControlPlaneProtocol`, which add the route's envelope fields. +async fn typed_refusal_envelope(response: Response, route: Option, worker: bool) -> Response { + let status = response.status(); + let (parts, body) = response.into_parts(); + let Ok(bytes) = axum::body::to_bytes(body, 64 * 1024).await else { + return (status, Json(json!({"reason":"external_payload_unavailable"}))).into_response(); + }; + let Ok(body) = serde_json::from_slice::(&bytes) else { + return Response::from_parts(parts, Body::from(bytes)); + }; + let mut response = admission_error(route.unwrap_or_else(|| Request::new(Body::empty())), worker, status, body).await; + if let Some(retry) = parts.headers.get("retry-after") { + response.headers_mut().insert("retry-after", retry.clone()); } + response +} + +/// The methods a named namespace may use on a route. +#[derive(Clone, Copy)] +enum Allowed { + Any, + Only(&'static [&'static str]), +} + +const READ: Allowed = Allowed::Only(&["GET"]); +const WRITE: Allowed = Allowed::Only(&["POST"]); +const READ_WRITE: Allowed = Allowed::Only(&["GET", "POST"]); + +/// What follows a family's prefix. +#[derive(Clone, Copy)] +enum Rest { + Anything, + NonEmpty, + OneSegment, +} + +/// Route families a named namespace may use, by prefix. +const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 8] = [ + // Runtime credentials are server-wide; the namespace only scopes a tenant. + ("/api/runtime-credentials/", Rest::Anything, Allowed::Any), + ("/api/external-payloads/v1/", Rest::Anything, READ), // The service catalog is namespace-scoped and, in native, always empty. - if path.starts_with("/api/service-catalog/") { - return method == "GET"; - } + ("/api/service-catalog/", Rest::Anything, READ), // Schedules are namespace-scoped rows, as in PHP. - if path == "/api/schedules" || path.starts_with("/api/schedules/") { - return true; - } - // Keep unimplemented families from reaching default-namespace selectors. - // Further families require their own shared namespace fixtures. - if path == "/api/workflows" { - return matches!(method, "GET" | "POST"); - } - if path == "/api/storage/test" { - return method == "POST"; - } + ("/api/schedules/", Rest::Anything, Allowed::Any), + // Standalone activities are host runs in the requested namespace. + ("/api/activities/", Rest::OneSegment, READ), // The registry is namespace-scoped state with no default-namespace selector. - if path == "/api/search-attributes" { - return matches!(method, "GET" | "POST"); - } - if path - .strip_prefix("/api/search-attributes/") - .is_some_and(|name| !name.is_empty() && !name.contains('/')) - { - return method == "DELETE"; - } + ("/api/search-attributes/", Rest::OneSegment, Allowed::Only(&["DELETE"])), // Deregistration deletes only the namespace's own registration. - if path - .strip_prefix("/api/workers/") - .is_some_and(|id| !id.is_empty() && !id.contains('/')) - { - return matches!(method, "GET" | "DELETE"); - } - // Task queue views are namespace-scoped reads. - if path == "/api/task-queues" - || path - .strip_prefix("/api/task-queues/") - .is_some_and(|queue| !queue.is_empty() && !queue.contains('/')) - { - return method == "GET"; - } - // Build-id cohorts are a namespace-scoped read too. Rollout changes are - // refused alike in every namespace. - if let Some(rest) = path.strip_prefix("/api/task-queues/") { - match rest.split('/').collect::>().as_slice() { - [queue, "build-ids"] if !queue.is_empty() => return method == "GET", - [queue, "build-ids", "promote" | "drain" | "resume"] if !queue.is_empty() => { - return method == "POST"; - } - _ => {} - } + ("/api/workers/", Rest::OneSegment, Allowed::Only(&["GET", "DELETE"])), + // Deployments are the namespace's own build-id rollouts. + ("/api/deployments/", Rest::NonEmpty, READ_WRITE), +]; + +fn named_namespace_operation(method: &str, path: &str) -> bool { + match namespace_route(path) { + Some(Allowed::Any) => true, + Some(Allowed::Only(methods)) => methods.contains(&method), + None => false, } - if path == "/api/workers" - || path == "/api/system/operator-metrics" - || path == "/api/system/health" - || path == "/api/system/operator-dashboard" - || path == "/api/system/operator-dashboard/bounded" - || path == "/api/system/operator-dashboard/bounded/workflow-types" +} + +/// The methods a named namespace may use on `path`. Other routes are refused, +/// so unimplemented families never reach default-namespace selectors; each +/// further family needs its own shared namespace fixtures. +fn namespace_route(path: &str) -> Option { + exact_namespace_route(path) + .or_else(|| { + NAMESPACE_FAMILIES.iter().find_map(|(prefix, shape, allowed)| { + let rest = path.strip_prefix(prefix)?; + let fits = match shape { + Rest::Anything => true, + Rest::NonEmpty => !rest.is_empty(), + Rest::OneSegment => !rest.is_empty() && !rest.contains('/'), + }; + fits.then_some(*allowed) + }) + }) + .or_else(|| path.strip_prefix("/api/task-queues/").and_then(task_queue_route)) + .or_else(|| path.strip_prefix("/api/workflows/").and_then(workflow_route)) + .or_else(|| path.strip_prefix("/api/worker/").and_then(worker_route)) +} + +fn exact_namespace_route(path: &str) -> Option { + Some(match path { + "/api/runtime-credentials" | "/api/schedules" => Allowed::Any, + "/api/workflows" | "/api/activities" | "/api/search-attributes" => READ_WRITE, + // The metrics describe the namespace's own state, read by GET or POST. + "/api/system/metrics" => READ_WRITE, + "/api/external-payloads/v1" | "/api/storage/test" => WRITE, + "/api/task-queues" + | "/api/deployments" + | "/api/workers" + | "/api/system/operator-metrics" + | "/api/system/health" + | "/api/system/operator-dashboard" + | "/api/system/operator-dashboard/bounded" + | "/api/system/operator-dashboard/bounded/workflow-types" + | "/api/system/prometheus-metrics" + | "/api/system/external-payload-cleanup" // Retention reads and prunes the namespace's own runs; the activity // timeout scan spans namespaces in PHP too. - || path == "/api/system/retention" - || path == "/api/system/activity-timeouts" - || path == "/api/system/repair" - { - return method == "GET"; - } - if path == "/api/system/retention/pass" || path == "/api/system/activity-timeouts/pass" { - return method == "POST"; + | "/api/system/retention" + | "/api/system/activity-timeouts" + | "/api/system/repair" => READ, + // The repair pass is server-wide in PHP too. The cleanup pass reclaims + // only the namespace's own expired payloads. + "/api/system/retention/pass" + | "/api/system/activity-timeouts/pass" + | "/api/system/repair/pass" + | "/api/system/external-payload-cleanup/pass" => WRITE, + _ => return None, + }) +} + +/// Task queue views, build-id cohorts and fairness are namespace-scoped reads; +/// rollout changes act on the namespace's own cohorts. +fn task_queue_route(rest: &str) -> Option { + match rest.split('/').collect::>().as_slice() { + [queue, ..] if queue.is_empty() => None, + [_] | [_, "build-ids" | "priority-fairness"] => Some(READ), + [_, "build-ids", "promote" | "drain" | "resume"] => Some(WRITE), + _ => None, } - if let Some(path) = path.strip_prefix("/api/workflows/") { - let parts: Vec<_> = path.split('/').collect(); - if matches!( - parts.as_slice(), - [_, "observation"] | [_, "runs", _, "observation"] - ) { - return method == "GET"; - } - if matches!(parts.as_slice(), [_, "runs", _, "activities", _, "result"]) { - return method == "GET"; - } +} + +fn workflow_route(rest: &str) -> Option { + match rest.split('/').collect::>().as_slice() { + [_, "observation"] + | [_, "runs", _, "observation"] + | [_, "runs", _, "activities", _, "result"] // Waterline's run detail: the run list, debug view and history export. // Each reads the instance within the request's namespace first. - if matches!( - parts.as_slice(), - [_, "runs"] | [_, "debug"] | [_, "runs", _, "debug"] | [_, "runs", _, "history", "export"] - ) { - return method == "GET"; - } - if matches!( - parts.as_slice(), - [_, "runs", _, "streams"] | [_, "runs", _, "streams", _] | [_, "runs", _, "streams", _, "items"] - ) { - return method == "GET"; - } - if matches!( - parts.as_slice(), - [_, "signal", _] | [_, "runs", _, "signal", _] - ) { - return method == "POST"; - } - if matches!( - parts.as_slice(), - [_, "query", _] | [_, "runs", _, "query", _] - ) { - return method == "POST"; - } - return matches!( - parts.as_slice(), - [_] | [_, "cancel" | "terminate"] - | [_, "runs", _] - | [ - _, - "runs", - _, - "history" | "cancel" | "terminate" | "activities" - ] - ); - } - if let Some(path) = path.strip_prefix("/api/worker/") { - let parts: Vec<_> = path.split('/').collect(); - return matches!( - parts.as_slice(), - ["register" | "heartbeat"] - | ["registrations", _] - | ["registrations", _, "deregister"] - | ["workflow-tasks" | "activity-tasks", "poll"] - | ["query-tasks", "poll"] - | ["query-tasks", _, "complete" | "fail" | "status"] - | ["workflow-tasks", _, "complete" | "history" | "heartbeat"] - | [ - "activity-tasks", - _, - "complete" | "fail" | "heartbeat" | "status" | "renew-lease" - ] - ); + | [_, "runs"] + | [_, "debug"] + | [_, "runs", _, "debug"] + | [_, "runs", _, "history", "export"] + | [_, "runs", _, "streams"] + | [_, "runs", _, "streams", _] + | [_, "runs", _, "streams", _, "items"] => Some(READ), + [_, "signal" | "query" | "update", _] + | [_, "runs", _, "signal" | "query" | "update", _] + | [_, "repair" | "request-cancellation"] + | [_, "runs", _, "repair" | "request-cancellation" | "redrive"] => Some(WRITE), + [_] + | [_, "cancel" | "terminate" | "archive"] + | [_, "runs", _] + | [_, "runs", _, "history" | "cancel" | "terminate" | "archive" | "activities"] => Some(Allowed::Any), + _ => None, } - false +} + +fn worker_route(rest: &str) -> Option { + matches!( + rest.split('/').collect::>().as_slice(), + ["register" | "heartbeat"] + | ["registrations", _] + | ["registrations", _, "deregister"] + | ["workflow-tasks" | "activity-tasks", "poll"] + | ["query-tasks", "poll"] + | ["query-tasks", _, "complete" | "fail" | "status"] + | ["workflow-tasks", _, "complete" | "fail" | "history" | "heartbeat"] + | ["activity-tasks", _, "complete" | "fail" | "heartbeat" | "status" | "renew-lease"] + ) + .then_some(Allowed::Any) } async fn list_namespaces(Extension(runtime): Extension) -> Result> { @@ -866,11 +908,17 @@ async fn cluster() -> Json { let principal_attribution: Value = serde_json::from_str(include_str!("principal-attribution-contract.v2.json")) .expect("valid bundled principal attribution contract"); + // Server 2.11.1's service catalog read contract. Native serves its eight + // reads; the SDK requires this advertisement before calling them. + let service_catalog: Value = + serde_json::from_str(include_str!("service-catalog-read-contract.v1.json")) + .expect("valid bundled service catalog read contract"); Json( json!({"version": env!("CARGO_PKG_VERSION"), "implementation": "rust", "development": true, "control_plane": {"version": "2", "request_contract": super::visibility::request_contract()}, "worker_protocol": {"version": super::WORKER_PROTOCOL_VERSION, "server_capabilities": capabilities()}, "auth_composition_contract": auth_composition, "principal_attribution_contract": principal_attribution, + "service_catalog_read_contract": service_catalog, "payload_codec": "avro", "qualified_for_php_takeover": false}), ) } @@ -955,6 +1003,39 @@ async fn start( Ok((status, Json(super::control_plane::Operation::Start.response(result)))) } +/// `POST /api/activities`: a standalone activity in its own host run. +async fn start_activity( + Extension(runtime): Extension, + command: CommandRequest, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + // PHP names the target `pending` until the start assigns an id. + let trimmed = |field: &str| body[field].as_str().map(str::trim).filter(|text| !text.is_empty()).map(str::to_owned); + let target = trimmed("activity_id").unwrap_or_else(|| "pending".into()); + let queue = trimmed("task_queue").unwrap_or_else(|| "default".into()); + let context = command.context(&body, &target, "standalone_activity_start", metadata(&[ + ("activity_type", trimmed("activity_type").map(Value::from)), + ("task_queue", Some(json!(queue))), + ])); + let started = runtime.start_standalone_activity(body, context).await?; + Ok((StatusCode::CREATED, Json(started))) +} + +async fn show_activity( + Extension(runtime): Extension, + Path(activity_id): Path, +) -> Result> { + Ok(Json(runtime.standalone_activity(&activity_id).await?)) +} + +async fn list_activities( + Extension(runtime): Extension, + Query(query): Query>, +) -> Result> { + let query = Value::Object(query.into_iter().map(|(key, value)| (key, Value::String(value))).collect()); + Ok(Json(runtime.standalone_activities(query).await?)) +} + async fn list_workflows( Extension(runtime): Extension, Query(query): Query, @@ -1022,6 +1103,63 @@ async fn close( Ok((status, Json(operation().response(response)))) } +/// `POST /api/workflows/{id}/archive` and its run-targeted route. +async fn archive( + runtime: Runtime, + request: CommandRequest, + workflow_id: &str, + run_id: Option<&str>, + body: Value, +) -> Result<(StatusCode, Json)> { + let operation = || super::control_plane::Operation::Archive; + let context = request.context(&body, workflow_id, "archive", metadata(&[ + ("request_id", present(&body, "request_id")), + ("reason", present(&body, "reason")), + ])); + let (status, response) = runtime + .archive_workflow(workflow_id, run_id, body, context) + .await + .map_err(|error| operation().refusal(error, workflow_id, run_id))?; + Ok((status, Json(operation().response(response)))) +} + +/// `POST /api/workflows/{id}/runs/{run}/redrive`. +async fn redrive_run( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, run_id)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let operation = || super::control_plane::Operation::Redrive; + let context = request.context(&body, &workflow_id, "redrive", metadata(&[ + ("request_id", present(&body, "request_id")), + ("continued_from_run_id", Some(json!(run_id))), + ])); + let (status, response) = runtime + .redrive_run(&workflow_id, &run_id, body, context) + .await + .map_err(|error| operation().refusal(error, &workflow_id, Some(&run_id)))?; + Ok((status, Json(operation().response(response)))) +} + +async fn archive_current( + Extension(runtime): Extension, + request: CommandRequest, + Path(workflow_id): Path, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + archive(runtime, request, &workflow_id, None, body).await +} + +async fn archive_run( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, run_id)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + archive(runtime, request, &workflow_id, Some(&run_id), body).await +} + async fn cancel_current( Extension(runtime): Extension, request: CommandRequest, @@ -1395,6 +1533,7 @@ fn metrics_config(runtime: &Runtime) -> super::operator_metrics::MetricsConfig { workflow_task_lease_seconds: runtime.queries.workflow_task_lease_seconds, activity_task_lease_seconds: runtime.queries.activity_task_lease_seconds, abandoned_upload_expiry_seconds: runtime.external.abandoned_seconds.max(1), + structural: runtime.structural, } } @@ -1432,6 +1571,14 @@ async fn repair_status(Extension(runtime): Extension) -> Result, body: axum::body::Bytes) -> Result { + let request = super::repair_pass::pass_request(&request_json(&body)?)?; + let report = runtime.repair_pass(&request).await?; + let failed = super::repair_pass::failed(&report); + Ok(pass_response((report, failed))) +} + async fn retention_status(Extension(runtime): Extension, Query(query): QueryMap) -> Result> { let limit = super::retention::status_limit(query.get("limit").map(String::as_str))?; Ok(Json(runtime.retention_status(limit).await?)) @@ -1473,6 +1620,26 @@ async fn activity_timeout_status( Ok(Json(runtime.activity_timeout_status(limit).await?)) } +async fn system_metrics(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.system_metrics().await?)) +} + +async fn prometheus_metrics(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.prometheus_metrics().await?)) +} + +async fn payload_cleanup_status(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.payload_cleanup_status(&metrics_config(&runtime)).await?)) +} + +/// PHP answers 207 when the pass blocked any reference. +async fn payload_cleanup_pass(Extension(runtime): Extension, body: axum::body::Bytes) -> Result { + let limit = super::payload_cleanup::pass_limit(&request_json(&body)?)?; + let report = runtime.payload_cleanup_pass(limit).await?; + let blocked = report["blocked"].as_i64().is_some_and(|blocked| blocked > 0); + Ok(pass_response((json!({"namespace": runtime.namespace(), "cleanup": report}), blocked))) +} + /// PHP answers 503 while a check reports an error. async fn system_health(Extension(runtime): Extension) -> Result { let (unavailable, body) = runtime.system_health(&metrics_config(&runtime)).await?; @@ -1514,13 +1681,75 @@ async fn task_queue_build_ids( ) -> Result> { runtime.task_queue_build_ids(&task_queue).await.map(Json) } -/// `POST /api/task-queues/{name}/build-ids/{promote|drain|resume}`. Native -/// keeps no build-id rollouts, so it refuses to promote, drain or resume one. -async fn build_id_rollout_unqualified() -> Result> { - Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "task_queue_build_id_rollout_unqualified", - )) +/// `POST /api/task-queues/{name}/build-ids/promote`. +async fn build_id_promote( + Extension(runtime): Extension, + Path(task_queue): Path, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + build_id_action(runtime, task_queue, body, super::deployments::Action::Promote).await +} +/// `POST /api/task-queues/{name}/build-ids/drain`. +async fn build_id_drain( + Extension(runtime): Extension, + Path(task_queue): Path, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + build_id_action(runtime, task_queue, body, super::deployments::Action::Drain).await +} +/// `POST /api/task-queues/{name}/build-ids/resume`. +async fn build_id_resume( + Extension(runtime): Extension, + Path(task_queue): Path, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + build_id_action(runtime, task_queue, body, super::deployments::Action::Resume).await +} +async fn build_id_action( + runtime: Runtime, + task_queue: String, + body: Value, + action: super::deployments::Action, +) -> Result<(StatusCode, Json)> { + let (status, body) = runtime.build_id_action(&task_queue, body, action).await?; + Ok((status, Json(body))) +} +/// `GET /api/task-queues/{name}/priority-fairness`. +async fn priority_fairness( + Extension(runtime): Extension, + Path(task_queue): Path, +) -> Result> { + runtime.priority_fairness(&task_queue).await.map(Json) +} +/// `GET /api/deployments`. +async fn list_deployments(Extension(runtime): Extension) -> Result> { + runtime.deployments().await.map(Json) +} +/// `GET /api/deployments/{name}`: the name holds slashes, +/// `namespace/task_queue@build_id`. +async fn show_deployment( + Extension(runtime): Extension, + Path(name): Path, +) -> Result<(StatusCode, Json)> { + let (status, body) = runtime.deployment(&name).await?; + Ok((status, Json(body))) +} +/// `POST /api/deployments/{name}/{promote|drain|resume|rollback}`. +async fn deployment_action( + Extension(runtime): Extension, + Path(name): Path, + uri: axum::http::Uri, +) -> Response { + let Some((name, action)) = name + .rsplit_once('/') + .and_then(|(name, action)| super::deployments::Action::named(action).map(|action| (name, action))) + else { + return route_fallback(axum::http::Method::POST, uri).await; + }; + match runtime.deployment_action(name, action).await { + Ok((status, body)) => (status, Json(body)).into_response(), + Err(error) => error.into_response(), + } } async fn describe_worker( Extension(runtime): Extension, @@ -1602,7 +1831,7 @@ async fn heartbeat_task( Path(task_id): Path, Json(body): Json, ) -> Result> { - runtime.heartbeat_task(&task_id, body).await.map(Json) + runtime.heartbeat_task(&task_id, body).await.map(|body| Json(worker_response(body))) } async fn deliver_cancellation( @@ -1624,6 +1853,8 @@ async fn complete_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { + // PHP's transport rules come before every guard, ownership included. + let body = super::completion_rules::checked(body).map_err(worker_refusal)?; if body["commands"].as_array().is_some_and(|commands| { commands .iter() @@ -1660,7 +1891,7 @@ async fn complete_workflow( "Workflow task not found.", ) }) - .map(Json) + .map(|body| Json(worker_response(body))) } /// Whether the request's worker protocol is `1.` or newer. @@ -1704,7 +1935,7 @@ async fn fail_workflow_task( "Workflow task not found.", ) }) - .map(Json) + .map(|body| Json(worker_response(body))) } async fn complete_activity( @@ -1725,7 +1956,7 @@ async fn complete_activity( "Activity task not found.", ) }) - .map(Json) + .map(|body| Json(worker_response(body))) } fn completion_refusal( @@ -1736,6 +1967,14 @@ fn completion_refusal( message: &str, ) -> super::RuntimeError { match error { + // The bridge's refusal of a batch it cannot apply, in PHP's + // completion shape. + super::RuntimeError::Refused { status, reason: "invalid_commands" } if status == StatusCode::CONFLICT => { + let mut response = json!({"task_id":task_id,"outcome":"completed","recorded":false,"run_id":null, + "run_status":null,"created_task_ids":[],"reason":"invalid_commands"}); + response[attempt_field] = attempt; + super::RuntimeError::Protocol { status, response: worker_response(response) } + } super::RuntimeError::Refused { status, reason } if status == StatusCode::CONFLICT => { let mut response = json!({"reason":reason,"message":reason,"task_id":task_id}); response[attempt_field] = attempt; @@ -1790,7 +2029,7 @@ async fn fail_activity( Path(task_id): Path, Json(body): Json, ) -> Result> { - runtime.fail_activity(&task_id, body).await.map(Json) + runtime.fail_activity(&task_id, body).await.map(|body| Json(worker_response(body))) } /// The query's command context, as PHP's controller builds it. fn query_context(request: &CommandRequest, body: &Value, workflow_id: &str, name: &str) -> Value { @@ -1905,6 +2144,21 @@ async fn service_catalog_show( } } +/// A request no native route serves gets PHP's answer for it: 501 when PHP +/// serves it and native does not yet, else PHP's 405 or 404. +async fn route_fallback(method: axum::http::Method, uri: axum::http::Uri) -> Response { + let path = uri.path(); + let (status, body) = match super::route_fallback::answer(method.as_str(), path) { + super::route_fallback::Answer::NotImplemented => return unavailable().await, + super::route_fallback::Answer::MethodNotAllowed(message) => { + (StatusCode::METHOD_NOT_ALLOWED, json!({"message":message,"reason":"method_not_allowed"})) + } + super::route_fallback::Answer::NotFound(message) => (StatusCode::NOT_FOUND, json!({"message":message,"reason":"not_found"})), + }; + // PHP's worker plane adds its protocol fields to every answer. + let body = if path.starts_with("/api/worker/") { worker_response(body) } else { body }; + (status, Json(body)).into_response() +} async fn unavailable() -> Response { refuse( StatusCode::NOT_IMPLEMENTED, diff --git a/rust/src/runtime/input_envelope.rs b/rust/src/runtime/input_envelope.rs index 4086c7947..8be9b815c 100644 --- a/rust/src/runtime/input_envelope.rs +++ b/rust/src/runtime/input_envelope.rs @@ -3,8 +3,7 @@ //! Its refusals are Laravel validation failures on `input.codec` and //! `input.blob`. use super::laravel_validation::failure_messages; -use super::{Result, RuntimeError, refuse}; -use axum::http::StatusCode; +use super::{Result, RuntimeError}; use serde_json::Value; /// `PayloadCodecContract::canonicalize`: null, empty or `avro` names the @@ -32,7 +31,7 @@ fn invalid(field: &str, message: String) -> RuntimeError { /// before the request reaches validation. pub(super) fn transport(body: &Value) -> Result<()> { if body["input"].get("external_storage").is_some() { - return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "external_payload_unsupported")); + return Err(super::external_transport::provider_reference()); } Ok(()) } diff --git a/rust/src/runtime/laravel_rules.rs b/rust/src/runtime/laravel_rules.rs index 28b3b471e..1f470b118 100644 --- a/rust/src/runtime/laravel_rules.rs +++ b/rust/src/runtime/laravel_rules.rs @@ -15,6 +15,8 @@ use serde_json::{Map, Value, json}; #[derive(Clone, Copy, Debug)] pub(crate) enum Rule { Required, + /// `present`: the attribute is in the request, even as null. + Present, Nullable, /// Validate only when the attribute is present. Sometimes, @@ -189,6 +191,7 @@ fn message(rule: Rule, attribute: &str, rules: &[Rule], value: Option<&Value>) - }; match rule { Rule::Required => format!("The {attribute} field is required."), + Rule::Present => format!("The {attribute} field must be present."), Rule::RequiredWith(other) => { format!("The {attribute} field is required when {} is present.", display(other, false)) } @@ -247,6 +250,7 @@ fn size(value: &Value, rules: &[Rule]) -> Option { fn passes(rule: Rule, data: &Value, value: Option<&Value>, rules: &[Rule]) -> bool { match rule { Rule::Required => filled(value), + Rule::Present => value.is_some(), Rule::RequiredWith(other) => !filled(lookup(data, other)) || filled(value), Rule::Nullable | Rule::Sometimes => true, Rule::NotBlank => value.and_then(Value::as_str).is_some_and(|text| text.chars().any(|character| !character.is_whitespace())), @@ -298,50 +302,67 @@ pub(crate) fn validate(body: &Value, rules: &Rules) -> Result<(), RuntimeError> /// Every failing attribute and its messages, in Laravel's order. pub(crate) fn errors(body: &Value, rules: &Rules) -> Vec<(String, Vec)> { - let mut checks: Vec<(String, bool, &[Rule])> = Vec::new(); - let mut expanded = Vec::new(); - for (pattern, list) in rules { - if pattern.contains('*') { - expanded.extend(expand(body, pattern).into_iter().map(|path| (path, true, *list))); - } else { - checks.push(((*pattern).to_owned(), false, *list)); - } + checks(body, rules) + .into_iter() + .filter_map(|(attribute, expanded, list)| { + let messages = attribute_errors(body, &attribute, expanded, list); + (!messages.is_empty()).then_some((attribute, messages)) + }) + .collect() +} + +/// The attributes to check: named ones in rule order, then each wildcard's +/// expansions, as Laravel lists them. The flag marks an expansion. +fn checks(body: &Value, rules: &Rules) -> Vec<(String, bool, &'static [Rule])> { + let named = rules + .iter() + .filter(|(pattern, _)| !pattern.contains('*')) + .map(|&(pattern, list)| (pattern.to_owned(), false, list)); + let expanded = rules + .iter() + .filter(|(pattern, _)| pattern.contains('*')) + .flat_map(|&(pattern, list)| expand(body, pattern).into_iter().map(move |path| (path, true, list))); + named.chain(expanded).collect() +} + +/// One attribute's messages. An implicit rule (required, required_with, +/// present) stops at its first failure, as Laravel's does. +fn attribute_errors(body: &Value, attribute: &str, expanded: bool, list: &[Rule]) -> Vec { + let value = lookup(body, attribute); + if value.is_none() && list.iter().any(|rule| matches!(rule, Rule::Sometimes)) { + return Vec::new(); } - checks.extend(expanded); - let mut errors: Vec<(String, Vec)> = Vec::new(); - for (attribute, raw, list) in checks { - let value = lookup(body, &attribute); - let nullable = list.iter().any(|rule| matches!(rule, Rule::Nullable)); - if value.is_none() && list.iter().any(|rule| matches!(rule, Rule::Sometimes)) { + let nullable = list.iter().any(|rule| matches!(rule, Rule::Nullable)); + let name = display(attribute, expanded); + let mut messages = Vec::new(); + for rule in list { + if matches!(rule, Rule::Nullable | Rule::Sometimes) { continue; } - let name = display(&attribute, raw); - let mut messages = Vec::new(); - for rule in list { - let implicit = matches!(rule, Rule::Required | Rule::RequiredWith(_)); - if matches!(rule, Rule::Nullable | Rule::Sometimes) { - continue; - } - // presentOrRuleIsImplicit, then isNotNullIfMarkedAsNullable. - let blank = value.is_some_and(|value| value.as_str().is_some_and(|text| text.trim().is_empty())); - if !implicit && (value.is_none() || blank) { - continue; - } - if !implicit && nullable && value.is_some_and(Value::is_null) { - continue; - } - if !passes(*rule, body, value, list) { - messages.push(message(*rule, &name, list, value)); - if implicit { - break; - } - } + let implicit = matches!(rule, Rule::Required | Rule::RequiredWith(_) | Rule::Present); + if !implicit && skipped(value, nullable) { + continue; } - if !messages.is_empty() { - errors.push((attribute, messages)); + if !passes(*rule, body, value, list) { + messages.push(message(*rule, &name, list, value)); + if implicit { + break; + } } } - errors + messages +} + +/// Laravel's presentOrRuleIsImplicit and isNotNullIfMarkedAsNullable: a rule +/// that is not implicit skips a missing or blank value, and a null one when +/// the attribute is nullable. +fn skipped(value: Option<&Value>, nullable: bool) -> bool { + match value { + None => true, + Some(Value::String(text)) => text.trim().is_empty(), + Some(Value::Null) => nullable, + Some(_) => false, + } } /// Laravel's 422 ValidationException body: the first message, a count of the diff --git a/rust/src/runtime/laravel_validation.rs b/rust/src/runtime/laravel_validation.rs index 064aaa6e7..6ba4c1c2d 100644 --- a/rust/src/runtime/laravel_validation.rs +++ b/rust/src/runtime/laravel_validation.rs @@ -4,12 +4,27 @@ use super::RuntimeError; use axum::http::StatusCode; use serde_json::{Map, Value, json}; +/// The characters Laravel's `Str::trim` removes from either end: PHP's +/// default trim set, `\s` and `Str::INVISIBLE_CHARACTERS`. A pinned, +/// reviewed set rather than a changing Unicode whitespace predicate. +pub(super) fn laravel_trim(text: &str) -> &str { + text.trim_matches(|ch| { + matches!(ch, + '\0' | '\t'..='\r' | ' ' | '\u{85}' | '\u{a0}' | '\u{ad}' | + '\u{34f}' | '\u{61c}' | '\u{115f}' | '\u{1160}' | '\u{1680}' | + '\u{17b4}' | '\u{17b5}' | '\u{180e}' | '\u{2000}'..='\u{200f}' | + '\u{2028}' | '\u{2029}' | '\u{202f}' | '\u{205f}' | + '\u{2060}'..='\u{2065}' | '\u{206a}'..='\u{206f}' | '\u{2800}' | + '\u{3000}' | '\u{3164}' | '\u{feff}' | '\u{ffa0}' | '\u{1d159}' | + '\u{1d173}'..='\u{1d17a}' | '\u{e0020}') + }) +} + /// Laravel's TrimStrings and ConvertEmptyStringsToNull run before validation. pub(super) fn normalized(value: Option<&Value>) -> Option { - let trimmed = |c: char| c.is_whitespace() || matches!(c, '\u{feff}' | '\u{200b}' | '\u{200e}'); match value { Some(Value::String(text)) => { - let text = text.trim_matches(trimmed); + let text = laravel_trim(text); Some(if text.is_empty() { Value::Null } else { @@ -176,7 +191,7 @@ fn size(value: &Value, numeric: bool) -> f64 { } } -fn passes(rule: Rule, value: Option<&Value>, numeric: bool) -> bool { +pub(super) fn passes(rule: Rule, value: Option<&Value>, numeric: bool) -> bool { let Some(value) = value else { return !matches!(rule, Required); }; @@ -201,7 +216,7 @@ fn passes(rule: Rule, value: Option<&Value>, numeric: bool) -> bool { } } -fn default_message(attribute: &str, rule: Rule, numeric: bool, array: bool) -> String { +pub(super) fn default_message(attribute: &str, rule: Rule, numeric: bool, array: bool) -> String { let kind = if numeric { "numeric" } else if array { "array" } else { "string" }; match (rule, kind) { (Required, _) => format!("The {attribute} field is required."), diff --git a/rust/src/runtime/memo_json.rs b/rust/src/runtime/memo_json.rs index 6e7c12df2..0be49db71 100644 --- a/rust/src/runtime/memo_json.rs +++ b/rust/src/runtime/memo_json.rs @@ -168,7 +168,11 @@ impl<'de> Deserialize<'de> for InputJson { key == "codec" && super::input_envelope::codec_refusal(&super::start_request::normalized(value.clone())).is_some() }); - Ok(InputJson(if refused_codec || super::start_request::envelope_keys(&keys) { + // PHP's transport middleware sees any input object naming an + // external reference before the controller takes its values, + // and refuses one that is not exactly codec and reference. + let external = keys.iter().any(|key| matches!(*key, "external_payload" | "external_storage")); + Ok(InputJson(if refused_codec || external || super::start_request::envelope_keys(&keys) { Value::Object(entries.into_iter().collect()) } else { Value::Array(entries.into_iter().map(|(_, value)| value).collect()) diff --git a/rust/src/runtime/memo_limits.rs b/rust/src/runtime/memo_limits.rs index 37af2ef3a..fbf931e67 100644 --- a/rust/src/runtime/memo_limits.rs +++ b/rust/src/runtime/memo_limits.rs @@ -1,4 +1,5 @@ -//! The package's DW-primary, WORKFLOW-legacy structural memo limit. +//! The package's DW-primary, WORKFLOW-legacy structural limits: the memo +//! ceiling and `StructuralLimits`' signal, update and payload ceilings. use super::{Result, refuse}; use axum::http::StatusCode; use regex_lite::Regex; @@ -14,9 +15,16 @@ pub(super) fn typed_null(value: &str) -> bool { matches!(value.to_ascii_lowercase().as_str(), "null" | "(null)") } +#[cfg(test)] fn limit(primary: Option<&str>, legacy: Option<&str>) -> Option { + limit_or(primary, legacy, DEFAULT_BYTES) +} + +/// A limit at or below zero disables its check, as `intConfig` and the +/// guards' `$limit <= 0` do. +fn limit_or(primary: Option<&str>, legacy: Option<&str>, default: usize) -> Option { let Some(value) = primary.filter(|value| !typed_null(value)).or(legacy) else { - return Some(DEFAULT_BYTES); + return Some(default); }; let bytes = cast_integer(value); (bytes > 0).then_some(bytes.max(0) as usize) @@ -54,21 +62,100 @@ pub(super) fn cast_numeric_string(value: &str) -> i64 { } pub(super) fn configured_limit() -> Result> { - let read = |key| match env::var(key) { + configured("MEMO_SIZE_BYTES", DEFAULT_BYTES, "invalid_memo_limit_configuration") +} + +fn configured(suffix: &str, default: usize, invalid: &'static str) -> Result> { + let read = |key: String| match env::var(key) { Ok(value) => Ok(Some(value)), Err(env::VarError::NotPresent) => Ok(None), - Err(env::VarError::NotUnicode(_)) => Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_memo_limit_configuration", - )), + Err(env::VarError::NotUnicode(_)) => Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, invalid)), }; - let primary = read("DW_V2_LIMIT_MEMO_SIZE_BYTES")?; + let primary = read(format!("DW_V2_LIMIT_{suffix}"))?; let legacy = if primary.as_deref().is_none_or(typed_null) { - read("WORKFLOW_V2_LIMIT_MEMO_SIZE_BYTES")? + read(format!("WORKFLOW_V2_LIMIT_{suffix}"))? } else { None }; - Ok(limit(primary.as_deref(), legacy.as_deref())) + Ok(limit_or(primary.as_deref(), legacy.as_deref(), default)) +} + +/// `StructuralLimits`' per-run signal and update ceilings and its +/// single-payload byte ceiling. `None` is a disabled check. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StructuralLimits { + /// Received signals a run may hold (DW_V2_LIMIT_PENDING_SIGNALS). + pub pending_signals: Option, + /// Accepted updates a run may hold (DW_V2_LIMIT_PENDING_UPDATES). + pub pending_updates: Option, + /// Bytes of one serialized payload blob (DW_V2_LIMIT_PAYLOAD_SIZE_BYTES). + pub payload_bytes: Option, +} + +impl Default for StructuralLimits { + fn default() -> Self { + Self { + pending_signals: Some(5000), + pending_updates: Some(500), + payload_bytes: Some(2_097_152), + } + } +} + +impl StructuralLimits { + pub fn from_env() -> Result { + let defaults = Self::default(); + let invalid = "invalid_structural_limit_configuration"; + Ok(Self { + pending_signals: configured("PENDING_SIGNALS", defaults.pending_signals.unwrap(), invalid)?, + pending_updates: configured("PENDING_UPDATES", defaults.pending_updates.unwrap(), invalid)?, + payload_bytes: configured("PAYLOAD_SIZE_BYTES", defaults.payload_bytes.unwrap(), invalid)?, + }) + } +} + +/// One exceeded ceiling, as `StructuralLimitExceededException` reports it. +#[derive(Clone, Debug)] +pub(super) struct Exceeded { + pub(super) kind: &'static str, + pub(super) value: i64, + pub(super) configured: i64, + pub(super) message: String, +} + +impl Exceeded { + fn new(kind: &'static str, value: usize, configured: usize, message: String) -> Self { + let int = |value: usize| i64::try_from(value).unwrap_or(i64::MAX); + Self { kind, value: int(value), configured: int(configured), message } + } + + /// `guardPendingSignals`: received signals at or above the ceiling. + pub(super) fn pending_signals(limit: Option, count: i64) -> Option { + let (limit, count) = (limit?, usize::try_from(count).unwrap_or(0)); + (count >= limit).then(|| Self::new("pending_signal_count", count, limit, + format!("Structural limit exceeded: {count} pending signals (limit {limit})."))) + } + + /// `guardPendingUpdates`: accepted updates at or above the ceiling. + pub(super) fn pending_updates(limit: Option, count: i64) -> Option { + let (limit, count) = (limit?, usize::try_from(count).unwrap_or(0)); + (count >= limit).then(|| Self::new("pending_update_count", count, limit, + format!("Structural limit exceeded: {count} pending updates (limit {limit})."))) + } + + /// `guardPayloadSize`: a serialized blob longer than the ceiling. + pub(super) fn payload(limit: Option, blob: &str) -> Option { + let (limit, bytes) = (limit?, blob.len()); + (bytes > limit).then(|| Self::new("payload_size", bytes, limit, + format!("Structural limit exceeded: payload size {bytes} bytes (limit {limit} bytes)."))) + } + + /// The command payload's diagnostic fields + /// (`structuralLimitExceededCommandPayloadAttributes`). + pub(super) fn fields(&self) -> serde_json::Value { + serde_json::json!({"structural_limit_kind":self.kind,"structural_limit_value":self.value, + "structural_limit_configured":self.configured}) + } } #[cfg(test)] @@ -106,4 +193,21 @@ mod tests { assert_eq!(limit(Some("25"), Some("9")), Some(25)); assert_eq!(limit(None, Some("null")), None); } + + #[test] + fn structural_ceilings_use_php_defaults_and_disable_at_zero() { + assert_eq!(limit_or(None, None, 5000), Some(5000)); + assert_eq!(limit_or(Some("2"), None, 5000), Some(2)); + assert_eq!(limit_or(Some("0"), None, 500), None); + assert_eq!(limit_or(Some("-3"), Some("9"), 500), None); + assert_eq!(limit_or(Some("null"), Some("9"), 500), Some(9)); + let signals = Exceeded::pending_signals(Some(2), 2).unwrap(); + assert_eq!((signals.kind, signals.value, signals.configured), ("pending_signal_count", 2, 2)); + assert_eq!(signals.message, "Structural limit exceeded: 2 pending signals (limit 2)."); + assert!(Exceeded::pending_signals(Some(2), 1).is_none()); + assert!(Exceeded::pending_updates(None, 900).is_none()); + let payload = Exceeded::payload(Some(300), &"x".repeat(556)).unwrap(); + assert_eq!(payload.message, "Structural limit exceeded: payload size 556 bytes (limit 300 bytes)."); + assert!(Exceeded::payload(Some(300), &"x".repeat(300)).is_none()); + } } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 88948c1c4..d60cb0473 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -63,19 +63,30 @@ mod http; mod laravel_validation; mod memo_json; mod memo_limits; +mod standalone_activities; +mod archive; +mod command_payloads; +mod completion_rules; +mod deployments; +mod priority_fairness; +mod route_fallback; +mod redrive; mod memos; pub mod mysql; mod namespace_cleanup; mod namespaces; mod operator_dashboard; mod operator_metrics; +mod payload_cleanup; mod payload_limits; mod poll_readiness; pub mod postgres; mod previews; +mod prometheus_metrics; mod queries; mod query_authority; mod repair_candidates; +mod repair_pass; mod retention; mod runtime_credentials; mod run_observation; @@ -108,6 +119,7 @@ mod start_definition; mod start_request; mod storage_test; mod store; +mod system_metrics; mod system_health; mod task_failures; mod task_repair; @@ -132,6 +144,7 @@ mod workflow_task_guard; pub use execution::Runtime; pub use external_config::ExternalPayloadConfig; +pub use memo_limits::StructuralLimits; pub use http::router; /// The worker protocol version native implements and advertises. @@ -220,7 +233,13 @@ impl IntoResponse for RuntimeError { eprintln!("{diagnostic}"); } let (status, reason) = match self { + Self::Protocol { status, response } if response["schema"] == external_transport::TYPED_SCHEMA => { + return external_transport::typed_response(status, response); + } Self::Protocol { status, response } => return (status, Json(response)).into_response(), + Self::Refused { status, reason } if external_transport::TYPED_REASONS.contains(&reason) => { + return external_transport::typed_refusal(status, reason); + } Self::Refused { status, reason } => (status, reason), _ => { // Database errors can contain stored payloads. Keep responses diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index c0ec3cac5..6d09ee903 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -25,7 +25,7 @@ use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; pub(super) const REDISPATCH_AFTER_SECONDS: i64 = 3; /// Laravel binds a bare timestamp as `Y-m-d H:i:s`. -fn whole_second(at: DateTime) -> DateTime { +pub(super) fn whole_second(at: DateTime) -> DateTime { at.with_nanosecond(0).unwrap_or(at) } @@ -232,7 +232,7 @@ where "schedules": self.schedule_metrics(now).await?, "workers": self.worker_metrics().await?, "backend": backend_capabilities(now, DB::PHP_CONNECTION), - "structural_limits": structural_limits(DB::PHP_CONNECTION), + "structural_limits": structural_limits(DB::PHP_CONNECTION, &config.structural, self.memo_size_limit), "update_wait": {"completion_timeout_seconds": 10, "poll_interval_milliseconds": 50}, "repair_policy": repair_policy(), "matching_role": matching_role(config), @@ -1027,7 +1027,7 @@ where /// PHP's `RuntimeExternalPayloadCleanupMetrics`: the expired registrations /// and the namespace's recorded cleanup passes. - async fn external_payload_cleanup_metrics(&self, config: &MetricsConfig) -> Result { + pub(super) async fn external_payload_cleanup_metrics(&self, config: &MetricsConfig) -> Result { let bound = whole_second(now()); let stats = Self::query("SELECT * FROM runtime_external_payload_cleanup_stats WHERE namespace=$1") @@ -1084,14 +1084,9 @@ where /// published PHP's repair pass would select across every namespace. pub(super) async fn repair_status(&self) -> Result { let at = now(); - let bound = whole_second(at); - let cutoff = whole_second(at - Duration::seconds(REDISPATCH_AFTER_SECONDS)); - let mut tasks = Self::query(repair_candidates::TASK_SCOPES); - for time in [bound, bound, bound, cutoff, cutoff, cutoff, bound] { - tasks = tasks.bind(DB::bind_time(time)); - } - let tasks = scope_rows::(tasks.fetch_all(&self.pool).await?)?; - let runs = scope_rows::(Self::query(repair_candidates::RUN_SCOPES).fetch_all(&self.pool).await?)?; + let unfiltered = super::repair_pass::PassRequest::default(); + let tasks = self.repairable_task_scopes(&unfiltered, at).await?; + let runs = self.missing_task_scopes(&unfiltered).await?; let oldest_run = match Self::query(repair_candidates::OLDEST_RUN).fetch_optional(&self.pool).await? { Some(row) => DB::optional_instant(&row, "wait_started_at")? .or(DB::optional_instant(&row, "started_at")?) @@ -1363,6 +1358,8 @@ pub(crate) struct MetricsConfig { pub(crate) workflow_task_lease_seconds: i64, pub(crate) activity_task_lease_seconds: i64, pub(crate) abandoned_upload_expiry_seconds: i64, + /// The honored signal, update and payload ceilings. + pub(crate) structural: super::StructuralLimits, } fn empty_repair_snapshot() -> Value { @@ -1386,16 +1383,19 @@ fn empty_repair_snapshot() -> Value { }) } -fn structural_limits(connection: &str) -> Value { +/// `StructuralLimits::snapshot`: honored ceilings as configured, where a +/// disabled one reads 0 (`intConfig`'s `max(0, value)`). +fn structural_limits(connection: &str, limits: &super::StructuralLimits, memo: Option) -> Value { + let shown = |limit: Option| limit.unwrap_or(0); let configured = json!({ "pending_activity_count": 2000, "pending_child_count": 1000, "pending_timer_count": 2000, - "pending_signal_count": 5000, - "pending_update_count": 500, + "pending_signal_count": shown(limits.pending_signals), + "pending_update_count": shown(limits.pending_updates), "command_batch_size": 1000, - "payload_size_bytes": 2_097_152, - "memo_size_bytes": 262_144, + "payload_size_bytes": shown(limits.payload_bytes), + "memo_size_bytes": shown(memo), "search_attribute_size_bytes": 40960, "history_transaction_size": 5000, "warning_threshold_percent": 80, @@ -1415,7 +1415,8 @@ fn structural_limits(connection: &str) -> Value { /// What this runtime needs from its backends, in PHP's capability shape. fn backend_capabilities(now: DateTime, connection: &str) -> Value { - let limits = structural_limits(connection); + // Only the backend's issues are reported here, which no ceiling changes. + let limits = structural_limits(connection, &super::StructuralLimits::default(), None); let issues = limits["issues"].clone(); let severity = if issues.as_array().is_some_and(|issues| !issues.is_empty()) { "info" @@ -1450,7 +1451,7 @@ fn backend_capabilities(now: DateTime, connection: &str) -> Value { /// Native claims ready tasks on each poll; it runs no repair loop. /// The grouped candidate rows of one repair candidate query. -fn scope_rows(rows: Vec) -> Result> +pub(super) fn scope_rows(rows: Vec) -> Result> where usize: ColumnIndex, for<'r> i64: Decode<'r, DB>, diff --git a/rust/src/runtime/payload_cleanup.rs b/rust/src/runtime/payload_cleanup.rs new file mode 100644 index 000000000..ed98bb6df --- /dev/null +++ b/rust/src/runtime/payload_cleanup.rs @@ -0,0 +1,317 @@ +//! `GET /api/system/external-payload-cleanup` and +//! `POST /api/system/external-payload-cleanup/pass`, as published Server 2.11 +//! serves them (`RuntimeExternalPayloadCleanup::runPass`). +//! +//! A pass first renews or drops the namespace's expired completion-upload +//! budgets, then reclaims its expired, unclaimed payload references, oldest +//! first. Each reference is reclaimed under its object lock: a reference that +//! shares its object with another only loses its row, and a sole reference +//! also deletes its object. Each pass is recorded in the namespace's cleanup +//! totals. +//! +//! Native stores objects only with the local driver. A reference it cannot +//! map to the namespace's local storage (no enabled local policy, or an object +//! outside the current root) is blocked rather than deleted. Native never +//! takes an online-backup hold; a hold row PHP left active still blocks +//! deletion, and a missing row means no hold. +use super::Result; +use super::activity_completion_uploads::Context; +use super::laravel_validation::{Rule, failure_messages, rule_errors}; +use super::operator_metrics::whole_second; +use super::store::now; +use crate::external_payload::{LocalStorage, REFERENCE_SCHEMA, Reference}; +use chrono::{DateTime, Duration, Utc}; +use serde_json::{Value, json}; + +/// `RuntimeExternalPayloadCleanup::DEFAULT_BATCH_SIZE`. +pub(crate) const DEFAULT_BATCH_SIZE: i64 = 100; +/// `RuntimeExternalPayloadCleanup::MAX_BATCH_SIZE`. +pub(crate) const MAX_BATCH_SIZE: i64 = 1000; + +/// The pass's rule: `limit` is `nullable|integer|min:1|max:1000`, by default 100. +pub(super) fn pass_limit(body: &Value) -> Result { + let body = &super::start_request::normalized(body.clone()); + let rules: &[(&str, &[Rule])] = &[("limit", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(MAX_BATCH_SIZE)])]; + if let Some(error) = failure_messages(rule_errors(body, rules, |_, _| None)) { + return Err(error); + } + Ok(super::laravel_validation::integer(&body["limit"]).unwrap_or(DEFAULT_BATCH_SIZE)) +} + +enum Outcome { + Skipped, + DeletedObject, + DeletedShared, + Blocked, + StorageFailure, +} + +#[derive(Default)] +struct Report { + processed: i64, + deleted_references: i64, + deleted_backing_objects: i64, + shared_objects_preserved: i64, + skipped: i64, + blocked: i64, + storage_driver_failures: i64, +} + +store_methods! { + /// `externalPayloadCleanupStatus`: the namespace's cleanup metrics. + pub(crate) async fn payload_cleanup_status(&self, config: &super::operator_metrics::MetricsConfig) -> Result { + Ok(json!({"namespace": self.namespace.as_ref(), "cleanup": self.external_payload_cleanup_metrics(config).await?})) + } + + /// The pass's report; the caller answers 207 when it blocked anything. + pub(crate) async fn payload_cleanup_pass( + &self, + limit: i64, + retention_seconds: i64, + local: Option<&LocalStorage>, + ) -> Result { + let limit = limit.clamp(1, MAX_BATCH_SIZE); + let cutoff = now(); + self.completion_budget_cleanup(limit, cutoff, retention_seconds.max(1)).await?; + let candidates = Self::query("SELECT id FROM runtime_external_payloads WHERE namespace=$1 AND retained_at IS NULL AND expires_at IS NOT NULL AND expires_at<=$2 ORDER BY expires_at,id LIMIT $3") + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(cutoff)) + .bind(limit) + .fetch_all(&self.pool) + .await?; + let mut report = Report::default(); + for candidate in &candidates { + report.processed += 1; + match self.cleanup_reference(&DB::string(candidate, "id")?, cutoff, local).await? { + Outcome::Skipped => report.skipped += 1, + Outcome::DeletedObject => { + report.deleted_references += 1; + report.deleted_backing_objects += 1; + } + Outcome::DeletedShared => { + report.deleted_references += 1; + report.shared_objects_preserved += 1; + } + Outcome::Blocked => report.blocked += 1, + Outcome::StorageFailure => { + report.blocked += 1; + report.storage_driver_failures += 1; + } + } + } + self.record_cleanup_stats(&report).await?; + let backlog = Self::scalar("SELECT COUNT(*) FROM runtime_external_payloads WHERE namespace=$1 AND retained_at IS NULL AND expires_at IS NOT NULL AND expires_at<=$2") + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(cutoff)) + .fetch_one(&self.pool) + .await?; + Ok(json!({ + "processed": report.processed, "deleted_references": report.deleted_references, + "deleted_backing_objects": report.deleted_backing_objects, + "shared_objects_preserved": report.shared_objects_preserved, "skipped": report.skipped, + "blocked": report.blocked, "storage_driver_failures": report.storage_driver_failures, + "backlog": backlog, "scan_limit": limit, + "scan_pressure": backlog > 0 && candidates.len() as i64 >= limit, + })) + } + + /// `RuntimePayloadCompletionUploads::cleanup`: an expired budget whose + /// activity may still complete gets another retention period; any other + /// expired budget is dropped. + async fn completion_budget_cleanup(&self, limit: i64, cutoff: DateTime, retention_seconds: i64) -> Result<()> { + let candidates = Self::query("SELECT id,context FROM runtime_payload_completion_budgets WHERE expires_at<=$1 AND namespace=$2 ORDER BY expires_at LIMIT $3") + .bind(DB::bind_time(cutoff)) + .bind(self.namespace.as_ref()) + .bind(limit) + .fetch_all(&self.pool) + .await?; + for candidate in &candidates { + let id = DB::string(candidate, "id")?; + // Native holds activity contexts only; any other cannot resume. + let context = Context::parse(&DB::document_row(candidate, "context")?.to_string()).ok(); + let mut tx = self.begin().await?; + // As in PHP, completion ownership is read before the object lock. + let may_resume = match &context { + Some(context) => { + let (task, attempt, owner) = context.lease_identity(); + self.activity_completion_may_resume(&mut tx, task, attempt, owner).await? + } + None => false, + }; + Self::payload_object_lock(&mut tx, &id).await?; + let Some(row) = Self::query("SELECT expires_at FROM runtime_payload_completion_budgets WHERE id=$1") + .bind(&id) + .fetch_optional(&mut *tx) + .await? + else { + tx.commit().await?; + continue; + }; + if DB::instant(&row, "expires_at")? > cutoff { + tx.commit().await?; + continue; + } + if may_resume { + let at = now(); + Self::query("UPDATE runtime_payload_completion_budgets SET expires_at=$1,updated_at=$2 WHERE id=$3") + .bind(DB::bind_time(at + Duration::seconds(retention_seconds))) + .bind(DB::bind_time(at)) + .bind(&id) + .execute(&mut *tx) + .await?; + } else { + Self::query("DELETE FROM runtime_payload_completion_budgets WHERE id=$1") + .bind(&id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + } + Ok(()) + } + + /// `cleanupReference`, under the reference's object lock. + async fn cleanup_reference(&self, id: &str, cutoff: DateTime, local: Option<&LocalStorage>) -> Result { + let Some(first) = Self::query("SELECT storage_uri FROM runtime_external_payloads WHERE id=$1") + .bind(id) + .fetch_optional(&self.pool) + .await? + else { + return Ok(Outcome::Skipped); + }; + let uri = DB::string(&first, "storage_uri")?; + let mut tx = self.begin().await?; + Self::payload_object_lock(&mut tx, &uri).await?; + let Some(row) = Self::query("SELECT * FROM runtime_external_payloads WHERE id=$1") + .bind(id) + .fetch_optional(&mut *tx) + .await? + else { + tx.commit().await?; + return Ok(Outcome::Skipped); + }; + if DB::optional_instant(&row, "retained_at")?.is_some() + || DB::optional_instant(&row, "expires_at")?.is_none_or(|expires| expires > cutoff) + { + tx.commit().await?; + return Ok(Outcome::Skipped); + } + let uri = DB::string(&row, "storage_uri")?; + let owners = Self::scalar("SELECT COUNT(*) FROM runtime_external_payloads WHERE storage_uri_sha256=$1 AND storage_uri=$2") + .bind(DB::string(&row, "storage_uri_sha256")?) + .bind(&uri) + .fetch_one(&mut *tx) + .await?; + if owners > 1 { + Self::query("DELETE FROM runtime_external_payloads WHERE id=$1").bind(id).execute(&mut *tx).await?; + tx.commit().await?; + return Ok(Outcome::DeletedShared); + } + let reference = Reference { + schema: REFERENCE_SCHEMA, + reference_id: id.to_owned(), + codec: "avro", + size_bytes: DB::number(&row, "size_bytes")?, + sha256: DB::string(&row, "sha256")?, + }; + let Some(local) = local.filter(|local| local.uri_for(&reference).is_ok_and(|current| current == uri)) else { + tx.commit().await?; + return Ok(Outcome::Blocked); + }; + if Self::backup_hold_active(&mut tx).await? { + tx.commit().await?; + return Ok(Outcome::Blocked); + } + if local.delete(&reference).await.is_err() { + tx.commit().await?; + return Ok(Outcome::StorageFailure); + } + Self::query("DELETE FROM runtime_external_payloads WHERE id=$1").bind(id).execute(&mut *tx).await?; + tx.commit().await?; + Ok(Outcome::DeletedObject) + } + + /// `ExternalPayloadBackupHold::deleting`: an owner whose hold has not + /// expired pauses reclamation. + async fn backup_hold_active(tx: &mut super::transaction::Transaction<'_, DB>) -> Result { + // PostgreSQL keeps the owner as a uuid; only its presence matters. + let Some(hold) = Self::query("SELECT CASE WHEN owner IS NULL THEN 0 ELSE 1 END AS held,expires_at FROM runtime_external_payload_backup_hold WHERE id=1") + .fetch_optional(&mut **tx) + .await? + else { + return Ok(false); + }; + Ok(DB::number(&hold, "held")? == 1 + && DB::optional_instant(&hold, "expires_at")?.is_some_and(|expires| expires > now())) + } + + /// `recordStats` for an existing namespace. Times are whole seconds, as + /// Eloquent stores them. + async fn record_cleanup_stats(&self, report: &Report) -> Result<()> { + let namespace = self.namespace.as_ref(); + if Self::scalar("SELECT COUNT(*) FROM workflow_namespaces WHERE name=$1") + .bind(namespace) + .fetch_one(&self.pool) + .await? + == 0 + { + return Ok(()); + } + let at = whole_second(now()); + let mut tx = self.begin().await?; + if Self::scalar("SELECT COUNT(*) FROM runtime_external_payload_cleanup_stats WHERE namespace=$1") + .bind(namespace) + .fetch_one(&mut *tx) + .await? + == 0 + { + Self::query("INSERT INTO runtime_external_payload_cleanup_stats(namespace,created_at,updated_at) VALUES ($1,$2,$3)") + .bind(namespace) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + } + let status = if report.storage_driver_failures > 0 { + "failed" + } else if report.blocked > 0 { + "blocked" + } else { + "completed" + }; + Self::query(concat!( + "UPDATE runtime_external_payload_cleanup_stats SET passes_total=passes_total+1,", + "deleted_references_total=deleted_references_total+$1,deleted_backing_objects_total=deleted_backing_objects_total+$2,", + "shared_objects_preserved_total=shared_objects_preserved_total+$3,blocked_outcomes_total=blocked_outcomes_total+$4,", + "storage_driver_failures_total=storage_driver_failures_total+$5,last_processed=$6,last_deleted_references=$7,", + "last_deleted_backing_objects=$8,last_shared_objects_preserved=$9,last_blocked_outcomes=$10,", + "last_storage_driver_failures=$11,last_pass_status=$12,last_completed_at=$13,updated_at=$14 WHERE namespace=$15" + )) + .bind(report.deleted_references) + .bind(report.deleted_backing_objects) + .bind(report.shared_objects_preserved) + .bind(report.blocked) + .bind(report.storage_driver_failures) + .bind(report.processed) + .bind(report.deleted_references) + .bind(report.deleted_backing_objects) + .bind(report.shared_objects_preserved) + .bind(report.blocked) + .bind(report.storage_driver_failures) + .bind(status) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(namespace) + .execute(&mut *tx) + .await?; + if report.storage_driver_failures > 0 { + Self::query("UPDATE runtime_external_payload_cleanup_stats SET last_storage_failure_at=$1 WHERE namespace=$2") + .bind(DB::bind_time(at)) + .bind(namespace) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } +} diff --git a/rust/src/runtime/php-routes.v2.11.1.json b/rust/src/runtime/php-routes.v2.11.1.json new file mode 100644 index 000000000..1f5d3a466 --- /dev/null +++ b/rust/src/runtime/php-routes.v2.11.1.json @@ -0,0 +1 @@ +{"source":"php artisan route:list --json, ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102","wide_parameters":{"api/deployments/{name}":"name"},"routes":[["GET|HEAD","/"],["GET|HEAD","api/activities"],["POST","api/activities"],["GET|HEAD","api/activities/{activityId}"],["POST","api/bridge-adapters/webhook/{adapter}"],["GET|HEAD","api/cluster/info"],["GET|HEAD","api/deployments"],["GET|HEAD","api/deployments/{name}"],["POST","api/deployments/{name}/drain"],["POST","api/deployments/{name}/promote"],["POST","api/deployments/{name}/resume"],["POST","api/deployments/{name}/rollback"],["POST","api/external-payloads/v1"],["GET|HEAD","api/external-payloads/v1/{referenceId}"],["GET|HEAD","api/health"],["GET|HEAD","api/namespaces"],["POST","api/namespaces"],["GET|HEAD","api/namespaces/{namespace}"],["PUT","api/namespaces/{namespace}"],["DELETE","api/namespaces/{namespace}"],["PUT","api/namespaces/{namespace}/external-storage"],["GET|HEAD","api/ready"],["GET|HEAD","api/runtime-credentials"],["GET|HEAD","api/runtime-credentials/{credentialId}"],["PUT","api/runtime-credentials/{credentialId}"],["DELETE","api/runtime-credentials/{credentialId}"],["POST","api/runtime-credentials/{credentialId}/rotate"],["GET|HEAD","api/schedules"],["POST","api/schedules"],["GET|HEAD","api/schedules/{scheduleId}"],["PUT","api/schedules/{scheduleId}"],["DELETE","api/schedules/{scheduleId}"],["POST","api/schedules/{scheduleId}/backfill"],["GET|HEAD","api/schedules/{scheduleId}/history"],["POST","api/schedules/{scheduleId}/pause"],["POST","api/schedules/{scheduleId}/resume"],["POST","api/schedules/{scheduleId}/trigger"],["GET|HEAD","api/search-attributes"],["POST","api/search-attributes"],["DELETE","api/search-attributes/{name}"],["GET|HEAD","api/service-catalog/calls"],["GET|HEAD","api/service-catalog/calls/{callId}"],["GET|HEAD","api/service-catalog/endpoints"],["GET|HEAD","api/service-catalog/endpoints/{endpointId}"],["GET|HEAD","api/service-catalog/operations"],["GET|HEAD","api/service-catalog/operations/{operationId}"],["GET|HEAD","api/service-catalog/services"],["GET|HEAD","api/service-catalog/services/{serviceId}"],["GET|HEAD","api/service-endpoints"],["POST","api/service-endpoints"],["GET|HEAD","api/service-endpoints/{endpointName}"],["PUT","api/service-endpoints/{endpointName}"],["DELETE","api/service-endpoints/{endpointName}"],["GET|HEAD","api/service-endpoints/{endpointName}/services"],["POST","api/service-endpoints/{endpointName}/services"],["GET|HEAD","api/service-endpoints/{endpointName}/services/{serviceName}"],["PUT","api/service-endpoints/{endpointName}/services/{serviceName}"],["DELETE","api/service-endpoints/{endpointName}/services/{serviceName}"],["GET|HEAD","api/service-endpoints/{endpointName}/services/{serviceName}/operations"],["POST","api/service-endpoints/{endpointName}/services/{serviceName}/operations"],["GET|HEAD","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}"],["PUT","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}"],["DELETE","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}"],["POST","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/execute"],["GET|HEAD","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}"],["POST","api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}/cancel"],["POST","api/storage/test"],["GET|HEAD","api/system/activity-timeouts"],["POST","api/system/activity-timeouts/pass"],["GET|HEAD","api/system/external-payload-cleanup"],["POST","api/system/external-payload-cleanup/pass"],["GET|HEAD","api/system/health"],["GET|POST|HEAD","api/system/metrics"],["GET|HEAD","api/system/operator-dashboard"],["GET|HEAD","api/system/operator-dashboard/bounded"],["GET|HEAD","api/system/operator-dashboard/bounded/workflow-types"],["GET|HEAD","api/system/operator-metrics"],["GET|HEAD","api/system/prometheus-metrics"],["GET|HEAD","api/system/repair"],["POST","api/system/repair/pass"],["GET|HEAD","api/system/retention"],["POST","api/system/retention/pass"],["GET|HEAD","api/task-queues"],["GET|HEAD","api/task-queues/{taskQueue}"],["GET|HEAD","api/task-queues/{taskQueue}/build-ids"],["POST","api/task-queues/{taskQueue}/build-ids/drain"],["POST","api/task-queues/{taskQueue}/build-ids/promote"],["POST","api/task-queues/{taskQueue}/build-ids/resume"],["GET|HEAD","api/task-queues/{taskQueue}/priority-fairness"],["GET|HEAD","api/worker-sessions"],["GET|HEAD","api/worker-sessions/{sessionId}"],["POST","api/worker/activity-tasks/poll"],["POST","api/worker/activity-tasks/{taskId}/acknowledge-cancellation"],["POST","api/worker/activity-tasks/{taskId}/complete"],["POST","api/worker/activity-tasks/{taskId}/fail"],["POST","api/worker/activity-tasks/{taskId}/heartbeat"],["POST","api/worker/activity-tasks/{taskId}/renew-lease"],["POST","api/worker/activity-tasks/{taskId}/status"],["POST","api/worker/heartbeat"],["POST","api/worker/query-tasks/poll"],["POST","api/worker/query-tasks/{queryTaskId}/complete"],["POST","api/worker/query-tasks/{queryTaskId}/fail"],["GET|HEAD","api/worker/query-tasks/{queryTaskId}/status"],["POST","api/worker/register"],["DELETE","api/worker/registrations/{workerId}"],["POST","api/worker/registrations/{workerId}/deregister"],["POST","api/worker/sessions"],["DELETE","api/worker/sessions/{sessionId}"],["POST","api/worker/sessions/{sessionId}/heartbeat"],["POST","api/worker/update-validation-tasks/poll"],["POST","api/worker/update-validation-tasks/{taskId}/approve"],["POST","api/worker/update-validation-tasks/{taskId}/reject"],["POST","api/worker/workflow-tasks/poll"],["POST","api/worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint"],["POST","api/worker/workflow-tasks/{taskId}/cancellation-scopes/deliver"],["POST","api/worker/workflow-tasks/{taskId}/cancellation-scopes/open"],["POST","api/worker/workflow-tasks/{taskId}/cancellation-scopes/prepare"],["POST","api/worker/workflow-tasks/{taskId}/complete"],["POST","api/worker/workflow-tasks/{taskId}/deliver-cancellation"],["POST","api/worker/workflow-tasks/{taskId}/fail"],["POST","api/worker/workflow-tasks/{taskId}/heartbeat"],["POST","api/worker/workflow-tasks/{taskId}/history"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/checkpoint"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/checkpoint-group"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/prepare"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/recover"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledge-cancellation"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat"],["POST","api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome"],["GET|HEAD","api/workers"],["GET|HEAD","api/workers/{workerId}"],["DELETE","api/workers/{workerId}"],["GET|HEAD","api/workflows"],["POST","api/workflows"],["POST","api/workflows/import/embedded-v2"],["POST","api/workflows/import/waterline-v1"],["GET|HEAD","api/workflows/{workflowId}"],["POST","api/workflows/{workflowId}/archive"],["POST","api/workflows/{workflowId}/cancel"],["GET|HEAD","api/workflows/{workflowId}/debug"],["GET|HEAD","api/workflows/{workflowId}/message-streams"],["GET|HEAD","api/workflows/{workflowId}/message-streams/{streamName}"],["POST","api/workflows/{workflowId}/message-streams/{streamName}/messages"],["GET|HEAD","api/workflows/{workflowId}/nexus-operations"],["GET|HEAD","api/workflows/{workflowId}/observation"],["POST","api/workflows/{workflowId}/query/{queryName}"],["POST","api/workflows/{workflowId}/repair"],["POST","api/workflows/{workflowId}/request-cancellation"],["GET|HEAD","api/workflows/{workflowId}/runs"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/activities"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/activities/{activityId}/result"],["POST","api/workflows/{workflowId}/runs/{runId}/archive"],["POST","api/workflows/{workflowId}/runs/{runId}/cancel"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/debug"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/history"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/history/export"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/nexus-operations"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/observation"],["POST","api/workflows/{workflowId}/runs/{runId}/query/{queryName}"],["POST","api/workflows/{workflowId}/runs/{runId}/redrive"],["POST","api/workflows/{workflowId}/runs/{runId}/repair"],["POST","api/workflows/{workflowId}/runs/{runId}/request-cancellation"],["POST","api/workflows/{workflowId}/runs/{runId}/signal/{signalName}"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/streams"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/streams/{streamName}"],["POST","api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close"],["POST","api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items"],["GET|HEAD","api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items"],["POST","api/workflows/{workflowId}/runs/{runId}/terminate"],["POST","api/workflows/{workflowId}/runs/{runId}/update/{updateName}"],["POST","api/workflows/{workflowId}/signal/{signalName}"],["POST","api/workflows/{workflowId}/terminate"],["POST","api/workflows/{workflowId}/update/{updateName}"],["GET|HEAD","storage/{path}"],["PUT","storage/{path}"],["GET|HEAD","up"]]} diff --git a/rust/src/runtime/priority_fairness.rs b/rust/src/runtime/priority_fairness.rs new file mode 100644 index 000000000..fe8fd5655 --- /dev/null +++ b/rust/src/runtime/priority_fairness.rs @@ -0,0 +1,120 @@ +//! `GET /api/task-queues/{queue}/priority-fairness`, as published PHP 2.11.1's +//! `TaskQueuePriorityFairnessSurface` answers it: each task kind's ready tasks +//! by priority tier and fairness class, and the recent dispatch score of each +//! class. +//! +//! PHP keeps those scores in `InMemoryTaskFairnessState`, which starts empty +//! on every request of its Apache server. So each class reports a score of 0, +//! and the fairness reorder never moves a poll's first candidate: a poll takes +//! the oldest ready task of the most urgent tier, as native's poll does. +use super::{Result, backend::Backend, schedule_cron::php_compare, store::Store}; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +/// `TaskPriority::DEFAULT`. +const DEFAULT_PRIORITY: i64 = 5; +/// `TaskFairnessKey::DEFAULT_CLASS`. +const DEFAULT_CLASS: &str = "__default__"; + +/// One fairness class in a priority tier. +struct Class { + key: String, + fairness_key: Option, + count: i64, + weight: i64, +} + +/// `surfaceFor`: one task kind's ready tasks, in the rows' order. +fn surface(rows: &[(Option, Option, Option)]) -> Value { + let mut tiers: Vec<(i64, Vec)> = Vec::new(); + let mut seen: Vec = Vec::new(); + for (priority, fairness_key, weight) in rows { + let priority = priority.unwrap_or(DEFAULT_PRIORITY); + let fairness_key = fairness_key.clone().filter(|key| !key.is_empty()); + let key = fairness_key.clone().unwrap_or_else(|| DEFAULT_CLASS.to_owned()); + if !seen.contains(&key) { + seen.push(key.clone()); + } + let index = match tiers.iter().position(|(tier, _)| *tier == priority) { + Some(index) => index, + None => { + tiers.push((priority, Vec::new())); + tiers.len() - 1 + } + }; + let classes = &mut tiers[index].1; + match classes.iter_mut().find(|class| class.key == key) { + Some(class) => class.count += 1, + // The class keeps its first task's weight. + None => classes.push(Class { key, fairness_key, count: 1, weight: weight.filter(|weight| *weight >= 1).unwrap_or(1) }), + } + } + // `ksort`, for the tiers and for each tier's classes. + tiers.sort_by_key(|(priority, _)| *priority); + let tiers: Vec = tiers + .into_iter() + .map(|(priority, mut classes)| { + classes.sort_by(|left, right| php_compare(&left.key, &right.key)); + let count: i64 = classes.iter().map(|class| class.count).sum(); + let classes: Vec = classes + .iter() + .map(|class| json!({"fairness_key":class.fairness_key,"count":class.count,"fairness_weight":class.weight})) + .collect(); + json!({"priority":priority,"count":count,"classes":classes}) + }) + .collect(); + let recent: Vec = seen + .iter() + .map(|key| json!({"fairness_key":(key != DEFAULT_CLASS).then_some(key),"score":0})) + .collect(); + json!({"ready_tasks":rows.len(),"priority_tiers":tiers,"recent_dispatch":recent}) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + pub(crate) async fn priority_fairness(&self, queue: &str) -> Result { + let mut surfaces = Vec::with_capacity(2); + for kind in ["workflow", "activity"] { + let rows = Self::query("SELECT priority,fairness_key,fairness_weight FROM workflow_tasks WHERE namespace=$1 AND queue=$2 AND task_type=$3 AND status='ready' ORDER BY id") + .bind(self.namespace.as_ref()).bind(queue).bind(kind).fetch_all(&self.pool).await?; + let rows = rows + .iter() + .map(|row| Ok((DB::optional_number(row, "priority")?, DB::optional_string(row, "fairness_key")?, DB::optional_number(row, "fairness_weight")?))) + .collect::>>()?; + surfaces.push(surface(&rows)); + } + let activity = surfaces.pop(); + let workflow = surfaces.pop(); + Ok(json!({"namespace":self.namespace.as_ref(),"queue":queue,"workflow_task":workflow,"activity_task":activity})) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tiers_and_classes_follow_php_s_snapshot() { + let row = |priority: i64, key: Option<&str>, weight: i64| (Some(priority), key.map(str::to_owned), Some(weight)); + let rows = [row(5, Some("tenant-a"), 1), row(5, Some("tenant-a"), 3), row(5, Some("tenant-b"), 2), row(5, None, 1), + row(1, Some("urgent"), 1), row(9, Some(""), 1)]; + assert_eq!(surface(&rows), json!({"ready_tasks":6,"priority_tiers":[ + {"priority":1,"count":1,"classes":[{"fairness_key":"urgent","count":1,"fairness_weight":1}]}, + {"priority":5,"count":4,"classes":[{"fairness_key":null,"count":1,"fairness_weight":1}, + {"fairness_key":"tenant-a","count":2,"fairness_weight":1},{"fairness_key":"tenant-b","count":1,"fairness_weight":2}]}, + {"priority":9,"count":1,"classes":[{"fairness_key":null,"count":1,"fairness_weight":1}]}], + "recent_dispatch":[{"fairness_key":"tenant-a","score":0},{"fairness_key":"tenant-b","score":0}, + {"fairness_key":null,"score":0},{"fairness_key":"urgent","score":0}]})); + assert_eq!(surface(&[]), json!({"ready_tasks":0,"priority_tiers":[],"recent_dispatch":[]})); + } +} diff --git a/rust/src/runtime/prometheus_metrics.rs b/rust/src/runtime/prometheus_metrics.rs new file mode 100644 index 000000000..2c41497c8 --- /dev/null +++ b/rust/src/runtime/prometheus_metrics.rs @@ -0,0 +1,508 @@ +//! `GET /api/system/prometheus-metrics`, as published Server 2.11 serves it +//! (`PrometheusMetricsSummary::snapshot`): the namespace's workflow, activity +//! and task-queue series for Prometheus scrapers, each family bounded with its +//! suppression disclosed. +//! +//! Native refuses the series-limit settings, so every family uses PHP's +//! default limit. Latency buckets compare each duration with a numeric bound +//! on every backend. Published PHP binds the bounds as text, which SQLite +//! orders after every number, so PHP on SQLite counts every duration in every +//! bucket (server#683); native reports the cumulative histogram there too. +//! Task and worker times compare with whole seconds, as Laravel binds them. +use super::Result; +use super::operator_metrics::whole_second; +use super::store::now; +use chrono::{DateTime, Duration, Utc}; +use serde_json::{Map, Value, json}; +use std::collections::HashMap; + +/// PHP's default series limit for each family. +const SERIES_LIMIT: usize = 100; + +/// `LATENCY_BUCKETS_SECONDS`, keyed as PHP casts each bound to a string. +const BUCKETS: [(&str, f64); 10] = [ + ("0.1", 0.1), + ("0.5", 0.5), + ("1", 1.0), + ("2.5", 2.5), + ("5", 5.0), + ("10", 10.0), + ("30", 30.0), + ("60", 60.0), + ("300", 300.0), + ("900", 900.0), +]; + +const WORKFLOW_SELECTION: &str = "bounded_task_queue_and_workflow_type_ascending"; +const ACTIVITY_SELECTION: &str = "bounded_task_queue_workflow_type_and_activity_type_ascending"; +const QUEUE_SELECTION: &str = "task_queue_name_ascending"; + +/// Cumulative bucket counts of a duration expression, one column per bound. +macro_rules! buckets { + ($available:literal, $duration:literal) => { + concat!( + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=0.1 THEN 1 END) AS bucket_0,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=0.5 THEN 1 END) AS bucket_1,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=1 THEN 1 END) AS bucket_2,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=2.5 THEN 1 END) AS bucket_3,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=5 THEN 1 END) AS bucket_4,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=10 THEN 1 END) AS bucket_5,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=30 THEN 1 END) AS bucket_6,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=60 THEN 1 END) AS bucket_7,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=300 THEN 1 END) AS bucket_8,", + "COUNT(CASE WHEN ", $available, " AND ", $duration, "<=900 THEN 1 END) AS bucket_9" + ) + }; +} + +/// The workflow series up to and including the last candidate (queue, type). +macro_rules! workflow_rows { + ($float:literal) => { + concat!( + "SELECT COALESCE(queue,'default') AS task_queue,workflow_type,COUNT(*) AS started_total,", + "COUNT(CASE WHEN status='completed' THEN 1 END) AS completed_total,", + "COUNT(CASE WHEN status='failed' THEN 1 END) AS failed_total,", + "COUNT(CASE WHEN status='cancelled' THEN 1 END) AS cancelled_total,", + "COUNT(CASE WHEN status='terminated' THEN 1 END) AS terminated_total,", + "COUNT(CASE WHEN status_bucket='running' THEN 1 END) AS running,", + "COUNT(duration_ms) AS latency_count,", + "CAST(COALESCE(SUM(duration_ms/1000.0),0) AS ", $float, ") AS latency_sum,", + buckets!("duration_ms IS NOT NULL", "duration_ms/1000.0"), + " FROM workflow_run_summaries WHERE namespace=$1", + " AND (COALESCE(queue,'default')<$2 OR (COALESCE(queue,'default')=$3 AND workflow_type<=$4))", + " GROUP BY COALESCE(queue,'default'),workflow_type" + ) + }; +} + +/// The activity series up to and including the last candidate key. +macro_rules! activity_rows { + ($float:literal, $duration:literal) => { + concat!( + "SELECT COALESCE(activity_executions.queue,workflow_runs.queue,'default') AS task_queue,", + "workflow_runs.workflow_type AS workflow_type,activity_executions.activity_type AS activity_type,", + "COUNT(*) AS started_total,", + "COUNT(CASE WHEN activity_executions.status='completed' THEN 1 END) AS completed_total,", + "COUNT(CASE WHEN activity_executions.status='failed' THEN 1 END) AS failed_total,", + "COUNT(CASE WHEN activity_executions.status='cancelled' THEN 1 END) AS cancelled_total,", + "COUNT(CASE WHEN activity_executions.status IN ('pending','running') THEN 1 END) AS running,", + "COUNT(CASE WHEN activity_executions.started_at IS NOT NULL AND activity_executions.closed_at IS NOT NULL THEN 1 END) AS latency_count,", + "CAST(COALESCE(SUM(CASE WHEN activity_executions.started_at IS NOT NULL AND activity_executions.closed_at IS NOT NULL THEN ", + $duration, + " END),0) AS ", + $float, + ") AS latency_sum,", + buckets!( + "activity_executions.started_at IS NOT NULL AND activity_executions.closed_at IS NOT NULL", + $duration + ), + " FROM activity_executions JOIN workflow_runs ON workflow_runs.id=activity_executions.workflow_run_id", + " WHERE workflow_runs.namespace=$1 AND (COALESCE(activity_executions.queue,workflow_runs.queue,'default')<$2", + " OR (COALESCE(activity_executions.queue,workflow_runs.queue,'default')=$3 AND (workflow_runs.workflow_type<$4", + " OR (workflow_runs.workflow_type=$5 AND activity_executions.activity_type<=$6))))", + " GROUP BY COALESCE(activity_executions.queue,workflow_runs.queue,'default'),workflow_runs.workflow_type,activity_executions.activity_type" + ) + }; +} + +/// `whereTaskRuntimeVisible`: open work, or work created or dispatched within +/// the reporting minute. +macro_rules! visible_tasks { + ($namespace:literal, $created:literal, $dispatched:literal) => { + concat!( + " FROM workflow_tasks WHERE namespace=", $namespace, + " AND task_type IN ('workflow','activity') AND (status IN ('ready','leased') OR created_at>=", $created, + " OR (last_dispatched_at IS NOT NULL AND last_dispatched_at>=", $dispatched, "))" + ) + }; +} + +/// A float as PHP's `json_encode` writes it: whole values without a fraction. +fn php_float(value: f64) -> Value { + if value.fract() == 0.0 && value.abs() < 9.0e15 { + json!(value as i64) + } else { + json!(value) + } +} + +/// PHP's `round($value, 6)` for the finite sums reported here. +fn round6(value: f64) -> f64 { + (value * 1_000_000.0).round() / 1_000_000.0 +} + +/// `seriesSnapshot`. +struct Snapshot { + series: Vec, + observed: usize, + suppressed_started: i64, + lower_bound: bool, +} + +impl Snapshot { + fn precision(&self) -> &'static str { + if self.lower_bound { "lower_bound" } else { "exact" } + } + fn suppressed(&self) -> usize { + self.observed.saturating_sub(self.series.len()) + } + /// `seriesLimitDisclosure`. + fn disclosure(&self, dimensions: &[&str], selection: &str) -> Value { + json!({ + "label_dimensions": dimensions, "limit": SERIES_LIMIT, + "observed_series_count": self.observed, "observed_series_count_precision": self.precision(), + "reported_series_count": self.series.len(), "suppressed_series_count": self.suppressed(), + "suppressed_started_total": self.suppressed_started, "suppressed_counts_precision": self.precision(), + "truncated": self.observed > self.series.len(), "selection": selection, + }) + } +} + +/// `BoundedMetricPolicy::labelSet` for a family's bounded dimensions. +fn label_set(dimensions: &[&str], selection: &str) -> Value { + let mut set = Map::new(); + set.insert("namespace".into(), json!("request_scope_not_label")); + for dimension in dimensions { + set.insert( + (*dimension).to_owned(), + json!({"cardinality_class": "bounded_series", "limit": SERIES_LIMIT, "selection": selection}), + ); + } + Value::Object(set) +} + +fn empty_task_queue(queue: &str) -> Map { + let mut entry = Map::new(); + entry.insert("task_queue".into(), json!(queue)); + for prefix in ["workflow", "activity"] { + for field in [ + "ready_tasks", + "ready_due_tasks", + "delayed_tasks", + "leased_tasks", + "expired_leases", + "tasks_added_last_minute", + "tasks_dispatched_last_minute", + ] { + entry.insert(format!("{prefix}_{field}"), json!(0)); + } + } + for field in [ + "active_pollers", + "workflow_slot_capacity", + "activity_slot_capacity", + "available_workflow_slots", + "available_activity_slots", + ] { + entry.insert(field.into(), json!(0)); + } + entry +} + +fn add(entry: &mut Map, field: &str, amount: i64) { + let current = entry.get(field).and_then(Value::as_i64).unwrap_or_default(); + entry.insert(field.to_owned(), json!(current + amount)); +} + +store_methods! { + pub(crate) async fn prometheus_metrics(&self) -> Result { + let now = now(); + let workflows = self.workflow_series().await?; + let activities = self.activity_series().await?; + let queues = self.task_queue_series(now).await?; + let families = [&workflows, &activities, &queues]; + let precision = if families.iter().any(|family| family.lower_bound) { "lower_bound" } else { "exact" }; + let series_count: usize = families.iter().map(|family| family.series.len()).sum(); + let observed: usize = families.iter().map(|family| family.observed).sum(); + let suppressed: usize = families.iter().map(|family| family.suppressed()).sum(); + let workflow_dimensions = ["task_queue", "workflow_type"]; + let activity_dimensions = ["task_queue", "workflow_type", "activity_type"]; + let series_limits = json!({ + "workflows": workflows.disclosure(&workflow_dimensions, WORKFLOW_SELECTION), + "activities": activities.disclosure(&activity_dimensions, ACTIVITY_SELECTION), + "task_queues": queues.disclosure(&["task_queue"], QUEUE_SELECTION), + }); + let workflow_labels = label_set(&workflow_dimensions, WORKFLOW_SELECTION); + let activity_labels = label_set(&activity_dimensions, ACTIVITY_SELECTION); + Ok(json!({ + "schema": "durable-workflow.prometheus-metrics.v1", + "generated_at": now.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "namespace": self.namespace.as_ref(), + "latency_buckets_seconds": BUCKETS.iter().map(|(_, bound)| php_float(*bound)).collect::>(), + "series": {"workflows": workflows.series, "activities": activities.series, "task_queues": queues.series}, + "series_count": series_count, + "observed_series_count": observed, + "observed_series_count_precision": precision, + "suppressed_series_count": suppressed, + "suppressed_series_count_precision": precision, + "cardinality": { + "series_limits": series_limits, + "metric_label_sets": { + "dw_workflow_runs_total": workflow_labels, + "dw_workflow_run_latency_seconds": workflow_labels, + "dw_activity_executions_total": activity_labels, + "dw_activity_execution_latency_seconds": activity_labels, + "dw_task_queue_runtime_state": label_set(&["task_queue"], QUEUE_SELECTION), + }, + }, + })) + } + + /// The counters and latency histogram shared by both runtime families. + fn runtime_series(row: &DB::Row, labels: Map, terminated: bool) -> Result { + let mut series = labels; + for field in ["started_total", "completed_total", "failed_total", "cancelled_total"] { + series.insert(field.into(), json!(DB::number(row, field)?)); + } + if terminated { + series.insert("terminated_total".into(), json!(DB::number(row, "terminated_total")?)); + } + series.insert("running".into(), json!(DB::number(row, "running")?)); + let mut buckets = Map::new(); + for (index, (bound, _)) in BUCKETS.iter().enumerate() { + buckets.insert((*bound).to_owned(), json!(DB::number(row, &format!("bucket_{index}"))?)); + } + let sum = DB::optional_float(row, "latency_sum")?.unwrap_or_default(); + series.insert("latency_seconds".into(), json!({ + "count": DB::number(row, "latency_count")?, "sum": php_float(round6(sum)), "buckets": buckets, + })); + Ok(Value::Object(series)) + } + + /// `workflowSeries`: the first series by task queue and workflow type. + async fn workflow_series(&self) -> Result { + let candidates: Vec<(String, String)> = Self::query( + "SELECT DISTINCT COALESCE(queue,'default') AS task_queue,workflow_type FROM workflow_run_summaries WHERE namespace=$1 ORDER BY task_queue,workflow_type LIMIT 101", + ) + .bind(self.namespace.as_ref()) + .fetch_all(&self.pool) + .await? + .iter() + .map(|row| Ok((DB::string(row, "task_queue")?, DB::optional_string(row, "workflow_type")?.unwrap_or_default()))) + .collect::>()?; + let Some((last_queue, last_type)) = candidates.last().cloned() else { + return Ok(Snapshot { series: Vec::new(), observed: 0, suppressed_started: 0, lower_bound: false }); + }; + let sql = match DB::NAME { + "PostgreSQL" => workflow_rows!("DOUBLE PRECISION"), + "MySQL" => workflow_rows!("DOUBLE"), + _ => workflow_rows!("REAL"), + }; + let rows = Self::query(sql) + .bind(self.namespace.as_ref()) + .bind(last_queue.clone()) + .bind(last_queue) + .bind(last_type) + .fetch_all(&self.pool) + .await?; + let mut keyed = HashMap::new(); + for row in &rows { + let key = (DB::string(row, "task_queue")?, DB::optional_string(row, "workflow_type")?.unwrap_or_default()); + keyed.insert(key, row); + } + let mut series = Vec::new(); + for key in candidates.iter().take(SERIES_LIMIT) { + if let Some(row) = keyed.get(key) { + let mut labels = Map::new(); + labels.insert("task_queue".into(), json!(key.0)); + labels.insert("workflow_type".into(), json!(key.1)); + series.push(Self::runtime_series(row, labels, true)?); + } + } + let mut suppressed_started = 0; + for key in candidates.iter().skip(SERIES_LIMIT) { + if let Some(row) = keyed.get(key) { + suppressed_started += DB::number(row, "started_total")?; + } + } + Ok(Snapshot { series, observed: candidates.len(), suppressed_started, lower_bound: candidates.len() > SERIES_LIMIT }) + } + + /// `activitySeries`: the first series by task queue, workflow type and + /// activity type. + async fn activity_series(&self) -> Result { + let candidates: Vec<(String, String, String)> = Self::query( + "SELECT DISTINCT COALESCE(activity_executions.queue,workflow_runs.queue,'default') AS task_queue,workflow_runs.workflow_type AS workflow_type,activity_executions.activity_type AS activity_type FROM activity_executions JOIN workflow_runs ON workflow_runs.id=activity_executions.workflow_run_id WHERE workflow_runs.namespace=$1 ORDER BY task_queue,workflow_type,activity_type LIMIT 101", + ) + .bind(self.namespace.as_ref()) + .fetch_all(&self.pool) + .await? + .iter() + .map(|row| { + Ok(( + DB::string(row, "task_queue")?, + DB::optional_string(row, "workflow_type")?.unwrap_or_default(), + DB::optional_string(row, "activity_type")?.unwrap_or_default(), + )) + }) + .collect::>()?; + let Some((last_queue, last_type, last_activity)) = candidates.last().cloned() else { + return Ok(Snapshot { series: Vec::new(), observed: 0, suppressed_started: 0, lower_bound: false }); + }; + let sql = match DB::NAME { + "PostgreSQL" => activity_rows!( + "DOUBLE PRECISION", + "EXTRACT(EPOCH FROM (activity_executions.closed_at-activity_executions.started_at))" + ), + "MySQL" => activity_rows!( + "DOUBLE", + "TIMESTAMPDIFF(MICROSECOND,activity_executions.started_at,activity_executions.closed_at)/1000000.0" + ), + _ => activity_rows!( + "REAL", + "(julianday(activity_executions.closed_at)-julianday(activity_executions.started_at))*86400.0" + ), + }; + let rows = Self::query(sql) + .bind(self.namespace.as_ref()) + .bind(last_queue.clone()) + .bind(last_queue) + .bind(last_type.clone()) + .bind(last_type) + .bind(last_activity) + .fetch_all(&self.pool) + .await?; + let mut keyed = HashMap::new(); + for row in &rows { + let key = ( + DB::string(row, "task_queue")?, + DB::optional_string(row, "workflow_type")?.unwrap_or_default(), + DB::optional_string(row, "activity_type")?.unwrap_or_default(), + ); + keyed.insert(key, row); + } + let mut series = Vec::new(); + for key in candidates.iter().take(SERIES_LIMIT) { + if let Some(row) = keyed.get(key) { + let mut labels = Map::new(); + labels.insert("task_queue".into(), json!(key.0)); + labels.insert("workflow_type".into(), json!(key.1)); + labels.insert("activity_type".into(), json!(key.2)); + series.push(Self::runtime_series(row, labels, false)?); + } + } + let mut suppressed_started = 0; + for key in candidates.iter().skip(SERIES_LIMIT) { + if let Some(row) = keyed.get(key) { + suppressed_started += DB::number(row, "started_total")?; + } + } + Ok(Snapshot { series, observed: candidates.len(), suppressed_started, lower_bound: candidates.len() > SERIES_LIMIT }) + } + + /// `taskQueueSeries`: open and recent task counts and fresh worker + /// capacity for the first task queues by name. + async fn task_queue_series(&self, now: DateTime) -> Result { + // Laravel binds these times in whole seconds, as operator metrics do. + let bound = whole_second(now); + let minute_ago = whole_second(now - Duration::minutes(1)); + let stale_cutoff = whole_second(now - Duration::seconds(self.worker_stale_after_seconds)); + let mut queues: Vec = Vec::new(); + for row in Self::query(concat!("SELECT DISTINCT COALESCE(queue,'default') AS task_queue", visible_tasks!("$1", "$2", "$3"), " ORDER BY task_queue LIMIT 101")) + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(minute_ago)) + .bind(DB::bind_time(minute_ago)) + .fetch_all(&self.pool) + .await? + { + queues.push(DB::string(&row, "task_queue")?); + } + for row in Self::query("SELECT DISTINCT COALESCE(task_queue,'default') AS task_queue FROM workflow_worker_registrations WHERE namespace=$1 AND status='active' AND last_heartbeat_at IS NOT NULL AND last_heartbeat_at>=$2 ORDER BY task_queue LIMIT 101") + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(stale_cutoff)) + .fetch_all(&self.pool) + .await? + { + queues.push(DB::string(&row, "task_queue")?); + } + queues.sort(); + queues.dedup(); + queues.truncate(SERIES_LIMIT + 1); + let observed = queues.len(); + queues.truncate(SERIES_LIMIT); + let mut entries: std::collections::BTreeMap> = std::collections::BTreeMap::new(); + let rows = Self::query(concat!( + "SELECT COALESCE(queue,'default') AS task_queue,task_type,status,COUNT(*) AS n,", + "COUNT(CASE WHEN status='ready' AND available_at>$1 THEN 1 END) AS delayed_count,", + "COUNT(CASE WHEN status='leased' AND lease_expires_at IS NOT NULL AND lease_expires_at<=$2 THEN 1 END) AS expired_lease_count,", + "COUNT(CASE WHEN created_at>=$3 THEN 1 END) AS added_last_minute,", + "COUNT(CASE WHEN last_dispatched_at IS NOT NULL AND last_dispatched_at>=$4 THEN 1 END) AS dispatched_last_minute", + visible_tasks!("$5", "$6", "$7"), + " GROUP BY COALESCE(queue,'default'),task_type,status" + )) + .bind(DB::bind_time(bound)) + .bind(DB::bind_time(bound)) + .bind(DB::bind_time(minute_ago)) + .bind(DB::bind_time(minute_ago)) + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(minute_ago)) + .bind(DB::bind_time(minute_ago)) + .fetch_all(&self.pool) + .await?; + for row in &rows { + let queue = DB::string(row, "task_queue")?; + if !queues.contains(&queue) { + continue; + } + let entry = entries.entry(queue.clone()).or_insert_with(|| empty_task_queue(&queue)); + let prefix = if DB::string(row, "task_type")? == "activity" { "activity" } else { "workflow" }; + let status = DB::string(row, "status")?; + let count = DB::number(row, "n")?; + if status == "ready" { + let delayed = DB::number(row, "delayed_count")?; + add(entry, &format!("{prefix}_ready_tasks"), count); + add(entry, &format!("{prefix}_delayed_tasks"), delayed); + add(entry, &format!("{prefix}_ready_due_tasks"), (count - delayed).max(0)); + } + if status == "leased" { + add(entry, &format!("{prefix}_leased_tasks"), count); + add(entry, &format!("{prefix}_expired_leases"), DB::number(row, "expired_lease_count")?); + } + add(entry, &format!("{prefix}_tasks_added_last_minute"), DB::number(row, "added_last_minute")?); + add(entry, &format!("{prefix}_tasks_dispatched_last_minute"), DB::number(row, "dispatched_last_minute")?); + } + // Capacity is summed here rather than in SQL, where MySQL's SUM is a + // decimal. + let workers = Self::query("SELECT COALESCE(task_queue,'default') AS task_queue,max_concurrent_workflow_tasks,max_concurrent_activity_tasks,available_workflow_slots,available_activity_slots FROM workflow_worker_registrations WHERE namespace=$1 AND status='active' AND last_heartbeat_at IS NOT NULL AND last_heartbeat_at>=$2") + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(stale_cutoff)) + .fetch_all(&self.pool) + .await?; + let mut capacity: HashMap = HashMap::new(); + for row in &workers { + let queue = DB::string(row, "task_queue")?; + if !queues.contains(&queue) { + continue; + } + let workflow = DB::optional_number(row, "max_concurrent_workflow_tasks")?; + let activity = DB::optional_number(row, "max_concurrent_activity_tasks")?; + let totals = capacity.entry(queue).or_default(); + totals[0] += 1; + totals[1] += workflow.unwrap_or_default(); + totals[2] += activity.unwrap_or_default(); + totals[3] += DB::optional_number(row, "available_workflow_slots")?.or(workflow).unwrap_or_default(); + totals[4] += DB::optional_number(row, "available_activity_slots")?.or(activity).unwrap_or_default(); + } + for (queue, totals) in capacity { + let entry = entries.entry(queue.clone()).or_insert_with(|| empty_task_queue(&queue)); + for (field, value) in [ + "active_pollers", + "workflow_slot_capacity", + "activity_slot_capacity", + "available_workflow_slots", + "available_activity_slots", + ] + .iter() + .zip(totals) + { + entry.insert((*field).to_owned(), json!(value)); + } + } + Ok(Snapshot { + series: entries.into_values().map(Value::Object).collect(), + observed, + suppressed_started: 0, + lower_bound: observed > SERIES_LIMIT, + }) + } +} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index eee15364b..341332c4a 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -552,7 +552,7 @@ where pub(super) async fn poll_query(&self, body: Value, config: &QueryTaskConfig) -> Result { let owner = text(&body, "worker_id")?; - text(&body, "task_queue")?; + self.refuse_draining_worker(owner, text(&body, "task_queue")?).await?; // `queryPollTimeoutSeconds`: the requested wait, or the worker poll // wait (DW_WORKER_POLL_TIMEOUT), capped by DW_QUERY_TASK_POLL_TIMEOUT. let timeout = match body.get("timeout_seconds") { diff --git a/rust/src/runtime/query_poll_requests.rs b/rust/src/runtime/query_poll_requests.rs index fe760d6a3..c07408f9a 100644 --- a/rust/src/runtime/query_poll_requests.rs +++ b/rust/src/runtime/query_poll_requests.rs @@ -2,7 +2,10 @@ //! (`QueryTaskPollRequestStore`). A retry of a poll request gets the answer the //! original got, a duplicate still in flight waits for that answer, and a newer //! request from the same worker ends an older one that is still waiting. -use super::*; +use super::{ + Backend, ColumnIndex, Decode, Duration, Encode, Executor, IntoArguments, QueryTaskConfig, Result, Store, + Transaction, Type, Value, authority, json, key, now, poll_result, text, +}; use sha2::{Digest, Sha256}; // Distinct from the query task prefix, so query scans never see these rows. diff --git a/rust/src/runtime/redrive.rs b/rust/src/runtime/redrive.rs new file mode 100644 index 000000000..004836fcd --- /dev/null +++ b/rust/src/runtime/redrive.rs @@ -0,0 +1,320 @@ +//! Redriving a failed run, as published PHP's `DefaultWorkflowControlPlane:: +//! redrive` and `FailedRunRedrivePlan` do: a run that failed right after an +//! activity failed continues in a new run whose history reuses the completed +//! activities before that step, so its workflow resumes at the failed step. +use super::laravel_rules::{self, Rule}; +use super::{Result, RuntimeError, backend::Backend, refuse, store::{Store, id, now}}; +use axum::http::StatusCode; +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::collections::BTreeMap; + +/// `WorkflowController::redriveRun`'s rules. +const RULES: &laravel_rules::Rules = &[("request_id", &[Rule::Nullable, Rule::String, Rule::Max(191)])]; + +pub(super) fn request_id(body: &Value) -> Result> { + let body = laravel_rules::normalize(body); + laravel_rules::validate(&body, RULES)?; + Ok(body["request_id"].as_str().map(str::to_owned)) +} + +/// Carbon's `toIso8601String`: whole seconds with an explicit offset. +fn iso(at: DateTime) -> String { + at.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() +} + +/// One step the new run reuses. +struct Reused { + payload: Value, + recorded_at: DateTime, +} + +/// `FailedRunRedrivePlan::forRun` over the source run's history. +fn plan(status: &str, events: &[(String, Value, DateTime)]) -> std::result::Result<(i64, Vec), &'static str> { + if status != "failed" { + return Err("run_not_failed"); + } + let mut started = false; + let mut failure: Option<&Value> = None; + let mut steps: BTreeMap)>> = BTreeMap::new(); + let mut last_activity: Option<(&str, &Value)> = None; + for (kind, payload, recorded_at) in events { + if failure.is_some() { + return Err("events_after_failure"); + } + match kind.as_str() { + "WorkflowStarted" if started => return Err("duplicate_start"), + "WorkflowStarted" => started = true, + "WorkflowFailed" if !started => return Err("missing_start"), + "WorkflowFailed" => failure = Some(payload), + "StartAccepted" if started => return Err("unexpected_start"), + "StartAccepted" => {} + "ActivityScheduled" | "ActivityStarted" | "ActivityHeartbeatRecorded" | "ActivityCompleted" | "ActivityFailed" => { + if payload["execution_mode"] == "local" || payload["local_activity"] == true { + return Err("unsupported_local_activity"); + } + let Some(sequence) = payload["sequence"].as_i64().filter(|sequence| started && *sequence >= 1) else { + return Err("invalid_activity_sequence"); + }; + steps.entry(sequence).or_default().push((kind, payload, *recorded_at)); + last_activity = Some((kind, payload)); + } + _ => return Err("unsupported_history"), + } + } + let Some(failure) = failure.filter(|_| started) else { + return Err("missing_terminal_history"); + }; + let Some(boundary) = failure["failed_step_sequence"].as_i64().filter(|boundary| *boundary >= 1) else { + return Err("unrecorded_failure_boundary"); + }; + if failure["failed_step_kind"] != "activity" { + return Err("unrecorded_failure_boundary"); + } + if steps.len() as i64 != boundary { + return Err("activity_sequence_gap"); + } + let mut reused = Vec::new(); + for sequence in 1..=boundary { + let Some(events) = steps.get(&sequence) else { + return Err("activity_sequence_gap"); + }; + let mut activity_type: Option<&str> = None; + let mut completion = None; + let mut failures = 0; + for (kind, payload, recorded_at) in events { + let Some(recorded) = payload["activity_type"].as_str().filter(|value| !value.is_empty()) else { + return Err("missing_activity_type"); + }; + if activity_type.is_some_and(|known| known != recorded) { + return Err("activity_type_mismatch"); + } + activity_type = Some(recorded); + if *kind == "ActivityCompleted" { + if completion.is_some() { + return Err("duplicate_activity_completion"); + } + completion = Some((*payload, *recorded_at)); + } + if *kind == "ActivityFailed" { + failures += 1; + } + } + if sequence == boundary { + if completion.is_some() || failures != 1 { + return Err("invalid_failed_step"); + } + continue; + } + let Some((payload, recorded_at)) = completion.filter(|_| failures == 0) else { + return Err("incomplete_activity_prefix"); + }; + if payload.get("result").is_none() { + return Err("missing_activity_result"); + } + if payload["result"].get("external_payload").is_some() || payload["result"].get("external_storage").is_some() { + return Err("external_activity_result_not_supported"); + } + if !payload["result"]["blob"].is_string() { + return Err("missing_activity_result"); + } + reused.push(Reused { payload: payload.clone(), recorded_at }); + } + match last_activity { + Some(("ActivityFailed", payload)) if payload["sequence"].as_i64() == Some(boundary) => Ok((boundary, reused)), + _ => Err("unrecorded_failure_boundary"), + } +} + +/// PHP's answer for a redrive that was not accepted. +fn rejected(workflow_id: &str, source: &str, reason: &str, status: StatusCode) -> (StatusCode, Value) { + (status, json!({"workflow_run_id":null,"continued_from_run_id":source,"resume_step_sequence":null,"task_id":null, + "reason":reason,"run_id":source,"command_status":"rejected","outcome":"rejected","accepted":false,"workflow_id":workflow_id})) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + pub(crate) async fn redrive_run( + &self, + workflow_id: &str, + source_id: &str, + request_id: Option, + context: Value, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let instance = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? + .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "run_not_found"))?; + let source = Self::query("SELECT * FROM workflow_runs WHERE id=$1 AND workflow_instance_id=$2") + .bind(source_id).bind(workflow_id).fetch_optional(&mut *tx).await? + .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "run_not_found"))?; + // A source already redriven answers with its continuation. + if let Some(link) = Self::query("SELECT child_workflow_run_id,sequence FROM workflow_links WHERE parent_workflow_run_id=$1 AND link_type='redrive'") + .bind(source_id).fetch_optional(&mut *tx).await? + { + let continued = DB::string(&link, "child_workflow_run_id")?; + return Ok((StatusCode::OK, json!({"workflow_run_id":continued,"continued_from_run_id":source_id, + "resume_step_sequence":DB::optional_number(&link, "sequence")?,"task_id":null,"reason":null, + "run_id":continued,"command_status":"accepted","outcome":"redriven","workflow_id":workflow_id}))); + } + let conflict = |reason: &str| Ok(rejected(workflow_id, source_id, reason, StatusCode::CONFLICT)); + if DB::optional_string(&instance, "current_run_id")?.as_deref() != Some(source_id) { + return conflict("source_run_not_current"); + } + let mut events = Vec::new(); + for row in Self::query("SELECT event_type,payload,recorded_at FROM workflow_history_events WHERE workflow_run_id=$1 ORDER BY sequence") + .bind(source_id).fetch_all(&mut *tx).await? + { + events.push((DB::string(&row, "event_type")?, DB::document_row(&row, "payload")?, DB::instant(&row, "recorded_at")?)); + } + let (resume, reused) = match plan(&DB::string(&source, "status")?, &events) { + Ok(plan) => plan, + Err(reason) => return conflict(reason), + }; + let memos = Self::scalar("SELECT COUNT(*) FROM workflow_memos WHERE workflow_run_id=$1").bind(source_id).fetch_one(&mut *tx).await?; + let attributes = Self::scalar("SELECT COUNT(*) FROM workflow_search_attributes WHERE workflow_run_id=$1").bind(source_id).fetch_one(&mut *tx).await?; + if memos + attributes > 0 { + return conflict("unsupported_visibility_metadata"); + } + let child = Self::scalar("SELECT COUNT(*) FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow'") + .bind(source_id).fetch_one(&mut *tx).await?; + let arguments = DB::optional_string(&source, "arguments")?; + if child > 0 || arguments.as_deref().is_some_and(|blob| blob.starts_with(super::external_resolution::STORED_PREFIX)) { + return conflict("unsupported_source_link_or_payload"); + } + let at = now(); + let execution_deadline = DB::optional_instant(&source, "execution_deadline_at")?; + if execution_deadline.is_some_and(|deadline| at >= deadline) { + return conflict("execution_deadline_elapsed"); + } + let run_timeout = DB::optional_number(&source, "run_timeout_seconds")?.filter(|seconds| *seconds > 0); + let run_deadline = run_timeout.map(|seconds| at + chrono::Duration::seconds(seconds)); + let mut started = events.iter().find(|(kind, _, _)| kind == "WorkflowStarted").map(|(_, payload, _)| payload.clone()).unwrap_or(Value::Null); + if let Some(fields) = started.as_object_mut() { + fields.remove("command"); + fields.remove("task"); + } + let filled = |value: &Value| !(value.is_null() || value == &json!([]) || value == &json!({}) || value == "" || value == &json!(false)); + if filled(&started["memo"]) || filled(&started["search_attributes"]) || filled(&started["parent_workflow_run_id"]) { + return conflict("unsupported_source_metadata"); + } + // An external workflow's definition must still match its worker cohort. + let workflow_type = DB::string(&source, "workflow_type")?; + let queue = DB::string(&source, "queue")?; + // `ServerWorkflowControlPlane::redrive`: the source run's own cohort, + // its build or the unversioned workers. + let workers = Self::start_workers(&mut tx, &self.namespace, &queue, self.worker_stale_after_seconds).await?; + let source_cohort = super::start_definition::Cohort::of_run(DB::optional_string(&source, "compatibility")?.as_deref()); + let cohort = super::start_definition::fingerprint_for(&workers, &workflow_type, &source_cohort); + let recorded = started["workflow_definition_fingerprint"].as_str().filter(|value| !value.is_empty()); + if DB::string(&source, "workflow_class")? != workflow_type || recorded.is_none() || recorded != cohort.as_deref() { + return conflict("workflow_definition_unavailable_or_changed"); + } + if let Some(request_id) = &request_id { + let used = Self::scalar("SELECT COUNT(*) FROM workflow_commands WHERE workflow_instance_id=$1 AND command_type='redrive' AND request_id=$2") + .bind(workflow_id).bind(request_id.as_str()).fetch_one(&mut *tx).await?; + if used > 0 { + return conflict("request_id_already_used"); + } + } + let run_id = id(); + let run_number = DB::number(&source, "run_number")? + 1; + Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,run_timeout_seconds,execution_deadline_at,run_deadline_at,started_at,created_at,updated_at,namespace,last_progress_at,business_key,visibility_labels,compatibility,connection,priority,fairness_key,fairness_weight,last_command_sequence) SELECT $1,workflow_instance_id,workflow_type,workflow_class,$2,'pending',payload_codec,arguments,queue,run_timeout_seconds,execution_deadline_at,$3,$4,$5,$6,namespace,$7,business_key,visibility_labels,compatibility,connection,priority,fairness_key,fairness_weight,1 FROM workflow_runs WHERE id=$8") + .bind(&run_id).bind(run_number).bind(DB::optional_time(run_deadline)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(source_id).execute(&mut *tx).await?; + let command_id = id(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,requested_workflow_run_id,resolved_workflow_run_id,command_type,target_scope,source,status,outcome,workflow_type,payload_codec,command_sequence,accepted_at,applied_at,context,request_id) VALUES ($1,$2,$3,$4,$5,'redrive','run','control_plane','accepted','redriven',$6,'avro',1,$7,$8,$9,$10)") + .bind(&command_id).bind(workflow_id).bind(&run_id).bind(source_id).bind(&run_id).bind(&workflow_type) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::document(&context)).bind(request_id.as_deref()) + .execute(&mut *tx).await?; + let link_id = id(); + Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,created_at,updated_at) VALUES ($1,'redrive',$2,$3,$4,$5,$6,TRUE,$7,$8)") + .bind(&link_id).bind(resume).bind(workflow_id).bind(source_id).bind(workflow_id).bind(&run_id) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut *tx).await?; + Self::query("UPDATE workflow_instances SET current_run_id=$1,run_count=$2,updated_at=$3 WHERE id=$4") + .bind(&run_id).bind(run_number).bind(DB::bind_time(at)).bind(workflow_id).execute(&mut *tx).await?; + Self::append(&mut tx, &run_id, "StartAccepted", json!({"workflow_command_id":command_id,"workflow_instance_id":workflow_id, + "workflow_run_id":run_id,"workflow_class":workflow_type,"workflow_type":workflow_type,"outcome":"redriven"}), None, Some(&command_id)).await?; + let source_started = DB::optional_instant(&source, "started_at")?; + let replayed = started["replayed_started_at"].as_str().map(str::to_owned).or_else(|| source_started.map(iso)); + if !started.is_object() { + started = json!({}); + } + for (key, value) in [("workflow_instance_id", json!(workflow_id)), ("workflow_run_id", json!(run_id)), + ("workflow_command_id", json!(command_id)), ("continued_from_run_id", json!(source_id)), + ("resume_step_sequence", json!(resume)), ("recovery_kind", json!("redrive")), + ("execution_deadline_at", json!(execution_deadline.map(iso))), ("run_deadline_at", json!(run_deadline.map(iso))), + ("replayed_started_at", json!(replayed)), ("workflow_link_id", json!(link_id))] { + started[key] = value; + } + Self::append(&mut tx, &run_id, "WorkflowStarted", started, None, Some(&command_id)).await?; + for step in reused { + let payload = &step.payload; + Self::append(&mut tx, &run_id, "ActivityCompleted", json!({"activity_execution_id":id(), + "activity_class":payload["activity_class"],"activity_type":payload["activity_type"],"sequence":payload["sequence"], + "result":payload["result"],"payload_codec":payload["payload_codec"].as_str().unwrap_or("avro"), + "reused_from_run_id":source_id,"reused_activity_execution_id":payload["activity_execution_id"], + "reused_recorded_at":payload["reused_recorded_at"].as_str().map(str::to_owned).unwrap_or_else(|| iso(step.recorded_at))}), None, None).await?; + } + let task_id = Self::create_task(&mut tx, &run_id, &queue, "workflow", json!({})).await?; + tx.summary_changed(source_id); + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::ACCEPTED, json!({"workflow_run_id":run_id,"continued_from_run_id":source_id,"resume_step_sequence":resume, + "task_id":task_id,"reason":null,"run_id":run_id,"command_status":"accepted","outcome":"redriven","workflow_id":workflow_id}))) + } +} + +/// A rejected or unknown redrive for a run outside this namespace. +pub(super) fn run_not_found(workflow_id: &str, run_id: &str) -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Workflow run not found.","reason":"run_not_found","workflow_id":workflow_id,"run_id":run_id}), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(kind: &str, payload: Value) -> (String, Value, DateTime) { + (kind.into(), payload, Utc::now()) + } + + #[test] + fn plan_follows_php_s_failed_run_rules() { + let done = json!({"sequence":1,"activity_type":"a","result":{"codec":"avro","blob":"x"},"payload_codec":"avro"}); + let failed = json!({"sequence":2,"activity_type":"b"}); + let history = vec![event("StartAccepted", json!({})), event("WorkflowStarted", json!({})), + event("ActivityScheduled", json!({"sequence":1,"activity_type":"a"})), event("ActivityCompleted", done.clone()), + event("ActivityScheduled", json!({"sequence":2,"activity_type":"b"})), event("ActivityFailed", failed.clone()), + event("WorkflowFailed", json!({"failed_step_sequence":2,"failed_step_kind":"activity"}))]; + let (resume, reused) = plan("failed", &history).unwrap(); + assert_eq!((resume, reused.len()), (2, 1)); + assert_eq!(plan("completed", &history).err(), Some("run_not_failed")); + let mut unbounded = history.clone(); + unbounded[6] = event("WorkflowFailed", json!({})); + assert_eq!(plan("failed", &unbounded).err(), Some("unrecorded_failure_boundary")); + let mut timer = history.clone(); + timer.insert(2, event("TimerScheduled", json!({"sequence":1}))); + assert_eq!(plan("failed", &timer).err(), Some("unsupported_history")); + let mut external = history.clone(); + external[3] = event("ActivityCompleted", json!({"sequence":1,"activity_type":"a","result":{"codec":"avro","external_payload":{}}})); + assert_eq!(plan("failed", &external).err(), Some("external_activity_result_not_supported")); + let mut gap = history.clone(); + gap[6] = event("WorkflowFailed", json!({"failed_step_sequence":3,"failed_step_kind":"activity"})); + assert_eq!(plan("failed", &gap).err(), Some("activity_sequence_gap")); + } +} diff --git a/rust/src/runtime/repair_candidates.rs b/rust/src/runtime/repair_candidates.rs index 323951192..5c8e02fe5 100644 --- a/rust/src/runtime/repair_candidates.rs +++ b/rust/src/runtime/repair_candidates.rs @@ -7,27 +7,6 @@ use serde_json::{Value, json}; /// `TaskRepairPolicy::SCAN_LIMIT`. pub(super) const SCAN_LIMIT: usize = 25; -/// `existingTaskQuery`: ready tasks that are due and whose dispatch failed, -/// whose claim failed, or whose dispatch is overdue, and leased tasks whose -/// lease expired. `$1`–`$3` and `$7` are "now"; `$4`–`$6` are the redispatch -/// cutoff, both in whole seconds as Laravel binds them. -macro_rules! repairable_tasks { - () => { - "((status='ready' AND (available_at IS NULL OR available_at<=$1) AND (\ - (last_dispatch_attempt_at IS NOT NULL AND last_dispatch_error IS NOT NULL AND last_dispatch_error<>'' \ - AND (last_dispatched_at IS NULL OR last_dispatch_attempt_at>last_dispatched_at) \ - AND (repair_available_at IS NULL OR repair_available_at<=$2)) \ - OR (last_claim_failed_at IS NOT NULL AND last_claim_error IS NOT NULL AND last_claim_error<>'' \ - AND ((repair_available_at IS NOT NULL AND repair_available_at<=$3) \ - OR (repair_available_at IS NULL AND last_claim_failed_at<=$4))) \ - OR ((last_dispatch_attempt_at IS NULL OR last_dispatch_error IS NULL OR last_dispatch_error='' \ - OR (last_dispatched_at IS NOT NULL AND last_dispatch_attempt_at<=last_dispatched_at)) \ - AND (last_claim_failed_at IS NULL OR last_claim_error IS NULL OR last_claim_error='') \ - AND (last_dispatched_at<=$5 OR (last_dispatched_at IS NULL AND created_at<=$6))))) \ - OR (status='leased' AND lease_expires_at IS NOT NULL AND lease_expires_at<=$7))" - }; -} - /// `missingTaskRunQuery`: open runs without a next task that need repair, or /// that wait for children which have all closed while no workflow task is open. macro_rules! missing_task_runs { @@ -45,18 +24,7 @@ macro_rules! missing_task_runs { }; } -pub(super) const TASK_SCOPES: &str = concat!( - "SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,MIN(created_at) AS oldest_candidate_at \ - FROM workflow_tasks WHERE ", - repairable_tasks!(), - " GROUP BY connection,queue,compatibility" -); -pub(super) const RUN_SCOPES: &str = concat!( - "SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,\ - MIN(COALESCE(wait_started_at,started_at,created_at)) AS oldest_candidate_at FROM workflow_run_summaries WHERE ", - missing_task_runs!(), - " GROUP BY connection,queue,compatibility" -); +pub(super) const MISSING_TASK_RUNS: &str = missing_task_runs!(); /// `oldestMissingRunCandidate`, in the backend's own null ordering, as PHP's. pub(super) const OLDEST_RUN: &str = concat!( "SELECT wait_started_at,started_at,created_at FROM workflow_run_summaries WHERE ", @@ -75,7 +43,7 @@ pub(super) struct ScopeRow { impl ScopeRow { /// `nullableString` of each scope column. - fn values(&self) -> [Option<&str>; 3] { + pub(super) fn values(&self) -> [Option<&str>; 3] { [&self.connection, &self.queue, &self.compatibility] .map(|value| value.as_deref().filter(|value| !value.is_empty())) } @@ -100,6 +68,19 @@ impl ScopeRow { /// `normalizedScopes` with each scope's selected ids counted: oldest scope /// first, each holding at most `limit` candidates. fn buckets(rows: &[ScopeRow], limit: usize) -> Vec<(String, usize)> { + normalized(rows) + .into_iter() + .map(|scope| { + let count: usize = rows.iter().filter(|row| row.within(scope)).map(|row| row.count).sum(); + (scope.key(), count.min(limit)) + }) + .filter(|(_, count)| *count > 0) + .collect() +} + +/// `normalizedScopes`: one row per scope key, keeping its oldest, ordered by +/// oldest candidate and then key. +pub(super) fn normalized(rows: &[ScopeRow]) -> Vec<&ScopeRow> { let mut chosen: Vec<&ScopeRow> = Vec::new(); for row in rows { match chosen.iter().position(|scope| scope.key() == row.key()) { @@ -116,13 +97,6 @@ fn buckets(rows: &[ScopeRow], limit: usize) -> Vec<(String, usize)> { order(left).cmp(&order(right)).then_with(|| left.key().cmp(&right.key())) }); chosen - .into_iter() - .map(|scope| { - let count: usize = rows.iter().filter(|row| row.within(scope)).map(|row| row.count).sum(); - (scope.key(), count.min(limit)) - }) - .filter(|(_, count)| *count > 0) - .collect() } /// `selectedScopeCounts`: one candidate from each scope in turn, up to diff --git a/rust/src/runtime/repair_pass.rs b/rust/src/runtime/repair_pass.rs new file mode 100644 index 000000000..27c6304af --- /dev/null +++ b/rust/src/runtime/repair_pass.rs @@ -0,0 +1,828 @@ +//! `POST /api/system/repair/pass`, as published Server 2.11 serves it +//! (`SystemController::repairPass` over `TaskWatchdog::runPass`): one +//! server-wide task repair pass, optionally scoped to runs, an instance, and +//! connections or queues. +//! +//! The pass recovers selected existing tasks (a ready task overdue for +//! dispatch is redispatched; an expired lease returns its task to ready), +//! creates a missing workflow task for a run that needs one, gives a run whose +//! deadline passed a task that closes it (or enforces an expired cancellation +//! cleanup deadline), and enforces expired activity deadlines. Selection is +//! PHP's poll-mode selection: up to 25 candidates of each kind, one from each +//! connection, queue and compatibility scope in turn, oldest first. +//! +//! An expired lease on a running activity is settled by its retry policy, +//! as `ActivityTimeoutEnforcer::enforceLeaseExpiry` settles it. +//! +//! A recovery native does not perform yet is reported as its candidate's +//! failure, so the pass answers 207 rather than claiming a repair: a missing +//! task for a run waiting on an activity, a timer, an update, a signal or a +//! child. +use super::Result; +use super::laravel_validation::{Rule, failure_messages, rule_errors}; +use super::operator_metrics::{REDISPATCH_AFTER_SECONDS, scope_rows, whole_second}; +use super::repair_candidates::{ScopeRow, normalized}; +use super::store::now; +use super::transaction::Transaction; +use chrono::{DateTime, Duration, Utc}; +use serde_json::{Map, Value, json}; +use sqlx::AssertSqlSafe; +use std::collections::VecDeque; +use std::time::Instant; + +/// `TaskRepairPolicy::scanLimit()`. +const SCAN_LIMIT: usize = 25; +/// `TaskRepairPolicy::loopThrottleSeconds()`. +const LOOP_THROTTLE_SECONDS: u64 = 5; + +/// The pass's request, after PHP's validation and trimming. +#[derive(Default)] +pub(crate) struct PassRequest { + pub(crate) run_ids: Vec, + pub(crate) instance_id: Option, + pub(crate) connection: Option, + pub(crate) queue: Option, + pub(crate) respect_throttle: bool, +} + +/// `repairPass`'s rules, in PHP's order. Laravel trims strings and reads +/// empty ones as null before validating. +pub(super) fn pass_request(body: &Value) -> Result { + let body = &super::start_request::normalized(body.clone()); + let mut errors = rule_errors(body, &[("run_ids", &[Rule::Nullable, Rule::Array, Rule::Max(100)])], |_, _| None); + let items: Vec<(String, Value)> = match &body["run_ids"] { + Value::Array(items) => items.iter().enumerate().map(|(index, item)| (index.to_string(), item.clone())).collect(), + Value::Object(fields) => fields.iter().map(|(key, item)| (key.clone(), item.clone())).collect(), + _ => Vec::new(), + }; + for (key, item) in &items { + let name = format!("run_ids.{key}"); + let mut value = Map::new(); + value.insert(name.clone(), item.clone()); + let rules: &[Rule] = &[Rule::String, Rule::Min(1), Rule::Max(128)]; + errors.extend(rule_errors(&Value::Object(value), &[(name.as_str(), rules)], |_, _| None)); + } + errors.extend(rule_errors( + body, + &[ + ("instance_id", &[Rule::Nullable, Rule::String, Rule::Min(1), Rule::Max(128)]), + ("connection", &[Rule::Nullable, Rule::String, Rule::Max(128)]), + ("queue", &[Rule::Nullable, Rule::String, Rule::Max(128)]), + ("respect_throttle", &[Rule::Nullable, Rule::Boolean]), + ], + |_, _| None, + )); + if let Some(error) = failure_messages(errors) { + return Err(error); + } + let text = |field: &str| body[field].as_str().map(str::to_owned); + Ok(PassRequest { + run_ids: items.iter().filter_map(|(_, id)| id.as_str().map(str::to_owned)).collect(), + instance_id: text("instance_id"), + connection: text("connection"), + queue: text("queue"), + // `$request->boolean()`: true, 1 and "1" are true. + respect_throttle: matches!(&body["respect_throttle"], Value::Bool(true)) + || body["respect_throttle"].as_i64() == Some(1) + || body["respect_throttle"].as_str() == Some("1"), + }) +} + +/// `emptyReport` with the controller's command-contract defaults. +fn empty_report(request: &PassRequest, throttled: bool) -> Map { + let value = json!({ + "connection": request.connection, "queue": request.queue, "run_ids": request.run_ids, + "instance_id": request.instance_id, "respect_throttle": request.respect_throttle, + "selected_command_contract_candidates": 0, "backfilled_command_contracts": 0, + "command_contract_backfill_unavailable": 0, "command_contract_failures": [], + "existing_task_failures": [], "missing_run_failures": [], "throttled": throttled, + "selected_existing_task_candidates": 0, "selected_missing_task_candidates": 0, "selected_total_candidates": 0, + "repaired_existing_tasks": 0, "repaired_missing_tasks": 0, "dispatched_tasks": 0, + "deadline_expired_candidates": 0, "deadline_expired_tasks_created": 0, "cancellation_deadlines_enforced": 0, + "deadline_expired_failures": [], "activity_timeout_candidates": 0, "activity_timeouts_enforced": 0, + "activity_timeout_failures": [], "service_call_timeout_candidates": 0, "service_call_timeouts_enforced": 0, + "service_call_timeout_failures": [], + }); + match value { + Value::Object(map) => map, + _ => unreachable!("the report is an object"), + } +} + +/// Whether the report holds a failure that PHP answers with 207. +pub(super) fn failed(report: &Value) -> bool { + ["existing_task_failures", "missing_run_failures", "command_contract_failures"] + .iter() + .any(|field| report[*field].as_array().is_some_and(|failures| !failures.is_empty())) +} + +fn bump(report: &mut Map, field: &str) { + let current = report[field].as_i64().unwrap_or_default(); + report.insert(field.to_owned(), json!(current + 1)); +} + +fn record_failure(report: &mut Map, field: &str, key: &str, id: &str, message: String) { + if let Some(Value::Array(failures)) = report.get_mut(field) { + failures.push(json!({key: id, "message": message})); + } +} + +/// `RequestedTaskScope::normalizeValues`: comma-separated, trimmed, unique. +fn scope_values(value: Option<&str>) -> Vec { + let mut values: Vec = Vec::new(); + for candidate in value.unwrap_or_default().split(',').map(str::trim).filter(|value| !value.is_empty()) { + if !values.iter().any(|value| value == candidate) { + values.push(candidate.to_owned()); + } + } + values +} + +enum Bind { + Text(String), + Time(DateTime), +} + +/// A statement built from fixed SQL text, with every value bound and numbered +/// for the backend. +pub(super) struct Sql { + text: String, + binds: Vec, + mysql: bool, +} + +impl Sql { + pub(super) fn new(text: &str, mysql: bool) -> Self { + Self { text: text.to_owned(), binds: Vec::new(), mysql } + } + + pub(super) fn push(&mut self, text: &str) -> &mut Self { + self.text.push_str(text); + self + } + + fn bind(&mut self, value: Bind) -> &mut Self { + self.binds.push(value); + if self.mysql { + self.text.push('?'); + } else { + self.text.push_str(&format!("${}", self.binds.len())); + } + self + } + + fn text(&mut self, value: &str) -> &mut Self { + self.bind(Bind::Text(value.to_owned())) + } + + fn time(&mut self, value: DateTime) -> &mut Self { + self.bind(Bind::Time(value)) + } + + fn list(&mut self, values: &[String]) -> &mut Self { + self.push("("); + for (index, value) in values.iter().enumerate() { + if index > 0 { + self.push(","); + } + self.text(value); + } + self.push(")") + } + + /// `RequestedTaskScope::applyValue` for one column: `default` matches a + /// null, empty or `default` value, and the default queue connection also + /// matches the default. + fn requested(&mut self, column: &str, value: Option<&str>) -> &mut Self { + let mut values = scope_values(value); + if values.is_empty() { + return self; + } + let default_connection = std::env::var("QUEUE_CONNECTION").unwrap_or_else(|_| "database".to_owned()); + if column == "connection" && values.contains(&default_connection) && !values.iter().any(|value| value == "default") { + values.push("default".to_owned()); + } + let exact: Vec = values.iter().filter(|value| *value != "default").cloned().collect(); + self.push(" AND ("); + if values.len() > exact.len() { + self.push(column).push(" IS NULL OR ").push(column).push("='' OR ").push(column).push("='default'"); + if !exact.is_empty() { + self.push(" OR ").push(column).push(" IN ").list(&exact); + } + } else { + self.push(column).push(" IN ").list(&exact); + } + self.push(")") + } + + /// `applyScope`: an absent scope value matches null, empty or the public + /// default; a present one matches exactly. + fn scope(&mut self, scope: &ScopeRow) -> &mut Self { + let [connection, queue, compatibility] = scope.values(); + for (column, value, default) in [("connection", connection, "default"), ("queue", queue, "default"), ("compatibility", compatibility, "any")] { + match value { + None => { + self.push(" AND (").push(column).push(" IS NULL OR ").push(column).push("='' OR ").push(column); + self.push("='").push(default).push("')"); + } + Some(value) => { + self.push(" AND ").push(column).push("=").text(value); + } + } + } + self + } + + /// `existingTaskQuery`'s repairable tasks: a due ready task whose dispatch + /// failed, whose claim failed, or whose dispatch and availability are + /// overdue, and a leased task whose lease expired. "Now" binds in whole + /// seconds, as Laravel binds a Carbon value; the redispatch cutoff binds + /// with microseconds, as PHP formats it. + fn repairable(&mut self, at: DateTime) -> &mut Self { + let bound = whole_second(at); + let cutoff = at - Duration::seconds(REDISPATCH_AFTER_SECONDS); + self.push("((status='ready' AND (available_at IS NULL OR available_at<=").time(bound).push(") AND (("); + self.push("last_dispatch_attempt_at IS NOT NULL AND last_dispatch_error IS NOT NULL AND last_dispatch_error<>'' "); + self.push("AND (last_dispatched_at IS NULL OR last_dispatch_attempt_at>last_dispatched_at) "); + self.push("AND (repair_available_at IS NULL OR repair_available_at<=").time(bound).push("))"); + self.push(" OR (last_claim_failed_at IS NOT NULL AND last_claim_error IS NOT NULL AND last_claim_error<>'' "); + self.push("AND ((repair_available_at IS NOT NULL AND repair_available_at<=").time(bound); + self.push(") OR (repair_available_at IS NULL AND last_claim_failed_at<=").time(cutoff).push(")))"); + self.push(" OR ((available_at IS NULL OR available_at<=").time(cutoff).push(") "); + self.push("AND (last_dispatch_attempt_at IS NULL OR last_dispatch_error IS NULL OR last_dispatch_error='' "); + self.push("OR (last_dispatched_at IS NOT NULL AND last_dispatch_attempt_at<=last_dispatched_at)) "); + self.push("AND (last_claim_failed_at IS NULL OR last_claim_error IS NULL OR last_claim_error='') "); + self.push("AND (last_dispatched_at<=").time(cutoff).push(" OR (last_dispatched_at IS NULL AND created_at<=").time(cutoff); + self.push("))))) OR (status='leased' AND lease_expires_at IS NOT NULL AND lease_expires_at<=").time(bound).push("))") + } + + /// `existingTaskQuery` with the pass's run, instance and scope filters. + fn repairable_tasks(&mut self, request: &PassRequest, at: DateTime) -> &mut Self { + self.repairable(at); + if !request.run_ids.is_empty() { + self.push(" AND workflow_run_id IN ").list(&request.run_ids); + } + if let Some(instance) = &request.instance_id { + self.push(" AND workflow_run_id IN (SELECT id FROM workflow_runs WHERE workflow_instance_id=").text(instance).push(")"); + } + self.requested("connection", request.connection.as_deref()).requested("queue", request.queue.as_deref()) + } + + /// `missingTaskRunQuery` with the pass's filters. + fn missing_task_runs(&mut self, request: &PassRequest) -> &mut Self { + self.push(super::repair_candidates::MISSING_TASK_RUNS); + if !request.run_ids.is_empty() { + self.push(" AND id IN ").list(&request.run_ids); + } + if let Some(instance) = &request.instance_id { + self.push(" AND workflow_instance_id=").text(instance); + } + self.requested("connection", request.connection.as_deref()).requested("queue", request.queue.as_deref()) + } +} + +/// `roundRobinCandidates`: one id from each scope in turn, up to the limit. +fn round_robin(mut buckets: Vec>, limit: usize) -> Vec { + let mut selected = Vec::new(); + while selected.len() < limit { + let mut progressed = false; + for ids in buckets.iter_mut() { + let Some(id) = ids.pop_front() else { + continue; + }; + selected.push(id); + progressed = true; + if selected.len() >= limit { + break; + } + } + if !progressed { + break; + } + } + selected +} + +/// What one candidate's recovery did. +enum Recovered { + /// A task is ready and was marked dispatched. + Task, + /// The candidate needed nothing, or a closed run's task was settled. + Nothing, + /// Native cannot perform this recovery yet; nothing was changed. + Unported(String), + /// A running activity lost its lease: its retry policy settles it. + ExpireLease, +} + +store_methods! { + async fn rows(&self, sql: Sql) -> Result> { + let mut query = sqlx::query::(AssertSqlSafe(sql.text)); + for bind in sql.binds { + query = match bind { + Bind::Text(text) => query.bind(text), + Bind::Time(time) => query.bind(DB::bind_time(time)), + }; + } + Ok(query.fetch_all(&self.pool).await?) + } + + async fn ids(&self, sql: Sql) -> Result> { + self.rows(sql).await?.iter().map(|row| DB::string(row, "id")).collect() + } + + /// `existingTaskScopeRows`. + pub(super) async fn repairable_task_scopes(&self, request: &PassRequest, at: DateTime) -> Result> { + let mut sql = Sql::new("SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,MIN(created_at) AS oldest_candidate_at FROM workflow_tasks WHERE ", DB::NAME == "MySQL"); + sql.repairable_tasks(request, at).push(" GROUP BY connection,queue,compatibility"); + scope_rows::(self.rows(sql).await?) + } + + /// `missingTaskScopeRows`. + pub(super) async fn missing_task_scopes(&self, request: &PassRequest) -> Result> { + let mut sql = Sql::new("SELECT connection,queue,compatibility,COUNT(*) AS candidate_count,MIN(COALESCE(wait_started_at,started_at,created_at)) AS oldest_candidate_at FROM workflow_run_summaries WHERE ", DB::NAME == "MySQL"); + sql.missing_task_runs(request).push(" GROUP BY connection,queue,compatibility"); + scope_rows::(self.rows(sql).await?) + } + + /// `TaskRepairCandidates::taskIds` in poll mode: each scope's oldest + /// repairable tasks, by availability, creation and id, taken in turn. + async fn repair_task_ids(&self, request: &PassRequest, at: DateTime) -> Result> { + let scopes = self.repairable_task_scopes(request, at).await?; + let mut buckets = Vec::new(); + for scope in normalized(&scopes) { + let mut sql = Sql::new("SELECT id FROM workflow_tasks WHERE ", DB::NAME == "MySQL"); + sql.repairable_tasks(request, at).scope(scope); + sql.push(&format!(" ORDER BY available_at,created_at,id LIMIT {SCAN_LIMIT}")); + let ids = self.ids(sql).await?; + if !ids.is_empty() { + buckets.push(ids.into()); + } + } + Ok(round_robin(buckets, SCAN_LIMIT)) + } + + /// `TaskRepairCandidates::runIds`: each scope's runs that need a task, by + /// wait, start and id, taken in turn. + async fn repair_run_ids(&self, request: &PassRequest) -> Result> { + let scopes = self.missing_task_scopes(request).await?; + let mut buckets = Vec::new(); + for scope in normalized(&scopes) { + let mut sql = Sql::new("SELECT id FROM workflow_run_summaries WHERE ", DB::NAME == "MySQL"); + sql.missing_task_runs(request).scope(scope); + sql.push(&format!(" ORDER BY wait_started_at,started_at,id LIMIT {SCAN_LIMIT}")); + let ids = self.ids(sql).await?; + if !ids.is_empty() { + buckets.push(ids.into()); + } + } + Ok(round_robin(buckets, SCAN_LIMIT)) + } + + /// `deadlineExpiredRunIds`: open runs past their execution or run + /// deadline without an open workflow task, and runs past their + /// cancellation cleanup deadline. + async fn deadline_expired_run_ids(&self, request: &PassRequest, at: DateTime) -> Result> { + let mut sql = Sql::new("SELECT id FROM workflow_runs WHERE status IN ('pending','running','waiting') AND ((execution_deadline_at IS NOT NULL AND execution_deadline_at<=", DB::NAME == "MySQL"); + sql.time(at).push(") OR (run_deadline_at IS NOT NULL AND run_deadline_at<=").time(at); + sql.push(") OR (cancellation_request_command_id IS NOT NULL AND cancellation_deadline_at IS NOT NULL AND cancellation_deadline_at<=").time(at); + sql.push(")) AND (NOT EXISTS (SELECT 1 FROM workflow_tasks t WHERE t.workflow_run_id=workflow_runs.id AND t.task_type='workflow' AND t.status IN ('ready','leased'))"); + sql.push(" OR (cancellation_request_command_id IS NOT NULL AND cancellation_deadline_at IS NOT NULL AND cancellation_deadline_at<=").time(at).push("))"); + if !request.run_ids.is_empty() { + sql.push(" AND id IN ").list(&request.run_ids); + } + if let Some(instance) = &request.instance_id { + sql.push(" AND workflow_instance_id=").text(instance); + } + sql.requested("connection", request.connection.as_deref()).requested("queue", request.queue.as_deref()); + sql.push(&format!(" ORDER BY id LIMIT {SCAN_LIMIT}")); + self.ids(sql).await + } + + /// `ActivityTimeoutEnforcer::expiredExecutionIds` within the requested + /// connections and queues. + async fn scoped_expired_activity_ids(&self, request: &PassRequest, at: DateTime) -> Result> { + let mut sql = Sql::new("SELECT id FROM activity_executions WHERE status IN ('pending','running') AND ((status='pending' AND schedule_deadline_at IS NOT NULL AND schedule_deadline_at<=", DB::NAME == "MySQL"); + sql.time(at).push(") OR (status='running' AND close_deadline_at IS NOT NULL AND close_deadline_at<=").time(at); + sql.push(") OR (schedule_to_close_deadline_at IS NOT NULL AND schedule_to_close_deadline_at<=").time(at); + sql.push(") OR (status='running' AND heartbeat_deadline_at IS NOT NULL AND heartbeat_deadline_at<=").time(at).push("))"); + sql.requested("connection", request.connection.as_deref()).requested("queue", request.queue.as_deref()); + sql.push(&format!(" ORDER BY id LIMIT {SCAN_LIMIT}")); + self.ids(sql).await + } + + /// `TaskWatchdog::runPass`, with its loop throttle shared by this + /// server's namespaces. + pub(crate) async fn repair_pass(&self, request: &PassRequest) -> Result { + { + let mut last = self.repair_pass_at.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let recent = last.is_some_and(|at| at.elapsed().as_secs() < LOOP_THROTTLE_SECONDS); + if request.respect_throttle && recent { + return Ok(Value::Object(empty_report(request, true))); + } + *last = Some(Instant::now()); + } + let mut report = empty_report(request, false); + let at = now(); + let existing = self.repair_task_ids(request, at).await?; + let missing = self.repair_run_ids(request).await?; + report.insert("selected_existing_task_candidates".into(), json!(existing.len())); + report.insert("selected_missing_task_candidates".into(), json!(missing.len())); + report.insert("selected_total_candidates".into(), json!(existing.len() + missing.len())); + for task_id in &existing { + let message = match self.recover_candidate_task(task_id).await { + Ok(Recovered::Task) => { + bump(&mut report, "repaired_existing_tasks"); + bump(&mut report, "dispatched_tasks"); + continue; + } + Ok(Recovered::Nothing | Recovered::ExpireLease) => continue, + Ok(Recovered::Unported(message)) => message, + Err(error) => error.to_string(), + }; + self.record_task_error(task_id, &message).await; + record_failure(&mut report, "existing_task_failures", "candidate_id", task_id, message); + } + for run_id in &missing { + let message = match self.recover_missing_run(run_id).await { + Ok(Recovered::Task) => { + bump(&mut report, "repaired_missing_tasks"); + bump(&mut report, "dispatched_tasks"); + continue; + } + Ok(Recovered::Nothing | Recovered::ExpireLease) => continue, + Ok(Recovered::Unported(message)) => message, + Err(error) => error.to_string(), + }; + record_failure(&mut report, "missing_run_failures", "run_id", run_id, message); + } + let deadlines = self.deadline_expired_run_ids(request, now()).await?; + report.insert("deadline_expired_candidates".into(), json!(deadlines.len())); + for run_id in &deadlines { + match self.deadline_expired_task(run_id).await { + Ok((created, enforced)) => { + if created { + bump(&mut report, "deadline_expired_tasks_created"); + bump(&mut report, "dispatched_tasks"); + } + if enforced { + bump(&mut report, "cancellation_deadlines_enforced"); + } + } + Err(error) => record_failure(&mut report, "deadline_expired_failures", "run_id", run_id, error.to_string()), + } + } + let activities = self.scoped_expired_activity_ids(request, now()).await?; + report.insert("activity_timeout_candidates".into(), json!(activities.len())); + for activity_id in &activities { + match self.enforce_activity_deadline(activity_id).await { + Ok(super::activity_deadlines::Enforcement::Enforced { .. }) => { + // Native always opens the next task: the retry, or the + // workflow task that sees the timeout. + bump(&mut report, "activity_timeouts_enforced"); + bump(&mut report, "dispatched_tasks"); + } + Ok(super::activity_deadlines::Enforcement::Skipped(reason)) => { + record_failure(&mut report, "activity_timeout_failures", "execution_id", activity_id, reason.to_owned()); + } + Err(error) => record_failure(&mut report, "activity_timeout_failures", "execution_id", activity_id, error.to_string()), + } + } + self.wake.notify_waiters(); + Ok(Value::Object(report)) + } + + /// `failedExistingTask`: the failure is kept on the task for operators. + /// Recording it must not stop the pass. + async fn record_task_error(&self, task_id: &str, message: &str) { + let _ = Self::query("UPDATE workflow_tasks SET last_error=$1 WHERE id=$2") + .bind(message) + .bind(task_id) + .execute(&self.pool) + .await; + } + + /// `recoverExistingTask` for one candidate, under its run's lock. + async fn recover_candidate_task(&self, task_id: &str) -> Result { + let mut tx = self.begin().await?; + let Some(snapshot) = Self::query("SELECT workflow_run_id FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_optional(&mut *tx).await? else { + tx.commit().await?; + return Ok(Recovered::Nothing); + }; + let run_id = DB::string(&snapshot, "workflow_run_id")?; + let Some(run) = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(&run_id).fetch_optional(&mut *tx).await? else { + tx.commit().await?; + return Ok(Recovered::Nothing); + }; + let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1 AND workflow_run_id=$2") + .bind(task_id) + .bind(&run_id) + .fetch_optional(&mut *tx) + .await? + else { + tx.commit().await?; + return Ok(Recovered::Nothing); + }; + let at = now(); + if !super::task_repair::needs_recovery::(&task, at)? { + tx.commit().await?; + return Ok(Recovered::Nothing); + } + let recovered = Self::recover_task_row(&mut tx, &run, &task, at).await?; + match recovered { + Recovered::Unported(_) => return Ok(recovered), + Recovered::ExpireLease => { + drop(tx); + return self.settle_lease(task_id).await; + } + _ => {} + } + Self::project_run_summary(&mut tx, &run_id).await?; + if let Recovered::Task = recovered { + Self::mark_dispatched(&mut tx, task_id).await?; + } + tx.commit().await?; + Ok(recovered) + } + + /// `TaskRepair::recoverExistingTask` for a task that needs recovery. + async fn recover_task_row(tx: &mut Transaction<'_, DB>, run: &DB::Row, task: &DB::Row, at: DateTime) -> Result { + let task_id = DB::string(task, "id")?; + let task_type = DB::string(task, "task_type")?; + let run_status = DB::string(run, "status")?; + let closed = matches!(run_status.as_str(), "completed" | "failed" | "cancelled" | "terminated"); + let execution = if task_type == "activity" { + match DB::document_row(task, "payload")?["activity_execution_id"].as_str() { + Some(execution_id) => Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(execution_id.to_owned()) + .fetch_optional(&mut **tx) + .await?, + None => None, + } + } else { + None + }; + let execution_status = execution.as_ref().map(|row| DB::string(row, "status")).transpose()?; + // Native leaves an activity open after its run closes only when the + // run completed without waiting for it (`ActivityAbandonment::allows`). + let abandoned = closed && matches!(execution_status.as_deref(), Some("pending" | "running")); + if closed && !abandoned { + // `settleTerminalTask`, closing a cancelled or terminated run's + // running attempt. + if matches!(run_status.as_str(), "cancelled" | "terminated") + && let Some(execution) = &execution + { + Self::close_current_attempt(tx, execution, "cancelled", at).await?; + } + let status = if DB::string(task, "status")? == "cancelled" { + "cancelled" + } else if run_status == "failed" { + "failed" + } else { + "completed" + }; + Self::query("UPDATE workflow_tasks SET status=$1,leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,updated_at=$2 WHERE id=$3") + .bind(status).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + return Ok(Recovered::Nothing); + } + if abandoned + && let Some(execution) = &execution + && super::activity_deadlines::expired_kind::(execution, at)?.is_some() + { + // Timeout enforcement owns the abandoned activity's outcome. + return Ok(Recovered::Nothing); + } + let repair_count = || { + DB::number(task, "repair_count")? + .checked_add(1) + .filter(|count| *count <= DB::REPAIR_COUNTER_MAX) + .ok_or_else(|| super::refuse(axum::http::StatusCode::CONFLICT, "repair_count_exhausted")) + }; + match DB::string(task, "status")?.as_str() { + "ready" => { + Self::query("UPDATE workflow_tasks SET repair_count=$1,repair_available_at=NULL,last_error=NULL,updated_at=$2 WHERE id=$3") + .bind(repair_count()?).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + } + "leased" => { + // Loss of a running activity's lease consumes its retry + // policy; it never redispatches the same attempt. + if execution_status.as_deref() == Some("running") { + return Ok(Recovered::ExpireLease); + } + let repair_count = repair_count()?; + // `closeExpiredActivityAttempt`. + if let Some(execution) = &execution { + Self::close_current_attempt(tx, execution, "expired", at).await?; + } + Self::query("UPDATE workflow_tasks SET status='ready',leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,repair_count=$1,repair_available_at=NULL,last_error=NULL,updated_at=$2 WHERE id=$3") + .bind(repair_count).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + } + "failed" if super::task_repair::replay_blocked::(task)? => { + // `recoverFailedReplayTask`. + let repair_count = repair_count()?; + let mut payload = DB::document_row(task, "payload")?; + if let Some(fields) = payload.as_object_mut() { + fields.retain(|key, _| !key.starts_with("replay_blocked")); + } + Self::query("UPDATE workflow_tasks SET status='ready',payload=$1,leased_at=NULL,lease_owner=NULL,lease_expires_at=NULL,repair_count=$2,repair_available_at=NULL,last_error=NULL,updated_at=$3 WHERE id=$4") + .bind(DB::document(&payload)).bind(repair_count).bind(DB::bind_time(at)).bind(&task_id).execute(&mut **tx).await?; + } + _ => return Ok(Recovered::Nothing), + } + tx.summary_changed(&DB::string(run, "id")?); + Ok(Recovered::Task) + } + + /// `ActivityTimeoutEnforcer::enforceLeaseExpiry` for a repair candidate: + /// a settled lease opens the retry or the workflow task that sees the + /// failure. A lease the enforcer leaves alone needs no repair. + async fn settle_lease(&self, task_id: &str) -> Result { + Ok(match self.enforce_lease_expiry(task_id).await? { + super::activity_deadlines::Enforcement::Enforced { .. } => Recovered::Task, + super::activity_deadlines::Enforcement::Skipped(_) => Recovered::Nothing, + }) + } + + /// `closeActivityAttempt` for an execution's current running attempt. + async fn close_current_attempt(tx: &mut Transaction<'_, DB>, execution: &DB::Row, status: &str, at: DateTime) -> Result<()> { + if let Some(attempt) = DB::optional_string(execution, "current_attempt_id")? { + Self::query("UPDATE activity_attempts SET status=$1,lease_expires_at=NULL,closed_at=COALESCE(closed_at,$2),updated_at=$3 WHERE id=$4 AND status='running'") + .bind(status).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(attempt).execute(&mut **tx).await?; + } + Ok(()) + } + + /// `TaskDispatcher::markDispatched` in poll mode. + async fn mark_dispatched(tx: &mut Transaction<'_, DB>, task_id: &str) -> Result<()> { + let at = now(); + Self::query("UPDATE workflow_tasks SET last_dispatch_attempt_at=$1,last_dispatched_at=$2,last_dispatch_error=NULL,repair_available_at=NULL WHERE id=$3 AND status='ready'") + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(task_id).execute(&mut **tx).await?; + Ok(()) + } + + /// `recoverMissingTask`: a run whose projected summary still needs repair + /// and names no next task gets its recoverable task back, or a new one. + async fn recover_missing_run(&self, run_id: &str) -> Result { + let mut tx = self.begin().await?; + let Some(run) = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(run_id).fetch_optional(&mut *tx).await? else { + tx.commit().await?; + return Ok(Recovered::Nothing); + }; + Self::project_run_summary(&mut tx, run_id).await?; + let Some(summary) = Self::query("SELECT liveness_state,next_task_id,wait_kind,resume_source_kind FROM workflow_run_summaries WHERE id=$1") + .bind(run_id) + .fetch_optional(&mut *tx) + .await? + else { + tx.commit().await?; + return Ok(Recovered::Nothing); + }; + if DB::optional_string(&summary, "liveness_state")?.as_deref() != Some("repair_needed") + || DB::optional_string(&summary, "next_task_id")?.is_some() + { + tx.commit().await?; + return Ok(Recovered::Nothing); + } + let at = now(); + // `recoverableTask`: a replay-blocked task first, then the earliest. + let mut recoverable = Vec::new(); + for task in Self::query("SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND status IN ('ready','leased','failed')") + .bind(run_id) + .fetch_all(&mut *tx) + .await? + { + let blocked = super::task_repair::replay_blocked::(&task)?; + if blocked || super::task_repair::needs_recovery::(&task, at)? { + recoverable.push(((!blocked, super::task_repair::recovery_order::(&task)?), task)); + } + } + recoverable.sort_by(|left, right| left.0.cmp(&right.0)); + let (recovered, task_id) = match recoverable.into_iter().next() { + Some((_, task)) => { + let task_id = DB::string(&task, "id")?; + (Self::recover_task_row(&mut tx, &run, &task, at).await?, task_id) + } + None => { + let wait_kind = DB::optional_string(&summary, "wait_kind")?.unwrap_or_default(); + let timer_resume = DB::optional_string(&summary, "resume_source_kind")?.as_deref() == Some("timer"); + if matches!(wait_kind.as_str(), "activity" | "timer" | "update" | "signal" | "child") + || (wait_kind == "condition" && timer_resume) + { + return Ok(Recovered::Unported(format!( + "Native does not yet create a missing task for a run waiting on {wait_kind}." + ))); + } + // `createMissingTask`'s workflow task, with + // `forMissingWorkflowTask`'s empty payload. + let queue = DB::string(&run, "queue")?; + let task_id = Self::create_task_dispatched(&mut tx, run_id, &queue, "workflow", json!({}), false).await?; + Self::query("UPDATE workflow_tasks SET repair_count=1 WHERE id=$1").bind(&task_id).execute(&mut *tx).await?; + (Recovered::Task, task_id) + } + }; + match recovered { + Recovered::Unported(_) => return Ok(recovered), + Recovered::ExpireLease => { + drop(tx); + return self.settle_lease(&task_id).await; + } + _ => {} + } + Self::project_run_summary(&mut tx, run_id).await?; + if let Recovered::Task = recovered { + Self::mark_dispatched(&mut tx, &task_id).await?; + } + tx.commit().await?; + Ok(recovered) + } + + /// `createDeadlineExpiredTask`: an expired cancellation cleanup deadline + /// is enforced under a control task; another expired deadline gets a + /// workflow task, and native closes the run when that task runs. Returns + /// whether a task was created and whether a cleanup deadline was enforced. + async fn deadline_expired_task(&self, run_id: &str) -> Result<(bool, bool)> { + let mut tx = self.begin().await?; + let Some(run) = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(run_id).fetch_optional(&mut *tx).await? else { + tx.commit().await?; + return Ok((false, false)); + }; + if !matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting") { + tx.commit().await?; + return Ok((false, false)); + } + let open = Self::query("SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased') ORDER BY id LIMIT 1") + .bind(run_id) + .fetch_optional(&mut *tx) + .await?; + let at = now(); + let expired = |field: &str| -> Result { Ok(DB::optional_instant(&run, field)?.is_some_and(|deadline| at >= deadline)) }; + let cleanup = DB::optional_string(&run, "cancellation_request_command_id")?.is_some() && expired("cancellation_deadline_at")?; + if !(expired("execution_deadline_at")? || expired("run_deadline_at")? || cleanup) { + tx.commit().await?; + return Ok((false, false)); + } + let queue = DB::string(&run, "queue")?; + if cleanup { + // Expiry is runtime authority; an absent worker must not keep the + // run open. + let control = match open { + Some(task) => task, + None => { + let payload = json!({"reason": "cleanup_deadline_expired"}); + let task_id = Self::create_task_dispatched(&mut tx, run_id, &queue, "workflow", payload, false).await?; + Self::query("UPDATE workflow_tasks SET repair_count=1 WHERE id=$1").bind(&task_id).execute(&mut *tx).await?; + Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(&task_id).fetch_one(&mut *tx).await? + } + }; + Self::finish_root_cancellation(&mut tx, &run, Some(&control)).await?; + tx.summary_changed(run_id); + tx.commit().await?; + return Ok((false, true)); + } + if open.is_some() { + tx.commit().await?; + return Ok((false, false)); + } + let task_id = Self::create_task_dispatched(&mut tx, run_id, &queue, "workflow", json!({"reason": "deadline_expired"}), false).await?; + Self::query("UPDATE workflow_tasks SET repair_count=1 WHERE id=$1").bind(&task_id).execute(&mut *tx).await?; + Self::project_run_summary(&mut tx, run_id).await?; + Self::mark_dispatched(&mut tx, &task_id).await?; + tx.commit().await?; + Ok((true, false)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_robin_takes_one_from_each_scope_in_turn() { + let bucket = |ids: &[&str]| ids.iter().map(|id| (*id).to_owned()).collect::>(); + let buckets = vec![bucket(&["a1", "a2", "a3"]), bucket(&["b1"]), bucket(&["c1", "c2"])]; + assert_eq!(round_robin(buckets, 5), ["a1", "b1", "c1", "a2", "c2"]); + } + + #[test] + fn requested_scopes_match_php() { + let mut sql = Sql::new("", false); + sql.requested("queue", Some(" a , default, a ,b")); + assert_eq!(sql.text, " AND (queue IS NULL OR queue='' OR queue='default' OR queue IN ($1,$2))"); + let mut sql = Sql::new("", true); + sql.requested("connection", Some("redis")); + assert_eq!(sql.text, " AND (connection IN (?))"); + let mut sql = Sql::new("", false); + sql.requested("connection", Some(" , ")); + assert_eq!(sql.text, ""); + } + + #[test] + fn request_rules_and_trimming() { + let request = pass_request(&json!({"run_ids": [" r "], "queue": "", "respect_throttle": "1"})).unwrap(); + assert_eq!((request.run_ids, request.queue, request.respect_throttle), (vec!["r".to_owned()], None, true)); + assert!(pass_request(&json!({"run_ids": "x"})).is_err()); + assert!(pass_request(&json!({"run_ids": [""]})).is_err()); + assert!(pass_request(&json!({"respect_throttle": "yes"})).is_err()); + assert!(pass_request(&Value::Null).is_ok()); + } +} diff --git a/rust/src/runtime/route_fallback.rs b/rust/src/runtime/route_fallback.rs new file mode 100644 index 000000000..0daec1add --- /dev/null +++ b/rust/src/runtime/route_fallback.rs @@ -0,0 +1,128 @@ +//! What published PHP 2.11.1 answers for a request no native route serves. +//! PHP's route table (`php-routes.v2.11.1.json`, from `route:list`) decides: +//! a path PHP serves with this method is a PHP capability native does not +//! have yet, refused with 501; a path PHP serves with other methods is PHP's +//! 405, naming them; any other path is PHP's 404. +use serde_json::Value; +use std::sync::OnceLock; + +struct Route { + methods: Vec, + segments: Vec, + /// A parameter that takes the rest of the path, as `->where('name', '.+')`. + wide: Option, +} + +fn routes() -> &'static [Route] { + static ROUTES: OnceLock> = OnceLock::new(); + ROUTES.get_or_init(|| { + let table: Value = serde_json::from_str(include_str!("php-routes.v2.11.1.json")).expect("PHP route table"); + let wide = &table["wide_parameters"]; + table["routes"] + .as_array() + .expect("routes") + .iter() + .map(|route| { + let uri = route[1].as_str().expect("uri").trim_matches('/'); + let segments: Vec = + if uri.is_empty() { Vec::new() } else { uri.split('/').map(str::to_owned).collect() }; + let wide = wide.as_object().and_then(|prefixes| { + prefixes + .iter() + .find(|(prefix, _)| uri.starts_with(prefix.as_str())) + .and_then(|(_, name)| name.as_str().map(str::to_owned)) + }); + Route { + methods: route[0].as_str().expect("methods").split('|').map(str::to_owned).collect(), + segments, + wide, + } + }) + .collect() + }) +} + +/// Whether the route's pattern matches the path, a `{parameter}` taking one +/// non-empty segment, or the rest of the path when it is the wide one. +fn matches(route: &Route, path: &[&str]) -> bool { + let mut at = 0; + for (index, segment) in route.segments.iter().enumerate() { + let parameter = segment.starts_with('{') && segment.ends_with('}'); + if parameter && route.wide.as_deref() == Some(&segment[1..segment.len() - 1]) { + // The wide parameter takes what the rest of the pattern leaves. + let rest = route.segments.len() - index - 1; + let Some(end) = path.len().checked_sub(rest).filter(|end| *end > at) else { + return false; + }; + if path[at..end].iter().any(|part| part.is_empty()) { + return false; + } + at = end; + continue; + } + let Some(part) = path.get(at) else { + return false; + }; + let matched = if parameter { !part.is_empty() } else { part == segment }; + if !matched { + return false; + } + at += 1; + } + at == path.len() +} + +#[derive(Debug, PartialEq)] +pub(super) enum Answer { + /// PHP serves this request; native does not yet. + NotImplemented, + MethodNotAllowed(String), + NotFound(String), +} + +pub(super) fn answer(method: &str, path: &str) -> Answer { + let trimmed = path.trim_matches('/'); + let parts: Vec<&str> = if trimmed.is_empty() { Vec::new() } else { trimmed.split('/').collect() }; + let shown = if trimmed.is_empty() { "/" } else { trimmed }; + let mut allowed: Vec<&str> = Vec::new(); + for route in routes().iter().filter(|route| matches(route, &parts)) { + if route.methods.iter().any(|allowed| allowed == method) { + return Answer::NotImplemented; + } + for method in &route.methods { + if !allowed.contains(&method.as_str()) { + allowed.push(method); + } + } + } + if allowed.is_empty() { + Answer::NotFound(format!("The route {shown} could not be found.")) + } else { + Answer::MethodNotAllowed(format!( + "The {method} method is not supported for route {shown}. Supported methods: {}.", + allowed.join(", ") + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn answers_follow_php_s_route_table() { + assert_eq!(answer("GET", "/api/nope"), Answer::NotFound("The route api/nope could not be found.".into())); + assert_eq!(answer("POST", "/api/workflows/x/redrive"), Answer::NotFound("The route api/workflows/x/redrive could not be found.".into())); + assert_eq!(answer("DELETE", "/api/workflows"), + Answer::MethodNotAllowed("The DELETE method is not supported for route api/workflows. Supported methods: GET, HEAD, POST.".into())); + assert_eq!(answer("POST", "/api/deployments"), + Answer::MethodNotAllowed("The POST method is not supported for route api/deployments. Supported methods: GET, HEAD.".into())); + assert_eq!(answer("PUT", "/api/workflows/x/signal/s"), + Answer::MethodNotAllowed("The PUT method is not supported for route api/workflows/x/signal/s. Supported methods: POST.".into())); + // The deployment name takes slashes; an action segment still ends it. + assert_eq!(answer("GET", "/api/deployments/default/q@v1"), Answer::NotImplemented); + assert_eq!(answer("POST", "/api/deployments/default/q@v1/promote"), Answer::NotImplemented); + assert_eq!(answer("HEAD", "/api/workflows"), Answer::NotImplemented); + assert_eq!(answer("GET", "/"), Answer::NotImplemented); + } +} diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 55aebc489..070e4e364 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -264,6 +264,10 @@ where fields.insert("business_key".into(), json!(DB::optional_string(run, "business_key")?)); fields.insert("resume_step_sequence".into(), json!(started["resume_step_sequence"].as_i64())); fields.insert("recovery_kind".into(), json!(started["recovery_kind"].as_str())); + // A redriven run continues its failed source, as its start records. + if started["recovery_kind"] == "redrive" && started["continued_from_run_id"].is_string() { + fields.insert("continued_from_run_id".into(), started["continued_from_run_id"].clone()); + } fields.insert("compatibility".into(), json!(compatibility)); fields.insert("compatibility_status".into(), json!(if supported { "compatible" } else { "no_compatible_worker" })); fields.insert("compatibility_supported_in_fleet".into(), json!(supported)); diff --git a/rust/src/runtime/search_attribute_upserts.rs b/rust/src/runtime/search_attribute_upserts.rs index ff717f81a..611d188aa 100644 --- a/rust/src/runtime/search_attribute_upserts.rs +++ b/rust/src/runtime/search_attribute_upserts.rs @@ -607,7 +607,10 @@ where ) -> Result<()> { let run_id = DB::string(run, "id")?; let task_id = DB::string(task, "id")?; - if command.get("attribute_types").is_some() { + // PHP fills in each upsert's registered types before its second + // capability check, so any upsert with attributes needs the capability + // (`validateWorkflowTaskSearchAttributeCommands`). + if command.get("attribute_types").is_some() || command["attributes"].is_object() || command["attributes"].is_array() { let worker_id = DB::optional_string(task, "lease_owner")? .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_not_leased"))?; let worker = Self::query("SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") diff --git a/rust/src/runtime/service-catalog-read-contract.v1.json b/rust/src/runtime/service-catalog-read-contract.v1.json new file mode 100644 index 000000000..7d6152e98 --- /dev/null +++ b/rust/src/runtime/service-catalog-read-contract.v1.json @@ -0,0 +1,34 @@ +{ + "schema": "durable-workflow.v2.service-catalog-read.contract", + "version": 1, + "control_plane_version": "2", + "roles": [ + "operator", + "admin" + ], + "namespace": "resolved_authenticated_observer", + "wire_surface": { + "endpoints": "/api/service-catalog/endpoints", + "endpoint": "/api/service-catalog/endpoints/{id}", + "services": "/api/service-catalog/services", + "service": "/api/service-catalog/services/{id}", + "operations": "/api/service-catalog/operations", + "operation": "/api/service-catalog/operations/{id}", + "calls": "/api/service-catalog/calls", + "call": "/api/service-catalog/calls/{id}" + }, + "pagination": { + "default_per_page": 50, + "max_per_page": 100, + "related_limit": 100, + "stable_ties": "id_ascending" + }, + "call_scopes": [ + "relevant", + "owned", + "caller", + "target" + ], + "foreign_payloads": "redacted", + "unknown_or_foreign_detail": "catalog_object_not_found" +} diff --git a/rust/src/runtime/standalone_activities.rs b/rust/src/runtime/standalone_activities.rs new file mode 100644 index 000000000..c5d25dd54 --- /dev/null +++ b/rust/src/runtime/standalone_activities.rs @@ -0,0 +1,438 @@ +//! Standalone activities, as published PHP serves them (`ActivityController`, +//! `StandaloneActivityStartService`): an activity started as a top-level job. +//! +//! The server records each one inside a host run of type +//! `dw.standalone_activity`. The run has no workflow task: its one activity +//! is the work, and the activity's terminal outcome closes the run. +use super::laravel_rules::{self, Rule}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend, store::{Store, id, now}, wire}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +/// `StandaloneActivityHostType::WORKFLOW_TYPE`. +pub(super) const HOST_TYPE: &str = "dw.standalone_activity"; + +fn identifier(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) +} + +/// `ActivityController::start`'s rules. +const START_RULES: &laravel_rules::Rules = &[ + ("activity_id", &[Rule::Nullable, Rule::String, Rule::Max(128), Rule::Pattern(identifier)]), + ("activity_type", &[Rule::Required, Rule::String, Rule::Max(255)]), + ("activity_class", &[Rule::Nullable, Rule::String, Rule::Max(255)]), + ("task_queue", &[Rule::Nullable, Rule::String, Rule::Max(255)]), + ("input", &[Rule::Nullable, Rule::Array]), + ("business_key", &[Rule::Nullable, Rule::String, Rule::Max(255)]), + ("retry_policy", &[Rule::Nullable, Rule::Array]), + ("retry_policy.max_attempts", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("retry_policy.backoff_seconds", &[Rule::Nullable, Rule::Array]), + ("retry_policy.backoff_seconds.*", &[Rule::Integer, Rule::Min(0)]), + ("retry_policy.non_retryable_error_types", &[Rule::Nullable, Rule::Array]), + ("retry_policy.non_retryable_error_types.*", &[Rule::String]), + ("start_to_close_timeout_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("schedule_to_start_timeout_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("schedule_to_close_timeout_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("heartbeat_timeout_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), +]; + +/// `ActivityController::index`'s rules. +const LIST_RULES: &laravel_rules::Rules = &[ + ("status", &[Rule::Nullable, Rule::String, Rule::In(&["running", "completed", "failed"])]), + ("page_size", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(200)]), + ("next_page_token", &[Rule::Nullable, Rule::String]), +]; + +/// A validated start request. +pub(super) struct StartRequest { + pub(super) activity_id: Option, + pub(super) activity_type: String, + pub(super) activity_class: String, + pub(super) task_queue: String, + pub(super) business_key: Option, + /// The worker-facing retry and timeout snapshot. + pub(super) policy: Value, +} + +/// A JSON integer, as PHP's `is_int` sees a decoded request value. +fn int(value: &Value) -> Option { + value.as_i64().filter(|_| value.is_i64() || value.is_u64()) +} + +/// Validates a start request (already normalized) like PHP. +pub(super) fn start_request(body: &Value) -> Result { + laravel_rules::validate(body, START_RULES)?; + let text = |field: &str| body[field].as_str().map(str::to_owned); + let activity_type = text("activity_type").expect("validated activity type"); + let policy = &body["retry_policy"]; + // `StandaloneActivityStartService`: only integers and non-empty strings + // reach the snapshot; anything else falls back to its default. + let backoff: Vec = policy["backoff_seconds"] + .as_array() + .map(|values| values.iter().filter_map(int).map(Value::from).collect()) + .or_else(|| int(&policy["backoff_seconds"]).map(|value| vec![json!(value)])) + .unwrap_or_default(); + let types: Vec = policy["non_retryable_error_types"] + .as_array() + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .filter(|kind| !kind.is_empty()) + .map(Value::from) + .collect() + }) + .unwrap_or_default(); + let command = json!({ + "retry_policy": {"max_attempts": int(&policy["max_attempts"]).unwrap_or(1), + "backoff_seconds": backoff, "non_retryable_error_types": types}, + "start_to_close_timeout": int(&body["start_to_close_timeout_seconds"]), + "schedule_to_start_timeout": int(&body["schedule_to_start_timeout_seconds"]), + "schedule_to_close_timeout": int(&body["schedule_to_close_timeout_seconds"]), + "heartbeat_timeout": int(&body["heartbeat_timeout_seconds"]), + }); + let policy = super::activity_failures::retry_policy(&command)? + .expect("an explicit policy always has a snapshot"); + Ok(StartRequest { + activity_id: text("activity_id"), + activity_class: text("activity_class").unwrap_or_else(|| activity_type.clone()), + activity_type, + // PHP's default queue is the database connection's queue. + task_queue: text("task_queue").unwrap_or_else(|| "default".into()), + business_key: text("business_key"), + policy, + }) +} + +/// A validated listing request. +pub(super) struct ListRequest { + pub(super) status: Option, + pub(super) page_size: i64, + pub(super) offset: i64, +} + +pub(super) fn list_request(query: &Value) -> Result { + let query = laravel_rules::normalize(query); + laravel_rules::validate(&query, LIST_RULES)?; + // `decodePageToken`: base64 of a decimal offset, else the first page. + let offset = query["next_page_token"] + .as_str() + .and_then(|token| { + use base64::Engine; + base64::engine::general_purpose::STANDARD.decode(token).ok() + }) + .and_then(|bytes| String::from_utf8(bytes).ok()) + .filter(|text| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit())) + .and_then(|text| text.parse::().ok()) + .unwrap_or(0); + Ok(ListRequest { + status: query["status"].as_str().map(str::to_owned), + page_size: super::laravel_validation::integer(&query["page_size"]).unwrap_or(50), + offset, + }) +} + +pub(super) fn page_token(offset: i64) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(offset.to_string()) +} + +fn not_found() -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Standalone activity not found.","reason":"activity_not_found"}), + } +} + +fn invalid_id(message: String) -> RuntimeError { + super::laravel_validation::failure_messages(vec![("activity_id".into(), vec![message])]) + .expect("one message") +} + +/// How a host run closes with its activity. +pub(super) enum HostClose<'a> { + /// `closeStandaloneHostRun` for a result: the run completes with it. + Completed { output: &'a str }, + /// A terminal failure or timeout: `WorkflowFailed` with this payload. + Failed { payload: Value, closed_reason: &'static str }, +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `StandaloneActivityStartService::start`: the host run, its start + /// command and history, the pending activity and its ready task, in one + /// transaction. + pub(crate) async fn start_standalone_activity( + &self, + request: StartRequest, + arguments: Option, + external: Option, + context: Value, + ) -> Result { + let mut tx = self.begin().await?; + if let Some(prepared) = &external { + self.claim_external_in(&mut tx, &prepared.reference, &prepared.uri).await?; + } + let activity_id = request.activity_id.clone().unwrap_or_else(id); + let existing = Self::query("SELECT i.namespace,i.workflow_type,i.run_count,r.status AS run_status,r.run_number FROM workflow_instances i LEFT JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1") + .bind(&activity_id).fetch_optional(&mut *tx).await?; + // `TaskQueueRoutingGate`, after the identifier reservation check. + let reserved = match &existing { + Some(instance) => DB::optional_string(instance, "namespace")?.as_deref() != Some(self.namespace.as_ref()), + None => false, + }; + if !reserved && let Some(block) = Self::routing_block(&mut tx, &self.namespace, &request.task_queue, self.worker_stale_after_seconds).await? { + let mut response = json!({"message":format!("Task queue [{}] is draining and cannot accept new standalone activity starts until an active worker cohort is available.", request.task_queue), + "reason":"task_queue_draining","activity_type":request.activity_type,"task_queue":request.task_queue}); + response.as_object_mut().expect("object").extend(block); + response["command_status"] = json!("rejected"); + response["rejection_reason"] = json!("task_queue_draining"); + return Err(RuntimeError::Protocol { status: StatusCode::CONFLICT, response }); + } + let started_at = now(); + let run_id = id(); + let run_number = match &existing { + None => { + Self::query("INSERT INTO workflow_instances(id,workflow_type,workflow_class,current_run_id,run_count,created_at,updated_at,started_at,namespace) VALUES ($1,$2,$3,$4,1,$5,$6,$7,$8)") + .bind(&activity_id).bind(HOST_TYPE).bind(HOST_TYPE).bind(&run_id).bind(DB::bind_time(started_at)) + .bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(self.namespace.as_ref()) + .execute(&mut *tx).await?; + 1 + } + Some(instance) => { + if DB::optional_string(instance, "namespace")?.as_deref() != Some(self.namespace.as_ref()) { + return Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"message":format!("Identifier [{activity_id}] is already reserved in another namespace."), + "reason":"activity_id_reserved_in_namespace","activity_id":activity_id, + "command_status":"rejected","rejection_reason":"activity_id_reserved_in_namespace"}), + }); + } + let workflow_type = DB::string(instance, "workflow_type")?; + if workflow_type != HOST_TYPE { + return Err(invalid_id(format!("Identifier [{activity_id}] is already reserved for workflow type [{workflow_type}] and cannot be reused for a standalone activity."))); + } + if DB::optional_string(instance, "run_status")? + .is_some_and(|status| matches!(status.as_str(), "pending" | "running" | "waiting")) + { + return Err(invalid_id(format!("Standalone activity [{activity_id}] is already running."))); + } + let number = DB::number(instance, "run_count")? + .max(DB::optional_number(instance, "run_number")?.unwrap_or(0)) + .max(0) + + 1; + Self::query("UPDATE workflow_instances SET current_run_id=$1,run_count=$2,updated_at=$3 WHERE id=$4") + .bind(&run_id).bind(number).bind(DB::bind_time(started_at)).bind(&activity_id) + .execute(&mut *tx).await?; + number + } + }; + let queue = request.task_queue.as_str(); + Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,started_at,created_at,updated_at,namespace,last_progress_at,business_key,last_command_sequence) VALUES ($1,$2,$3,$4,$5,'running','avro',$6,$7,$8,$9,$10,$11,$12,$13,1)") + .bind(&run_id).bind(&activity_id).bind(HOST_TYPE).bind(HOST_TYPE).bind(run_number).bind(arguments.as_deref()) + .bind(queue).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)) + .bind(self.namespace.as_ref()).bind(DB::bind_time(started_at)).bind(request.business_key.as_deref()) + .execute(&mut *tx).await?; + let command_id = id(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,command_sequence,accepted_at,applied_at,context) VALUES ($1,$2,$3,$4,'start','control_plane','accepted','started_new',$5,'avro',$6,1,$7,$8,$9)") + .bind(&command_id).bind(&activity_id).bind(&run_id).bind(&run_id).bind(HOST_TYPE).bind(arguments.as_deref()) + .bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::document(&context)) + .execute(&mut *tx).await?; + let identity = json!({"workflow_instance_id":activity_id,"workflow_run_id":run_id, + "workflow_class":HOST_TYPE,"workflow_type":HOST_TYPE,"workflow_command_id":command_id, + "business_key":request.business_key}); + let mut accepted = identity.clone(); + for (key, value) in [("visibility_labels", Value::Null), ("memo", Value::Null), + ("search_attributes", Value::Null), ("outcome", json!("started_new"))] { + accepted[key] = value; + } + Self::append(&mut tx, &run_id, "StartAccepted", accepted, None, Some(&command_id)).await?; + Self::append(&mut tx, &run_id, "WorkflowStarted", identity, None, Some(&command_id)).await?; + let execution_id = id(); + let schedule_deadline = super::activity_deadlines::deadline(&request.policy, "schedule_to_start_timeout", started_at)?; + let close_deadline = super::activity_deadlines::deadline(&request.policy, "schedule_to_close_timeout", started_at)?; + Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_type,activity_class,status,attempt_count,payload_codec,arguments,queue,retry_policy,schedule_deadline_at,schedule_to_close_deadline_at) VALUES ($1,$2,1,$3,$4,'pending',0,'avro',$5,$6,$7,$8,$9)") + .bind(&execution_id).bind(&run_id).bind(&request.activity_type).bind(&request.activity_class) + .bind(arguments.as_deref()).bind(queue).bind(DB::document(&request.policy)) + .bind(DB::optional_time(schedule_deadline)).bind(DB::optional_time(close_deadline)) + .execute(&mut *tx).await?; + let activity = Self::query("SELECT * FROM activity_executions WHERE id=$1") + .bind(&execution_id) + .fetch_one(&mut *tx) + .await?; + let scheduled_at = now(); + Self::append_at(&mut tx, &run_id, "ActivityScheduled", Self::activity_payload(&activity)?, None, None, Some(scheduled_at)).await?; + // PHP dispatches this task when it creates it. + let task_id = Self::create_task(&mut tx, &run_id, queue, "activity", json!({"activity_execution_id": execution_id})).await?; + Self::create_activity_wait(&mut tx, &run_id, &execution_id, &task_id, scheduled_at).await?; + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + let time = |at: Option>| super::task_queues::json_time(at); + Ok(json!({"activity_id":activity_id,"activity_execution_id":execution_id,"workflow_id":activity_id, + "workflow_run_id":run_id,"workflow_type":HOST_TYPE,"activity_type":request.activity_type, + "activity_class":request.activity_class,"task_queue":queue,"namespace":self.namespace.as_ref(), + "status":"running","payload_codec":"avro","started_at":time(Some(started_at)), + "schedule_to_start_deadline_at":time(schedule_deadline),"schedule_to_close_deadline_at":time(close_deadline), + "command_status":"accepted","command_source":"control_plane"})) + } + + /// The current host run of `activity_id` in this namespace and its + /// activity, as the describe and list views read them. + pub(crate) async fn standalone_host(&self, activity_id: &str) -> Result<(Value, Value)> { + let run = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1 AND i.namespace=$2") + .bind(activity_id).bind(self.namespace.as_ref()).fetch_optional(&self.pool).await? + .ok_or_else(not_found)?; + if DB::string(&run, "workflow_type")? != HOST_TYPE && DB::string(&run, "workflow_class")? != HOST_TYPE { + return Err(not_found()); + } + self.host_view(&run).await + } + + /// One page of host runs, newest first, as the run list orders them. + pub(crate) async fn standalone_hosts(&self, request: &ListRequest) -> Result<(Vec<(Value, Value)>, bool)> { + let filter = super::visibility::VisibilityFilter::of_type(HOST_TYPE, request.status.clone(), request.page_size, request.offset); + let rows = self.visibility_rows(&filter, request.page_size + 1).await?; + let more = rows.len() as i64 > request.page_size; + let mut views = Vec::new(); + for row in rows.iter().take(request.page_size as usize) { + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1") + .bind(DB::string(row, "id")?) + .fetch_one(&self.pool) + .await?; + views.push(self.host_view(&run).await?); + } + Ok((views, more)) + } + + async fn host_view(&self, run: &DB::Row) -> Result<(Value, Value)> { + let time = |at: Option>| super::task_queues::json_time(at); + let status = DB::string(run, "status")?; + let host = json!({"workflow_instance_id":DB::string(run,"workflow_instance_id")?,"id":DB::string(run,"id")?, + "workflow_type":DB::string(run,"workflow_type")?,"queue":DB::string(run,"queue")?, + "business_key":DB::optional_string(run,"business_key")?,"status":status, + "status_bucket":super::visibility::status_bucket(&status)?, + "closed_reason":DB::optional_string(run,"closed_reason")?, + "started_at":time(DB::optional_instant(run,"started_at")?),"closed_at":time(DB::optional_instant(run,"closed_at")?), + "last_progress_at":time(DB::optional_instant(run,"last_progress_at")?), + "compatibility":DB::optional_string(run,"compatibility")?,"payload_codec":DB::optional_string(run,"payload_codec")?}); + let execution = match Self::query("SELECT * FROM activity_executions WHERE workflow_run_id=$1 ORDER BY sequence LIMIT 1") + .bind(DB::string(run, "id")?) + .fetch_optional(&self.pool) + .await? + { + None => Value::Null, + Some(row) => json!({"id":DB::string(&row,"id")?,"activity_type":DB::string(&row,"activity_type")?, + "activity_class":DB::string(&row,"activity_class")?,"status":DB::string(&row,"status")?, + "attempt_count":DB::number(&row,"attempt_count")?,"current_attempt_id":DB::optional_string(&row,"current_attempt_id")?, + "schedule_deadline_at":time(DB::optional_instant(&row,"schedule_deadline_at")?), + "schedule_to_close_deadline_at":time(DB::optional_instant(&row,"schedule_to_close_deadline_at")?), + "last_heartbeat_at":time(DB::optional_instant(&row,"last_heartbeat_at")?), + "result":DB::optional_string(&row,"result")?}), + }; + Ok((host, execution)) + } + + /// Closes a standalone activity's host run with its activity's terminal + /// outcome (`closeStandaloneHostRun`, `closeStandaloneHostRunAfterTimeout`) + /// instead of resuming workflow code. Another run is left untouched. + pub(super) async fn close_standalone_host( + tx: &mut Transaction<'_, DB>, + run_id: &str, + close: HostClose<'_>, + ) -> Result { + let run = Self::query("SELECT workflow_type,workflow_class FROM workflow_runs WHERE id=$1") + .bind(run_id) + .fetch_one(&mut **tx) + .await?; + if DB::string(&run, "workflow_type")? != HOST_TYPE && DB::string(&run, "workflow_class")? != HOST_TYPE { + return Ok(false); + } + let at = now(); + match close { + HostClose::Completed { output } => { + Self::query("UPDATE workflow_runs SET status='completed',closed_reason='completed',output=$1,closed_at=$2,last_progress_at=$3 WHERE id=$4") + .bind(output).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(run_id).execute(&mut **tx).await?; + Self::append(tx, run_id, "WorkflowCompleted", json!({"output":wire(output),"payload_codec":"avro"}), None, None).await?; + } + HostClose::Failed { payload, closed_reason } => { + Self::query("UPDATE workflow_runs SET status='failed',closed_reason=$1,closed_at=$2,last_progress_at=$3 WHERE id=$4") + .bind(closed_reason).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(run_id).execute(&mut **tx).await?; + Self::append(tx, run_id, "WorkflowFailed", payload, None, None).await?; + } + } + tx.summary_changed(run_id); + Ok(true) + } +} + +/// `closeStandaloneHostRun`'s failure payload from the activity's terminal +/// event: its failure fields, with nulls dropped. +pub(super) fn failed_payload(event: &Value, activity_id: &str) -> Value { + let mut payload = json!({"failure_id":event["failure_id"],"source_kind":"activity_execution","source_id":activity_id, + "failure_category":event["failure_category"],"non_retryable":event["non_retryable"], + "exception_type":event["exception_type"],"exception_class":event["exception_class"], + "message":event["message"],"exception":event["exception"]}); + payload.as_object_mut().expect("payload object").retain(|_, value| !value.is_null()); + payload +} + +/// `closeStandaloneHostRunAfterTimeout`'s payload. +pub(super) fn timed_out_payload(failure_id: &str, activity_id: &str, category: &str, non_retryable: bool, class: &str, message: &str) -> Value { + json!({"failure_id":failure_id,"source_kind":"activity_execution","source_id":activity_id, + "failure_category":category,"non_retryable":non_retryable,"exception_type":null, + "exception_class":class,"message":message, + "exception":{"class":class,"message":message,"code":0,"file":"","line":0,"trace":[],"properties":[]}}) +} + +/// `ActivityController::formatAttempt` over one attempt of the run's +/// activity view. +fn attempt(view: &Value, execution: &Value) -> Value { + let id = view["id"].as_str().filter(|id| !id.is_empty()); + let task = view["task_id"].as_str().filter(|id| !id.is_empty()); + json!({"id":id,"activity_attempt_id":id,"activity_execution_id":execution["id"],"workflow_task_id":task,"task_id":task, + "attempt_number":view["attempt_number"].as_i64(),"status":view["status"].as_str(), + "lease_owner":view["lease_owner"].as_str(),"lease_expires_at":view["lease_expires_at"], + "started_at":view["started_at"],"last_heartbeat_at":view["last_heartbeat_at"], + "last_heartbeat_progress":if view["last_heartbeat_progress"].is_object() || view["last_heartbeat_progress"].is_array() { + view["last_heartbeat_progress"].clone() } else { Value::Null }, + "closed_at":view["closed_at"],"can_continue":view["can_continue"] == true, + "cancel_requested":view["cancel_requested"] == true,"stop_reason":view["stop_reason"].as_str(), + "is_current":id.is_some() && id == execution["current_attempt_id"].as_str()}) +} + +/// The attempt fields every describe and list entry carries. +pub(super) fn attempt_fields(execution: &Value, activities: &Value) -> Value { + let attempts: Vec = activities + .as_array() + .and_then(|activities| activities.iter().find(|activity| !execution.is_null() && activity["id"] == execution["id"])) + .and_then(|activity| activity["attempts"].as_array()) + .map(|attempts| attempts.iter().filter(|attempt| attempt.is_object()).map(|view| attempt(view, execution)).collect()) + .unwrap_or_default(); + let current = attempts + .iter() + .find(|attempt| attempt["is_current"] == true) + .or(attempts.last()) + .cloned() + .unwrap_or(Value::Null); + let current_id = if current.is_null() { execution["current_attempt_id"].clone() } else { current["activity_attempt_id"].clone() }; + let current_status = current["status"].clone(); + json!({"activity_execution_id":execution["id"],"attempt_count":execution["attempt_count"], + "current_attempt_id":current_id,"current_attempt_status":current_status,"current_attempt":current, + "attempts":attempts}) +} diff --git a/rust/src/runtime/start_definition.rs b/rust/src/runtime/start_definition.rs index b77f553c1..d62a51dea 100644 --- a/rust/src/runtime/start_definition.rs +++ b/rust/src/runtime/start_definition.rs @@ -3,7 +3,9 @@ //! `WorkflowStarted` records that fingerprint with source `worker`, so //! queries and later tasks can tell whether a worker runs the original //! definition. Native has no build-id rollouts, so PHP's promoted-rollout -//! cohort never applies. +//! cohort never applies. The cohort is also the run's build pin: a run +//! started in one build's cohort is stamped with that build as its +//! `compatibility`, and only that build's workers take its tasks. use serde_json::Value; /// One active registration on the start's task queue. @@ -15,7 +17,7 @@ pub(super) struct CohortWorker { } #[derive(Debug, PartialEq)] -enum Cohort { +pub(super) enum Cohort { Build(String), Unversioned, } @@ -51,20 +53,55 @@ fn cohort<'a>(workers: impl Iterator) -> Option } } -/// `resolveForStart` then `definitionFingerprintForCohort`: the fingerprint -/// every worker in the start's cohort that supports the type advertises, or -/// `None` when any of them has not heartbeated, advertises none, or disagrees. -pub(super) fn fingerprint(workers: &[CohortWorker], workflow_type: &str) -> Option { +impl Cohort { + /// The build a run started in this cohort is pinned to. + pub(super) fn pin(&self) -> Option<&str> { + match self { + Self::Build(build) => Some(build), + Self::Unversioned => None, + } + } + + /// The cohort of a run already pinned (or not) to a build, as redrive + /// and the declared contract look it up. + pub(super) fn of_run(compatibility: Option<&str>) -> Self { + match compatibility.map(str::trim).filter(|build| !build.is_empty()) { + Some(build) => Self::Build(build.to_owned()), + None => Self::Unversioned, + } + } + + /// Whether a registration with this build id belongs to the cohort. + pub(super) fn admits(&self, build_id: Option<&str>) -> bool { + let build_id = build_id.map(str::trim).filter(|build| !build.is_empty()); + match self { + Self::Build(build) => build_id == Some(build.as_str()), + Self::Unversioned => build_id.is_none(), + } + } +} + +/// `resolveForStart` without a promoted rollout: the one cohort among the +/// type's live workers, else the one cohort on the queue, else none. +pub(super) fn resolve(workers: &[CohortWorker], workflow_type: &str) -> Option { + let workflow_type = workflow_type.trim(); + cohort(workers.iter().filter(|worker| worker.supports(workflow_type))).or_else(|| cohort(workers.iter())) +} + +/// `resolveForStart` then `definitionFingerprintForCohort`. +#[cfg(test)] +fn fingerprint(workers: &[CohortWorker], workflow_type: &str) -> Option { + fingerprint_for(workers, workflow_type, &resolve(workers, workflow_type)?) +} + +/// `definitionFingerprintForCohort`: the fingerprint every worker in the +/// cohort that supports the type advertises, or `None` when any of them has +/// not heartbeated, advertises none, or disagrees. +pub(super) fn fingerprint_for(workers: &[CohortWorker], workflow_type: &str, cohort: &Cohort) -> Option { let workflow_type = workflow_type.trim(); - let cohort = cohort(workers.iter().filter(|worker| worker.supports(workflow_type))) - .or_else(|| cohort(workers.iter()))?; let mut agreed: Option = None; for worker in workers.iter().filter(|worker| worker.supports(workflow_type)) { - let member = match &cohort { - Cohort::Build(build) => worker.build() == Some(build.as_str()), - Cohort::Unversioned => worker.build().is_none(), - }; - if !member { + if !cohort.admits(worker.build()) { continue; } let advertised = worker.fingerprints.get(workflow_type).and_then(Value::as_str)?; @@ -121,4 +158,24 @@ mod tests { // No worker supports the type: the queue's single cohort has no member. assert_eq!(fingerprint(&[built("v1", "f1")], "invoice"), None); } + + #[test] + fn the_cohort_is_the_run_s_build_pin_like_php() { + let built = |build: &str, types: Value| worker(Some(build), types, json!({})); + let pin = |workers: &[CohortWorker], kind: &str| resolve(workers, kind).and_then(|cohort| cohort.pin().map(str::to_owned)); + // One build on the queue pins every start to it. + assert_eq!(pin(&[built("v1", json!(["echo", "solo"]))], "echo").as_deref(), Some("v1")); + // Two builds for the type leave the run unpinned; a type only one + // build serves still pins to that build. + let fleet = [built("v1", json!(["echo", "solo"])), built("v2", json!(["echo"]))]; + assert_eq!(pin(&fleet, "echo"), None); + assert_eq!(pin(&fleet, "solo").as_deref(), Some("v1")); + // Unversioned workers alone, or none at all, pin nothing. + assert_eq!(pin(&[worker(None, json!(["echo"]), json!({}))], "echo"), None); + assert_eq!(resolve(&[worker(None, json!(["echo"]), json!({}))], "echo"), Some(Cohort::Unversioned)); + assert_eq!(resolve(&[], "echo"), None); + assert!(Cohort::of_run(Some(" v1 ")).admits(Some("v1"))); + assert!(!Cohort::of_run(Some("v1")).admits(None)); + assert!(Cohort::of_run(Some("")).admits(Some(" "))); + } } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index b798a252a..93c299d12 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -17,12 +17,19 @@ pub(super) struct PreparedControlArguments { } /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; -pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; +pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; /// `nextEligibleSignal`: a run's received signals in PHP's order: command /// sequence (unsequenced last), receipt time, then id. const RECEIVED_SIGNALS: &str = "SELECT id,signal_name,signal_wait_id,command_sequence FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received' AND closed_at IS NULL ORDER BY CASE WHEN command_sequence IS NULL THEN 1 ELSE 0 END,command_sequence,received_at,id LIMIT 100"; pub(super) const POLL_RECEIPT_CLEANUP_SQL: &str = "DELETE FROM dw_poll_receipts WHERE expires_at<=$1 AND task_id IN (SELECT id FROM workflow_tasks WHERE status!='leased' OR lease_expires_at<=$2)"; pub(super) const WORKER_REGISTRATION_SQL: &str = "INSERT INTO workflow_worker_registrations(namespace,worker_id,task_queue,runtime,sdk_version,build_id,supported_workflow_types,supported_activity_types,capabilities,capability_manifest,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) ON CONFLICT(worker_id,namespace) DO UPDATE SET task_queue=excluded.task_queue,runtime=excluded.runtime,sdk_version=excluded.sdk_version,build_id=excluded.build_id,supported_workflow_types=excluded.supported_workflow_types,supported_activity_types=excluded.supported_activity_types,capabilities=excluded.capabilities,capability_manifest=excluded.capability_manifest,last_heartbeat_at=excluded.last_heartbeat_at,updated_at=excluded.updated_at"; +/// A registration's build id as the pollers compare it: trimmed, and none +/// when blank. A task pinned to a build goes only to that build's workers. +pub(super) fn worker_build(row: &DB::Row) -> Result> { + Ok(DB::optional_string(row, "build_id")? + .map(|build| build.trim().to_owned()) + .filter(|build| !build.is_empty())) +} pub(super) struct Store { pub(super) pool: Pool, pub(super) wake: Arc, @@ -39,6 +46,9 @@ pub(super) struct Store { pub(super) search_attribute_limit: i64, /// When a silent registration stops counting (`server.workers.stale_after_seconds`). pub(super) worker_stale_after_seconds: i64, + /// When this server last ran a task repair pass, shared by its namespaces + /// as PHP shares its loop throttle key. + pub(super) repair_pass_at: Arc>>, } /// The `closed_reason` a terminal history event gives its run, or `None` for /// an event that does not close the run. @@ -118,6 +128,7 @@ where update_wait_seconds: 10, search_attribute_limit: 100, worker_stale_after_seconds: 30, + repair_pass_at: Arc::default(), } } pub(super) fn in_namespace(&self, namespace: &str) -> Self { @@ -133,6 +144,7 @@ where update_wait_seconds: self.update_wait_seconds, search_attribute_limit: self.search_attribute_limit, worker_stale_after_seconds: self.worker_stale_after_seconds, + repair_pass_at: self.repair_pass_at.clone(), } } pub(super) async fn database_live(&self) -> bool { @@ -199,6 +211,26 @@ where self.claim_external_in(&mut tx, &prepared.reference, &prepared.uri) .await?; } + // `TaskQueueRoutingGate`: a queue whose live workers all drain takes + // no new start. A workflow id reserved in another namespace is + // refused for that first. + let request = super::start_request::validate(&body)?; + let reserved = match &request.workflow_id { + Some(workflow_id) => Self::query("SELECT namespace FROM workflow_instances WHERE id=$1") + .bind(workflow_id.as_str()).fetch_optional(&mut *tx).await? + .map(|instance| DB::optional_string(&instance, "namespace")).transpose()?.flatten() + .is_some_and(|reserved| reserved != self.namespace.as_ref()), + None => false, + }; + if !reserved && let Some(block) = Self::routing_block(&mut tx, &self.namespace, &request.task_queue, self.worker_stale_after_seconds).await? { + let mut response = json!({"workflow_id":request.workflow_id,"command_status":"rejected","command_source":"control_plane", + "outcome":"rejected_task_queue_draining","reason":"task_queue_draining","rejection_reason":"task_queue_draining", + "message":format!("Task queue [{}] is draining and cannot accept new workflow starts until an active worker cohort is available.", request.task_queue), + "workflow_type":request.workflow_type,"task_queue":request.task_queue}); + response.as_object_mut().expect("object").extend(block); + response.as_object_mut().expect("object").retain(|_, value| !value.is_null()); + return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response }); + } let started = Self::start_in( &mut tx, &body, @@ -256,6 +288,14 @@ where let run_id = id(); let command_id = id(); let started_at = now(); + // `WorkflowStartVersionPin::resolveForStart`: the run is pinned to its + // cohort's build, and records the definition that cohort agrees on. + let workers = Self::start_workers(tx, namespace, queue, stale_after).await?; + let cohort = match Self::promoted_cohort(tx, namespace, queue).await? { + Some(cohort) => Some(cohort), + None => super::start_definition::resolve(&workers, workflow_type), + }; + let pin = cohort.as_ref().and_then(|cohort| cohort.pin().map(str::to_owned)); // DefaultWorkflowControlPlane::start: each deadline follows its own // timeout, and a run without timeouts has no deadline. let execution_deadline = execution_timeout.map(|seconds| started_at + chrono::Duration::seconds(seconds)); @@ -271,12 +311,12 @@ where .bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(namespace) .bind(DB::optional_document(&authored_memo)).bind(request.business_key.as_deref()).execute(&mut **tx).await?; // PHP records the start as the run's first progress. - Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,run_timeout_seconds,execution_deadline_at,run_deadline_at,started_at,created_at,updated_at,namespace,last_progress_at,business_key,priority,fairness_key,fairness_weight) VALUES ($1,$2,$3,$4,1,'pending','avro',$5,$6,NULLIF($7,0),$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)") + Self::query("INSERT INTO workflow_runs(id,workflow_instance_id,workflow_type,workflow_class,run_number,status,payload_codec,arguments,queue,run_timeout_seconds,execution_deadline_at,run_deadline_at,started_at,created_at,updated_at,namespace,last_progress_at,business_key,priority,fairness_key,fairness_weight,compatibility) VALUES ($1,$2,$3,$4,1,'pending','avro',$5,$6,NULLIF($7,0),$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)") .bind(&run_id).bind(workflow_id).bind(workflow_type).bind(workflow_type).bind(&arguments) .bind(queue).bind(run_timeout.unwrap_or(0)).bind(DB::optional_time(execution_deadline)).bind(DB::optional_time(run_deadline)) .bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).bind(namespace) .bind(DB::bind_time(started_at)).bind(request.business_key.as_deref()).bind(request.priority) - .bind(request.fairness_key.as_deref()).bind(request.fairness_weight).execute(&mut **tx).await?; + .bind(request.fairness_key.as_deref()).bind(request.fairness_weight).bind(pin.as_deref()).execute(&mut **tx).await?; if let Some(memo) = memo { for (key, value) in memo { Self::query(DB::MEMO_INITIAL_INSERT) @@ -321,16 +361,20 @@ where started["run_timeout_seconds"] = json!(run_timeout); started["execution_deadline_at"] = json!(execution_deadline); started["run_deadline_at"] = json!(run_deadline); - if let Some(fingerprint) = Self::start_definition_fingerprint(tx, namespace, queue, workflow_type, stale_after).await? { + if let Some(fingerprint) = cohort.as_ref().and_then(|cohort| super::start_definition::fingerprint_for(&workers, workflow_type, cohort)) { started["workflow_definition_fingerprint"] = json!(fingerprint); started["workflow_definition_fingerprint_source"] = json!("worker"); } + started["compatibility"] = json!(pin); // PHP filters null fields out of the started payload, then adds the // declared contract. if let Some(fields) = started.as_object_mut() { fields.retain(|_, value| !value.is_null()); } - Self::attach_workflow_contract(tx, queue, workflow_type, &mut started).await?; + // PHP enriches the start with a contract only from its cohort. + if let Some(cohort) = &cohort { + Self::attach_workflow_contract(tx, queue, workflow_type, Some(cohort), &mut started).await?; + } Self::append( tx, &run_id, @@ -352,13 +396,12 @@ where /// `WorkflowStartVersionPin::definitionFingerprintForCohort` over the /// queue's active registrations: unexpired or never heartbeated, and /// not draining or deregistered. - async fn start_definition_fingerprint( + pub(super) async fn start_workers( tx: &mut Transaction<'_, DB>, namespace: &str, queue: &str, - workflow_type: &str, stale_after: i64, - ) -> Result> { + ) -> Result> { let cutoff = now() - chrono::Duration::seconds(stale_after.max(1)); let rows = Self::query("SELECT build_id,supported_workflow_types,workflow_definition_fingerprints,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND task_queue=$2 AND (last_heartbeat_at IS NULL OR last_heartbeat_at>=$3) AND (status IS NULL OR status='active')") .bind(namespace).bind(queue).bind(DB::bind_time(cutoff)).fetch_all(&mut **tx).await?; @@ -371,7 +414,7 @@ where heartbeated: DB::optional_instant(row, "last_heartbeat_at")?.is_some(), }); } - Ok(super::start_definition::fingerprint(&workers, workflow_type)) + Ok(workers) } /// `DefaultWorkflowControlPlane::start` for an instance that already has @@ -501,7 +544,8 @@ where commands.push(public); } let current = DB::optional_string(&row,"current_run_id")?.as_deref() == Some(DB::string(&row,"id")?.as_str()); - let input_envelope = wire(&DB::string(&row,"arguments")?); + // A standalone activity's host run may have no input. + let input_envelope = DB::optional_string(&row,"arguments")?.map(|arguments| wire(&arguments)).unwrap_or(Value::Null); let output_envelope = output.as_deref().map(wire); let mut described = json!({"workflow_id": workflow_id, "run_id": DB::string(&row,"id")?, "workflow_type": DB::string(&row,"workflow_type")?, @@ -563,6 +607,8 @@ where mut prepared: PreparedControlArguments, transition_codec: Arc, context: Option, + limits: super::memo_limits::StructuralLimits, + external: Option<&super::external_resolution::PreparedExternal>, ) -> Result { // The authenticated request's context; a direct call records only // the request id and signal name. @@ -584,6 +630,11 @@ where )); } let mut tx = self.begin().await?; + // The referenced object now belongs to the signal; a refusal rolls + // the claim back with everything else. + if let Some(prepared) = external { + self.claim_external_in(&mut tx, &prepared.reference, &prepared.uri).await?; + } let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1 AND namespace=$2") .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?.ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; let current_run = DB::string(&instance, "current_run_id")?; @@ -632,17 +683,17 @@ where .as_array() .is_some_and(|contracts| contracts.iter().any(|contract| contract["name"] == name)); let mut validation_errors = json!([]); - let rejection = if !active { + let mut rejection = if !active { Some(( StatusCode::CONFLICT, "run_not_active", - "rejected_not_active", + Some("rejected_not_active"), )) } else if !declared && durable { Some(( StatusCode::NOT_FOUND, "unknown_signal", - "rejected_unknown_signal", + Some("rejected_unknown_signal"), )) } else if !has_parameters { None @@ -657,12 +708,29 @@ where Some(( StatusCode::UNPROCESSABLE_ENTITY, "invalid_signal_arguments", - "rejected_invalid_arguments", + Some("rejected_invalid_arguments"), )) } Err(error) => return Err(error), } }; + // `guardPendingSignals`, then `rejectOversizedSignal`: an admissible + // signal over a structural ceiling is refused durably, without an + // outcome or history. + let mut exceeded = None; + if rejection.is_none() { + if limits.pending_signals.is_some() { + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") + .bind(¤t_run).fetch_one(&mut *tx).await?; + exceeded = super::memo_limits::Exceeded::pending_signals(limits.pending_signals, pending); + } + if exceeded.is_none() { + exceeded = super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments); + } + if exceeded.is_some() { + rejection = Some((StatusCode::CONFLICT, "structural_limit_exceeded", None)); + } + } let command_id = id(); let signal_id = id(); let command_sequence = DB::number(&run, "last_command_sequence")? @@ -689,12 +757,25 @@ where "declared_signals":names,"signal_admission":"handler_not_declared","reason":reason, "message":format!("Workflow signal [{name}] is unknown for workflow type [{workflow_type}]: the durable command contract does not declare that handler. Command contract source [durable_history], declared signals [{labels}].")}); } + // An oversized signal keeps no arguments: its command carries the + // limit instead, and its signal row the limit's message. + let (mut row_arguments, mut row_errors) = (prepared.arguments.clone(), validation_errors.clone()); + let mut command_values = prepared.values.clone(); + let mut command_errors = validation_errors.clone(); + let mut command_diagnostics = diagnostics.clone(); + if let Some(exceeded) = exceeded.as_ref().filter(|exceeded| exceeded.kind == "payload_size") { + command_values = vec![]; + command_errors = json!({"arguments":[exceeded.message]}); + command_diagnostics = exceeded.fields(); + row_arguments = super::queries::empty_arguments(transition_codec.clone()).await?; + row_errors = command_errors.clone(); + } prepared.command = prepare_rejected_command( transition_codec, name.to_owned(), - prepared.values.clone(), - validation_errors.clone(), - diagnostics.clone(), + command_values, + command_errors, + command_diagnostics, "signal_codec_unavailable", ) .await?; @@ -705,7 +786,7 @@ where .bind(DB::document(&context)).execute(&mut *tx).await?; Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name) - .bind(outcome).bind(command_sequence).bind(&prepared.arguments).bind(DB::document(&validation_errors)).bind(reason) + .bind(outcome).bind(command_sequence).bind(&row_arguments).bind(DB::document(&row_errors)).bind(reason) .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut *tx).await?; Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") .bind(command_sequence) @@ -802,14 +883,28 @@ where let children = Self::query("SELECT sequence FROM workflow_child_calls WHERE parent_workflow_run_id=$1 ORDER BY sequence DESC LIMIT 1") .bind(run_id).fetch_optional(&mut **tx).await?.map(|row| DB::number(&row,"sequence")).transpose()?.unwrap_or(0); let run = - Self::query("SELECT cancellation_delivery_sequence FROM workflow_runs WHERE id=$1") + Self::query("SELECT cancellation_delivery_sequence,run_number FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) .await?; + // A redriven run replays the activities before its resume step from + // reused completions, which have no execution rows. + let mut reused = 0; + if DB::number(&run, "run_number")? > 1 { + let started = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") + .bind(run_id).fetch_optional(&mut **tx).await?; + if let Some(started) = started { + let payload = DB::document_row(&started, "payload")?; + if payload["recovery_kind"] == "redrive" { + reused = payload["resume_step_sequence"].as_i64().unwrap_or(1) - 1; + } + } + } Ok(activities .max(timers) .max(wait_sequence) .max(children) + .max(reused) .max(DB::optional_number(&run, "cancellation_delivery_sequence")?.unwrap_or(0))) } @@ -1212,8 +1307,9 @@ where .bind(DB::bind_time(now())) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_worker_registrations SET status='active',workflow_command_contracts=$1 WHERE worker_id=$2 AND namespace=$3") - .bind(DB::document(&contracts)).bind(worker_id).bind(self.namespace.as_ref()).execute(&mut *tx).await?; + let status = Self::registration_status(&mut tx, self.namespace.as_ref(), queue, body["build_id"].as_str()).await?; + Self::query("UPDATE workflow_worker_registrations SET status=$1,workflow_command_contracts=$2 WHERE worker_id=$3 AND namespace=$4") + .bind(status).bind(DB::document(&contracts)).bind(worker_id).bind(self.namespace.as_ref()).execute(&mut *tx).await?; Self::record_registration( &mut tx, self.namespace.as_ref(), @@ -1228,7 +1324,7 @@ where Ok( json!({"worker_id": worker_id, "registration_token":token, "registered": true, "namespace": self.namespace.as_ref(), "task_queue": queue, "runtime": body["runtime"], "build_id": body["build_id"], "heartbeat_interval_seconds": config.worker_heartbeat_interval_seconds, - "status": "active", "capabilities": body["capabilities"], "capability_manifest": body["capability_manifest"]}), + "status": status, "capabilities": body["capabilities"], "capability_manifest": body["capability_manifest"]}), ) } @@ -1245,6 +1341,7 @@ where return Err(refuse(StatusCode::NOT_FOUND, "worker_not_found")); } self.retire_incarnation(&mut tx, worker_id).await?; + Self::revoke_worker_activity_leases(&mut tx, self.namespace.as_ref(), worker_id).await?; // Revocation expires the existing fence; later polling creates a new // attempt before work can be accepted from another worker. let recovered = DB::affected(Self::query("UPDATE workflow_tasks SET lease_expires_at=$1 WHERE lease_owner=$2 AND namespace=$3 AND status='leased' AND task_type='workflow'") @@ -1279,8 +1376,7 @@ where pages: super::history_pages::HistoryPages, default_wait: u64, ) -> Result { - text(&body, "worker_id")?; - text(&body, "task_queue")?; + self.refuse_draining_worker(text(&body, "worker_id")?, text(&body, "task_queue")?).await?; // Laravel's `nullable|integer|min:0|max:60` bounds a requested wait; // the configured default is not capped. let timeout = match body.get("timeout_seconds") { @@ -1346,16 +1442,16 @@ where let queue = text(body, "task_queue")?; // The claim refuses a removed or moved registration. let registration = Self::query( - "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + "SELECT task_queue,build_id FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", ) .bind(worker_id) .bind(self.namespace.as_ref()) .fetch_optional(&self.pool) .await?; - match registration { - Some(row) if DB::string(&row, "task_queue")? == queue => {} + let build = match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => worker_build::(&row)?, _ => return Ok(true), - } + }; // A concurrent poll with the same request ID may have claimed already; // the claim replays its receipt. if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) @@ -1372,6 +1468,7 @@ where .bind(kind) .bind(DB::bind_time(at)) .bind(DB::bind_time(at)) + .bind(build.as_deref()) .fetch_optional(&self.pool) .await? .is_some()) @@ -1425,12 +1522,14 @@ where "supported_activity_types" }, )?; + let build = worker_build::(&worker)?; let candidates = Self::query(DB::TASK_CANDIDATES_SQL) .bind(self.namespace.as_ref()) .bind(queue) .bind(kind) .bind(DB::bind_time(now())) .bind(DB::bind_time(now())) + .bind(build.as_deref()) .fetch_all(&mut *tx) .await?; for task in candidates { @@ -1626,7 +1725,10 @@ where claim["attempt_number"] = json!(attempt); claim["activity_type"] = json!(type_name); claim["idempotency_key"] = json!(activity_id); - claim["arguments"] = wire(&DB::string(&activity, "arguments")?); + // A standalone activity started without input has none. + if let Some(arguments) = DB::optional_string(&activity, "arguments")? { + claim["arguments"] = wire(&arguments); + } claim["retry_policy"] = DB::optional_document_row(&activity, "retry_policy")?.unwrap_or(Value::Null); // PHP omits the field when the activity has no deadline. @@ -1641,6 +1743,9 @@ where claim["workflow_update_id"] = json!(update_id); } claim["sticky_replay_mode"] = json!("cold_replay"); + claim["compatibility"] = json!(DB::optional_string(&task, "compatibility")? + .filter(|build| !build.trim().is_empty()) + .or(DB::optional_string(&run, "compatibility")?.filter(|build| !build.trim().is_empty()))); if let Some(cancellation) = cancellation { claim["cancellation_request"] = cancellation; } @@ -1734,8 +1839,10 @@ where .execute(&mut *tx) .await?; tx.commit().await?; + // PHP's heartbeat also reports the run's and the task's status. let mut response = json!({"task_id": task_id, "workflow_task_attempt": body["workflow_task_attempt"], - "lease_owner": body["lease_owner"], "lease_expires_at": expires, "renewed": true}); + "lease_owner": body["lease_owner"], "lease_expires_at": expires, "renewed": true, + "run_status": DB::string(&run, "status")?, "task_status": "leased", "reason": null}); if let Some(cancellation) = Self::cancellation_request(&run)? { response["cancellation_request"] = cancellation; } @@ -1848,12 +1955,8 @@ where // replay block, without a retry, until an operator repairs the run. let range_blocked = range_blocked || (!waiting && super::task_failures::blocks_replay(&body["failure"])); - if !waiting && self.namespace.as_ref() != "default" { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "development_workflow_task_retry_namespace_unqualified", - )); - } + // The task is read within the request's namespace, and its retry + // copies the run's namespace. let mut tx = self.begin().await?; let task = self.task_row(&mut tx, task_id).await?; if DB::string(&task, "task_type")? != "workflow" { @@ -2012,36 +2115,21 @@ where // Validate the entire batch before mutation. Unknown commands and // options cannot be silently accepted as unsupported durable effects. for (index, command) in commands.iter().enumerate() { + // The command normalizer's field scopes and value rules, with + // PHP's messages, before native's own refusals. + super::completion_rules::normalizer_checks(index, command)?; let command_type = text(command, "type")?; - if self.namespace.as_ref() != "default" - && !matches!( - command_type, - "complete_workflow" | "fail_workflow" | "schedule_activity" | "start_timer" | "start_child_workflow" - ) - && !(commands.len() == 1 - && matches!(command_type, "open_condition_wait" | "open_signal_wait")) - { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "development_namespace_command_unqualified", - )); - } match command_type { "fail_workflow" => { super::workflow_failures::validate(command)?; } - "complete_workflow" if index + 1 == commands.len() => { + "complete_workflow" => { reject_fields(command, &["type", "result"])?; if !external.contains_key(&index) { envelope(command, "result")?; } } - "continue_as_new" - if index + 1 == commands.len() - && commands[..index] - .iter() - .all(|prior| matches!(prior["type"].as_str(), Some("upsert_memo" | "upsert_search_attributes"))) => - { + "continue_as_new" => { super::continuations::validate(command)?; } "schedule_activity" => { @@ -2065,7 +2153,7 @@ where } super::activity_failures::retry_policy(command)?; } - "start_child_workflow" if commands.len() == 1 => { + "start_child_workflow" => { super::children::validate_child(command)?; } "start_timer" => { @@ -2128,12 +2216,12 @@ where text(command, "signal_name")?; super::wait_timeouts::timeout(command)?; } - "complete_update" if commands.len() == 1 => { + "complete_update" => { reject_fields(command, &["type", "update_id", "result"])?; text(command, "update_id")?; envelope(command, "result")?; } - "fail_update" if commands.len() == 1 => { + "fail_update" => { super::updates::update_failure(command)?; } _ => { @@ -2165,18 +2253,23 @@ where .bind(&run_id) .fetch_one(&mut *tx) .await?; + let created = DB::optional_document_row(&task, "receipt")? + .and_then(|receipt| receipt.get(COMPLETION_RECEIPT).map(|receipt| receipt["created_task_ids"].clone())) + .unwrap_or_else(|| json!([])); return Ok( json!({"task_id": task_id, "run_id": run_id, "run_status": DB::string(¤t,"status")?, "workflow_task_attempt": body["workflow_task_attempt"], - "outcome": "completed", "recorded": false, "reason": "already_completed"}), + "outcome": "completed", "recorded": false, "reason": "already_completed", "created_task_ids": created}), ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. - if let Some(command) = commands.iter().find(|command| command["type"] == "start_child_workflow") - && super::children::policies(command)?.cooperative() - { + let mut cooperative_child = false; + for command in commands.iter().filter(|command| command["type"] == "start_child_workflow") { + cooperative_child |= super::children::policies(command)?.cooperative(); + } + if cooperative_child { let mut unavailable = Vec::new(); if !protocol .and_then(|version| version.strip_prefix("1.")) @@ -2199,6 +2292,11 @@ where }); } } + // `parseCommands`: at most one terminal command, anywhere in the + // batch. It applies after every other command. + if commands.iter().filter(|command| is_terminal(command)).count() > 1 { + return Err(refuse(StatusCode::CONFLICT, "invalid_commands")); + } let run_id = DB::string(&task, "workflow_run_id")?; let run = Self::active_run(&mut tx, &run_id).await?; // `timeoutIfDeadlineExpired`: published PHP applies a run's deadlines @@ -2239,31 +2337,20 @@ where "development_selection_cancellation_unqualified", )); } - let update_id = task_payload["workflow_update_id"].as_str(); - if commands.iter().any(|command| { - matches!( - command["type"].as_str(), - Some("complete_update" | "fail_update") - ) - }) != update_id.is_some() - || update_id.is_some_and(|update_id| { - commands.len() != 1 - || if commands[0]["type"] == "fail_update" { - super::updates::update_failure(&commands[0]) - .map_or(true, |failure| failure.update_id != update_id) - } else { - commands[0]["update_id"] != update_id - } - }) - { - return Err(refuse(StatusCode::CONFLICT, "update_task_mismatch")); - } + Self::validate_update_commands(&mut tx, &run_id, &task, commands).await?; let mut sequence = Self::authored_sequence(&mut tx, &run_id).await?; + // The tasks this batch's commands create, in order, as PHP reports them. + let mut created = Vec::new(); Self::apply_task_signal(&mut tx, &run_id, task_id, &task, transition_codec).await?; - for (index, command) in commands.iter().enumerate() { + // Published parseCommands applies every other command first, whatever + // the terminal command's position in the batch. + let order = (0..commands.len()) + .filter(|index| !is_terminal(&commands[*index])) + .chain((0..commands.len()).filter(|index| is_terminal(&commands[*index]))); + for index in order { + let command = &commands[index]; if command["type"] == "fail_workflow" { - // Published parseCommands applies every nonterminal command - // first, regardless of the terminal command's batch position. + // Applied after the loop. continue; } else if matches!( command["type"].as_str(), @@ -2277,12 +2364,11 @@ where { Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1").bind(&run_id).execute(&mut *tx).await?; } else { - self.continue_run(&mut tx, &run, &task, command, sequence) - .await?; + created.push(self.continue_run(&mut tx, &run, &task, command, sequence).await?); } } else if command["type"] == "start_child_workflow" { sequence += 1; - Self::start_child(&mut tx, &run, task_id, command, sequence).await?; + created.push(Self::start_child(&mut tx, &run, task_id, command, sequence).await?); } else if command["type"] == "record_side_effect" { // `applyRecordSideEffect`: one authored sequence and its result. sequence += 1; @@ -2367,14 +2453,18 @@ where Some(recorded_at), ) .await?; - let activity_task = Self::create_task( + // PHP publishes only timer tasks a completion creates, so this + // activity task waits for a poller undispatched. + let activity_task = Self::create_task_dispatched( &mut tx, &run_id, &queue, "activity", json!({"activity_execution_id": activity_id}), + false, ) .await?; + created.push(activity_task.clone()); Self::create_activity_wait( &mut tx, &run_id, @@ -2421,6 +2511,7 @@ where timer_task_payload, ) .await?; + created.push(timer_task.clone()); Self::query("UPDATE workflow_tasks SET available_at=$1 WHERE id=$2") .bind(DB::bind_time(fire_at)) .bind(&timer_task) @@ -2563,10 +2654,16 @@ where .bind(task_id) .execute(&mut *tx) .await?; - Self::record_completion(&mut tx, task_id, &body).await?; + Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; Self::resume_undelivered_cancellation(&mut tx, &run_id, &DB::string(&run, "queue")?) .await?; Self::enqueue_next_message(&mut tx, &run_id, &DB::string(&run, "queue")?).await?; + // PHP's executor leaves every run a completed task did not close + // waiting, even when the task recorded only markers or side effects. + Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1 AND status IN ('pending','running')") + .bind(&run_id) + .execute(&mut *tx) + .await?; let current = Self::query("SELECT status FROM workflow_runs WHERE id=$1") .bind(&run_id) .fetch_one(&mut *tx) @@ -2579,7 +2676,7 @@ where tx.commit().await?; self.wake.notify_waiters(); Ok( - json!({"task_id": task_id, "run_id": run_id, "run_status": run_status, "workflow_task_attempt": body["workflow_task_attempt"], "outcome": "completed", "recorded": true, "reason": null}), + json!({"task_id": task_id, "run_id": run_id, "run_status": run_status, "workflow_task_attempt": body["workflow_task_attempt"], "outcome": "completed", "recorded": true, "reason": null, "created_task_ids": created}), ) } @@ -2786,6 +2883,7 @@ where payload["activity_attempt_id"] = json!(attempt_id); payload["attempt_number"] = json!(DB::number(&attempt, "attempt_number")?); payload["result"] = wire(&result); + payload["payload_codec"] = json!("avro"); payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, @@ -2818,15 +2916,21 @@ where .execute(&mut *tx) .await?; Self::record_completion(&mut tx, task_id, &body).await?; - let next_task_id = Self::selection_resume( - &mut tx, - &run_id, - &DB::string(&run, "queue")?, - json!({"activity_execution_id": activity_id}), - &metadata, - true, - ) - .await?; + // A standalone activity's host run closes with its result. + let closed = Self::close_standalone_host(&mut tx, &run_id, super::standalone_activities::HostClose::Completed { output: &result }).await?; + let next_task_id = if closed { + None + } else { + Self::selection_resume( + &mut tx, + &run_id, + &DB::string(&run, "queue")?, + json!({"activity_execution_id": activity_id}), + &metadata, + true, + ) + .await? + }; tx.commit().await?; self.wake.notify_waiters(); Ok( @@ -2837,6 +2941,8 @@ where // Expired root authority wins before any ordinary due timer fires. self.expire_cooperative_deadlines().await?; self.expire_activity_deadlines().await?; + // An activity task whose lease expired is settled, not re-claimed. + self.expire_activity_leases().await?; // New timers use the canonical PHP timestamp shape on SQLite and typed // UTC values elsewhere. Existing PHP data still cannot be adopted. // The read probe avoids acquiring the transition lock when idle. @@ -2987,6 +3093,10 @@ where // receipt. A later completion must refuse it, not decode SQL NULL. let receipt = DB::optional_document_row(task, "receipt")? .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_not_leased"))?; + let receipt = match receipt.get(COMPLETION_RECEIPT) { + Some(stored) => stored["request"].clone(), + None => receipt, + }; if receipt == *body { return Ok(true); } @@ -3158,22 +3268,36 @@ where queue: &str, kind: &str, payload: Value, + ) -> Result { + Self::create_task_dispatched(tx, run_id, queue, kind, payload, true).await + } + + /// A new ready task. As PHP in poll mode, a task it dispatches is stamped + /// dispatched when created; one it leaves to pollers is not. + pub(super) async fn create_task_dispatched( + tx: &mut Transaction<'_, DB>, + run_id: &str, + queue: &str, + kind: &str, + payload: Value, + dispatched: bool, ) -> Result { tx.summary_changed(run_id); let task_id = id(); // A task carries its run's priority and fairness tags, as in PHP. - let run = Self::query("SELECT namespace,priority,fairness_key,fairness_weight FROM workflow_runs WHERE id=$1") + let run = Self::query("SELECT namespace,priority,fairness_key,fairness_weight,compatibility FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) .await?; let namespace = DB::string(&run, "namespace")?; - // As PHP in poll mode, a task is dispatched to pollers when it is created. let created = now(); + let dispatched_at = dispatched.then_some(created); let fairness_key = DB::optional_string(&run, "fairness_key")?; - Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,task_type,status,payload,queue,available_at,namespace,created_at,updated_at,last_dispatched_at,last_dispatch_attempt_at,priority,fairness_key,fairness_weight) VALUES ($1,$2,$3,'ready',$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)") + Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,task_type,status,payload,queue,available_at,namespace,created_at,updated_at,last_dispatched_at,last_dispatch_attempt_at,priority,fairness_key,fairness_weight,compatibility) VALUES ($1,$2,$3,'ready',$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)") .bind(&task_id).bind(run_id).bind(kind).bind(DB::document(&payload)).bind(queue).bind(DB::bind_time(created)).bind(&namespace) - .bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)) + .bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::optional_time(dispatched_at)).bind(DB::optional_time(dispatched_at)) .bind(DB::number(&run, "priority")?).bind(fairness_key.as_deref()).bind(DB::number(&run, "fairness_weight")?) + .bind(DB::optional_string(&run, "compatibility")?.as_deref()) .execute(&mut **tx).await?; Ok(task_id) } @@ -3189,7 +3313,7 @@ where Self::append_at(tx, run_id, event_type, payload, task_id, command_id, None).await } - async fn append_at( + pub(super) async fn append_at( tx: &mut Transaction<'_, DB>, run_id: &str, event_type: &str, @@ -3379,7 +3503,7 @@ where "activity": {"id":DB::string(activity,"id")?,"idempotency_key":DB::string(activity,"id")?, "sequence":DB::number(activity,"sequence")?,"type":DB::string(activity,"activity_type")?, "class":DB::string(activity,"activity_class")?,"local_activity":false, - "activity_type": DB::string(activity,"activity_type")?, "arguments": wire(&DB::string(activity,"arguments")?), + "activity_type": DB::string(activity,"activity_type")?, "arguments": DB::optional_string(activity,"arguments")?.map(|arguments| wire(&arguments)), "retry_policy":DB::optional_document_row(activity,"retry_policy")?, "status":DB::string(activity,"status")?,"attempt_count":DB::number(activity,"attempt_count")?, "closed_at":DB::optional_instant(activity,"closed_at")?, @@ -3398,6 +3522,9 @@ where payload["activity"][field] = json!(value); } } + if payload["activity"]["arguments"].is_null() { + payload["activity"].as_object_mut().unwrap().remove("arguments"); + } if payload["activity"]["retry_policy"].is_null() { // A legacy absent snapshot stays absent in its durable history. // The activity claim may still advertise a nullable policy. @@ -3579,7 +3706,7 @@ pub(super) async fn prepare_rejected_command( // Only small server-authored diagnostics cross this conversion. // Original customer arguments remain official decoded Values, so a // JSON projection cannot round an int64 or turn bytes into strings. - // A number has no exact diagnostic form, so it is refused. + // Only an int64, such as a structural limit, is an exact number. fn diagnostic(value: Value) -> Option { Some(match value { Value::Null => Payload::Null, @@ -3594,7 +3721,7 @@ pub(super) async fn prepare_rejected_command( .map(|(key, value)| Some((key, diagnostic(value)?))) .collect::>()?, ), - Value::Number(_) => return None, + Value::Number(number) => Payload::Long(number.as_i64()?), }) } let unavailable = || refuse(StatusCode::SERVICE_UNAVAILABLE, unavailable); @@ -3735,6 +3862,19 @@ pub(super) fn validate_arguments( Ok(()) } +/// A workflow task completion's stored receipt: its request and the tasks it +/// created, so a repeat answers as the original did. Older receipts, and +/// activity receipts, are the bare request. +const COMPLETION_RECEIPT: &str = "workflow_task_completion"; + +/// PHP's `TERMINAL_TYPES`: the commands that close the run. +fn is_terminal(command: &Value) -> bool { + matches!( + command["type"].as_str(), + Some("complete_workflow" | "fail_workflow" | "continue_as_new") + ) +} + fn timer_deadline(command: &Value) -> Result> { command["delay_seconds"] .as_i64() @@ -3789,12 +3929,17 @@ fn version_marker_payload(command: &Value) -> Result { let version = integer("version")?; let min = integer("min_supported")?; let max = integer("max_supported")?; - if change_id.is_empty() || min > max || version < min || version > max { + if change_id.is_empty() { return Err(refuse( StatusCode::UNPROCESSABLE_ENTITY, "invalid_version_marker", )); } + // PHP's bridge refuses a version outside its supported range with 409 + // `invalid_commands`, recording nothing. + if min > max || version < min || version > max { + return Err(refuse(StatusCode::CONFLICT, "invalid_commands")); + } Ok(json!({"change_id":change_id,"version":version,"min_supported":min,"max_supported":max})) } diff --git a/rust/src/runtime/system_metrics.rs b/rust/src/runtime/system_metrics.rs new file mode 100644 index 000000000..9aee23d94 --- /dev/null +++ b/rust/src/runtime/system_metrics.rs @@ -0,0 +1,193 @@ +//! `GET|POST /api/system/metrics`, as published Server 2.11 serves it +//! (`SystemController::metrics`): the namespace's workflow task failures, +//! the server's projection drift, and the namespace's request admission, +//! durable-state and external payload usage, each with PHP's bounded label +//! policy. +//! +//! Native enforces no namespace request admission, durable-state quota or +//! external payload quota, and the configuration audit refuses settings that +//! would configure one. Their limits are therefore null and their rejection +//! counters zero, as PHP reports for an unconfigured server; usage is +//! measured. +use super::Result; +use super::store::now; +use serde_json::{Map, Value, json}; + +/// PHP's default `server.metrics.workflow_task_failure_type_limit`. +const WORKFLOW_TYPE_LIMIT: i64 = 20; + +const PROJECTION_TABLES: [&str; 5] = + ["run_summaries", "run_waits", "run_timeline_entries", "run_timer_entries", "run_lineage_entries"]; + +const ADMISSION_REASONS: [&str; 3] = [ + "namespace_request_rate_exhausted", + "namespace_request_concurrency_exhausted", + "namespace_request_admission_unavailable", +]; + +const PAYLOAD_REASONS: [&str; 3] = [ + "external_payload_namespace_bytes_exhausted", + "external_payload_namespace_objects_exhausted", + "external_payload_namespace_quota_unavailable", +]; + +/// `NamespaceDurableStateQuota::usage`, resource by resource, in PHP's order. +const DURABLE_STATE: [(&str, &str); 16] = [ + ("workflow_instances", "SELECT COUNT(*) FROM workflow_instances WHERE namespace=$1"), + ("workflow_runs", "SELECT COUNT(*) FROM workflow_runs WHERE namespace=$1"), + ("open_workflow_runs", "SELECT COUNT(*) FROM workflow_runs WHERE namespace=$1 AND status IN ('pending','running','waiting')"), + ("schedules", "SELECT COUNT(*) FROM workflow_schedules WHERE namespace=$1"), + ("schedule_history_events", "SELECT COUNT(*) FROM workflow_schedule_history_events WHERE namespace=$1"), + ("worker_registrations", "SELECT COUNT(*) FROM workflow_worker_registrations WHERE namespace=$1"), + ("workflow_history_events", "SELECT COUNT(*) FROM workflow_history_events WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1)"), + ("workflow_tasks", "SELECT COUNT(*) FROM workflow_tasks WHERE namespace=$1"), + ("pending_workflow_tasks", "SELECT COUNT(*) FROM workflow_tasks WHERE namespace=$1 AND status='ready'"), + ("workflow_timers", "SELECT COUNT(*) FROM workflow_run_timers WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1)"), + ("pending_workflow_timers", "SELECT COUNT(*) FROM workflow_run_timers WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND status='pending'"), + ("workflow_run_waits", "SELECT COUNT(*) FROM workflow_run_waits WHERE workflow_instance_id IN (SELECT id FROM workflow_instances WHERE namespace=$1)"), + ("open_workflow_run_waits", "SELECT COUNT(*) FROM workflow_run_waits WHERE workflow_instance_id IN (SELECT id FROM workflow_instances WHERE namespace=$1) AND status='open'"), + ("workflow_commands", "SELECT COUNT(*) FROM workflow_commands WHERE workflow_instance_id IN (SELECT id FROM workflow_instances WHERE namespace=$1)"), + ("workflow_streams", "SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE namespace=$1)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE namespace=$2)"), + ("workflow_stream_items", "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE namespace=$1)+(SELECT COUNT(*) FROM workflow_inbound_stream_items WHERE namespace=$2)"), +]; + +/// The namespace's failed workflow tasks, joined to their runs. +macro_rules! failed_tasks { + () => { + "FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.task_type='workflow' AND t.status='failed' AND t.attempt_count>0" + }; +} + +fn zeros(reasons: &[&str]) -> Value { + Value::Object(reasons.iter().map(|reason| ((*reason).to_owned(), json!(0))).collect()) +} + +fn nulls(keys: &[&str]) -> Value { + Value::Object(keys.iter().map(|key| ((*key).to_owned(), Value::Null)).collect()) +} + +store_methods! { + async fn namespace_count(&self, sql: &'static str, binds: usize) -> Result { + let mut query = Self::scalar(sql); + for _ in 0..binds { + query = query.bind(self.namespace.as_ref()); + } + Ok(query.fetch_one(&self.pool).await?) + } + + pub(crate) async fn system_metrics(&self) -> Result { + let failures = self.workflow_task_failures().await?; + let drift = self.projection_drift().await?; + let admission = json!({ + "max_requests_per_minute": null, "max_concurrent_requests": null, + "requests_this_minute": 0, "rejections_this_minute": 0, + "rejections_by_reason": zeros(&ADMISSION_REASONS), "configuration_status": "valid", + "label_cardinality_policy": {"namespace": "request_scope_not_label", "reason": "finite_three_reason_inventory"}, + }); + let durable = self.durable_state_usage().await?; + let payloads = json!({ + "max_bytes": null, "max_objects": null, + "used_bytes": self.namespace_count(DB::EXTERNAL_PAYLOAD_BYTES, 1).await?.max(0), + "used_objects": self.namespace_count("SELECT COUNT(*) FROM runtime_external_payloads WHERE namespace=$1", 1).await?, + "remaining_bytes": null, "remaining_objects": null, + "rejections_this_minute": 0, "rejections_by_reason": zeros(&PAYLOAD_REASONS), + "configuration_status": "valid", "measurement_status": "available", + "label_cardinality_policy": {"namespace": "request_scope_not_label", "reason": "finite_three_reason_inventory"}, + }); + let families = [ + ("dw_workflow_task_consecutive_failures", failures), + ("dw_projection_drift_total", drift), + ("dw_namespace_request_admission_rejections", admission), + ("dw_namespace_durable_state_usage", durable), + ("dw_runtime_external_payload_namespace_usage", payloads), + ]; + let label_sets: Map = families + .iter() + .map(|(name, family)| ((*name).to_owned(), family["label_cardinality_policy"].clone())) + .collect(); + let metrics: Map = families.into_iter().map(|(name, family)| (name.to_owned(), family)).collect(); + Ok(json!({ + "generated_at": now().to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + "namespace": self.namespace.as_ref(), + "metrics": metrics, + "cardinality": {"metric_label_sets": label_sets}, + })) + } + + /// `WorkflowTaskFailureMetrics::snapshot`: failed workflow tasks, the + /// first failure counting as one. + async fn workflow_task_failures(&self) -> Result { + // PostgreSQL's MAX keeps the integer column's width, so decode it by + // row. No COALESCE: MariaDB turns COALESCE(MAX(unsigned),0) into DECIMAL. + const MAX: &str = concat!("SELECT MAX(t.attempt_count) AS n ", failed_tasks!()); + const COUNT: &str = concat!("SELECT COUNT(t.id) ", failed_tasks!()); + const TYPES: &str = concat!("SELECT COUNT(DISTINCT r.workflow_type) ", failed_tasks!()); + const ROWS: &str = concat!( + "SELECT r.workflow_type AS workflow_type,MAX(t.attempt_count) AS max_consecutive_failures,COUNT(*) AS failed_task_count ", + failed_tasks!(), + " GROUP BY r.workflow_type ORDER BY max_consecutive_failures DESC,r.workflow_type LIMIT 20" + ); + let max = DB::optional_number(&Self::query(MAX).bind(self.namespace.as_ref()).fetch_one(&self.pool).await?, "n")? + .unwrap_or(0); + let failed = self.namespace_count(COUNT, 1).await?; + let types = self.namespace_count(TYPES, 1).await?; + let rows = Self::query(ROWS).bind(self.namespace.as_ref()).fetch_all(&self.pool).await?; + let mut series = Vec::new(); + let mut reported = 0; + for row in &rows { + let tasks = DB::number(row, "failed_task_count")?; + reported += tasks; + series.push(json!({"workflow_type": DB::string(row, "workflow_type")?, + "max_consecutive_failures": DB::number(row, "max_consecutive_failures")?, "failed_task_count": tasks})); + } + let shown = series.len() as i64; + Ok(json!({ + "max_consecutive_failures": max, "failed_task_count": failed, "workflow_type_count": types, + "workflow_type_limit": WORKFLOW_TYPE_LIMIT, "workflow_types_truncated": types > shown, + "suppressed_workflow_type_count": (types - shown).max(0), "suppressed_failed_task_count": (failed - reported).max(0), + "label_cardinality_policy": {"namespace": "request_scope_not_label", "workflow_type": { + "cardinality_class": "bounded_series", "limit": WORKFLOW_TYPE_LIMIT, + "selection": "top_by_max_consecutive_failures_then_name"}}, + "by_workflow_type": series, + })) + } + + /// `ProjectionDriftMetrics::snapshot`, server-wide. Native maintains + /// run waits with history and derives the timeline, timer and lineage + /// views from it, so only a run without its summary needs a rebuild. + async fn projection_drift(&self) -> Result { + let missing = Self::scalar("SELECT COUNT(*) FROM workflow_runs r WHERE NOT EXISTS (SELECT 1 FROM workflow_run_summaries s WHERE s.id=r.id)") + .fetch_one(&self.pool) + .await?; + let by_table: Vec = PROJECTION_TABLES + .iter() + .map(|table| json!({"table": table, "needs_rebuild": if *table == "run_summaries" { missing } else { 0 }})) + .collect(); + Ok(json!({ + "total": missing, "table_count": PROJECTION_TABLES.len(), "tables_with_drift": i64::from(missing > 0), "scope": "server", + "label_cardinality_policy": {"namespace": "server_scope_no_label", "table": { + "cardinality_class": "finite_projection_table_inventory", "values": PROJECTION_TABLES, + "selection": "fixed_projection_table_inventory"}}, + "by_table": by_table, + })) + } + + /// `NamespaceDurableStateQuota::metrics` with no configured limits. + async fn durable_state_usage(&self) -> Result { + let mut usage = Map::new(); + for (resource, sql) in DURABLE_STATE { + let binds = if sql.contains("$2") { 2 } else { 1 }; + usage.insert(resource.to_owned(), json!(self.namespace_count(sql, binds).await?)); + } + let resources: Vec<&str> = DURABLE_STATE.iter().map(|(resource, _)| *resource).collect(); + let mut reasons: Vec = resources.iter().map(|resource| format!("namespace_{resource}_exhausted")).collect(); + reasons.push("namespace_durable_state_quota_unavailable".into()); + let reasons: Vec<&str> = reasons.iter().map(String::as_str).collect(); + Ok(json!({ + "limits": nulls(&resources), "usage": usage, "remaining": nulls(&resources), + "rejections_this_minute": 0, "rejections_by_reason": zeros(&reasons), + "configuration_status": "valid", "measurement_status": "available", + "label_cardinality_policy": {"namespace": "request_scope_not_label", "reason": "finite_reason_inventory"}, + })) + } +} diff --git a/rust/src/runtime/task_queue_build_ids.rs b/rust/src/runtime/task_queue_build_ids.rs index 1b2b91132..5ab9501db 100644 --- a/rust/src/runtime/task_queue_build_ids.rs +++ b/rust/src/runtime/task_queue_build_ids.rs @@ -4,12 +4,9 @@ //! registrations grouped by trimmed build id, with each build id's pending //! workflow tasks. //! -//! Native keeps no build-id rollouts. Takeover refuses a database that holds -//! any, and native never writes one. So every build id reads as PHP reads one -//! without a rollout row: drain intent `active`, no drain, promote or rollback -//! time, and not selected for new starts, since -//! `WorkerBuildIdNewStartSelector::selectedKeyFromRollouts` selects nothing -//! from no rollouts. +//! Each build id carries its rollout row's drain intent and times, and +//! whether new starts are pinned to it. A rollout without workers or pending +//! tasks still gets a row. use super::{ Result, backend::Backend, @@ -125,11 +122,11 @@ impl Cohort { } } - /// `buildIdRolloutStatus` for a build id without a rollout. - fn rollout_status(&self) -> &'static str { + /// `buildIdRolloutStatus`. + fn rollout_status(&self, intent_draining: bool) -> &'static str { if self.active > 0 { - if self.draining > 0 { "active_with_draining" } else { "active" } - } else if self.draining > 0 { + if intent_draining || self.draining > 0 { "active_with_draining" } else { "active" } + } else if self.draining > 0 || intent_draining { "draining" } else if self.stale > 0 { "stale_only" @@ -328,18 +325,34 @@ where let stale_after = config.worker_stale_after_seconds.max(1); let mut cohorts = self.build_id_cohorts(queue, now(), stale_after).await?; let pending = self.pending_workflow_tasks(queue).await?; - // A build id with pending workflow tasks and no worker still gets a row. + let mut tx = self.begin().await?; + let rollouts = Self::rollouts(&mut tx, self.namespace.as_ref(), queue).await?; + tx.commit().await?; + let selected = super::deployments::selected_key(&rollouts).map(str::to_owned); + // A build id with pending workflow tasks and no worker still gets a + // row; one with a rollout gets its row from the rollout below. for (key, counts) in &pending { let group = if key.is_empty() { UNVERSIONED } else { key.as_str() }; - if !cohorts.iter().any(|cohort| cohort.key == group) { + if !cohorts.iter().any(|cohort| cohort.key == group) && !rollouts.iter().any(|rollout| &rollout.key == key) { cohorts.push(Cohort::new(group.to_owned(), counts.build_id.clone())); } } + let grouped: Vec = cohorts.iter().map(|cohort| cohort.key.clone()).collect(); + // A rollout no worker or pending task names is a row of its own. + for rollout in &rollouts { + let group = if rollout.key.is_empty() { UNVERSIONED } else { rollout.key.as_str() }; + if !grouped.iter().any(|key| key == group) { + let build = (!rollout.key.is_empty()).then(|| rollout.key.clone()); + cohorts.push(Cohort::new(group.to_owned(), build)); + } + } let mut entries: Vec<(i64, String, Value)> = cohorts .into_iter() .map(|cohort| { - let status = cohort.rollout_status(); let rollout_key = cohort.build_id.as_deref().unwrap_or(""); + let rollout = rollouts.iter().find(|rollout| rollout.key == rollout_key); + let intent = rollout.map_or("active", |rollout| rollout.drain_intent.as_str()); + let status = cohort.rollout_status(intent == super::deployments::DRAINING); let empty = Pending::empty(cohort.build_id.clone()); let counts = pending .iter() @@ -348,8 +361,11 @@ where let (fingerprint_count, conflicts) = cohort.fingerprint_summary(); let last_heartbeat = json_time(cohort.last_heartbeat); let order = last_heartbeat.as_str().unwrap_or("").to_owned(); - let entry = json!({"build_id":cohort.build_id,"rollout_status":status,"drain_intent":"active", - "drained_at":null,"promoted_at":null,"rolled_back_at":null,"new_start_selected":false, + let entry = json!({"build_id":cohort.build_id,"rollout_status":status,"drain_intent":intent, + "drained_at":super::deployments::zulu(rollout.and_then(|rollout| rollout.drained_at)), + "promoted_at":super::deployments::zulu(rollout.and_then(|rollout| rollout.promoted_at)), + "rolled_back_at":super::deployments::zulu(rollout.and_then(|rollout| rollout.rolled_back_at)), + "new_start_selected":selected.as_deref() == Some(rollout_key), "active_worker_count":cohort.active,"draining_worker_count":cohort.draining, "stale_worker_count":cohort.stale,"total_worker_count":cohort.total, "runtimes":php_sorted_keys(cohort.runtimes),"sdk_versions":php_sorted_keys(cohort.sdk_versions), diff --git a/rust/src/runtime/task_repair.rs b/rust/src/runtime/task_repair.rs index d82373fab..ccc46f040 100644 --- a/rust/src/runtime/task_repair.rs +++ b/rust/src/runtime/task_repair.rs @@ -18,7 +18,7 @@ use tokio::sync::Semaphore; /// `TaskRepairPolicy::readyTaskNeedsRedispatch` or `leaseExpired` for one /// task row, as of `at`. -fn needs_recovery(task: &DB::Row, at: DateTime) -> Result +pub(super) fn needs_recovery(task: &DB::Row, at: DateTime) -> Result where usize: ColumnIndex, for<'r> i64: Decode<'r, DB>, @@ -50,13 +50,29 @@ where None => claim_failed_at.is_some_and(|failed| failed <= cutoff), }); } + // `dispatchOverdue`: a task made available after its last dispatch is + // overdue from its availability. let available = DB::optional_instant(task, "available_at")?; - let reference = dispatched.or(DB::optional_instant(task, "created_at")?); + let reference = dispatched + .or(DB::optional_instant(task, "created_at")?) + .map(|reference| available.filter(|available| *available > reference).unwrap_or(reference)); Ok(available.is_none_or(|available| available <= at) && reference.is_some_and(|reference| reference <= cutoff)) } +/// `TaskRepair::replayBlocked`: a failed workflow task whose payload marks +/// its replay blocked. +pub(super) fn replay_blocked(task: &DB::Row) -> Result +where + usize: ColumnIndex, + for<'r> i64: Decode<'r, DB>, +{ + Ok(DB::string(task, "task_type")? == "workflow" + && DB::string(task, "status")? == "failed" + && DB::optional_document_row(task, "payload")?.is_some_and(|payload| payload["replay_blocked"] == true)) +} + /// `recoverableTask`'s order: earliest available, then created, then id. -fn recovery_order(task: &DB::Row) -> Result<(i64, i64, String)> +pub(super) fn recovery_order(task: &DB::Row) -> Result<(i64, i64, String)> where usize: ColumnIndex, for<'r> i64: Decode<'r, DB>, @@ -123,12 +139,6 @@ where { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_request")); } - if self.namespace.as_ref() != "default" { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "development_workflow_task_repair_namespace_unqualified", - )); - } let mut tx = self.begin().await?; let instance = Self::query("SELECT * FROM workflow_instances WHERE id=$1 AND namespace=$2") .bind(workflow_id) diff --git a/rust/src/runtime/updates.rs b/rust/src/runtime/updates.rs index f92870ba2..2521ddf90 100644 --- a/rust/src/runtime/updates.rs +++ b/rust/src/runtime/updates.rs @@ -16,8 +16,6 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::Semaphore; -const MAX_UPDATES_PER_RUN: i64 = 64; - /// What `rejectUpdateCommand` records for one refused update. struct UpdateRejection<'a, R> { workflow_id: &'a str, @@ -28,8 +26,10 @@ struct UpdateRejection<'a, R> { request_id: Option<&'a str>, context: Option, reason: &'static str, - outcome: &'static str, + outcome: Option<&'static str>, validation_errors: Value, + /// A structural ceiling keeps no arguments and records its limit. + exceeded: Option, } /// `DefaultWorkflowControlPlane::updateStatus` for a rejected update. @@ -108,7 +108,7 @@ fn update_argument_errors(contract: &Value, name: &str, values: &[crate::codec:: } Value::Object(errors) } -const UPDATE_ROW: &str = "SELECT u.*,c.workflow_type,c.message_sequence FROM workflow_updates u JOIN workflow_commands c ON c.id=u.workflow_command_id WHERE u.id=$1"; +const UPDATE_ROW: &str = "SELECT u.*,c.workflow_type,c.message_sequence,c.context AS command_context FROM workflow_updates u JOIN workflow_commands c ON c.id=u.workflow_command_id WHERE u.id=$1"; #[derive(Clone, Copy)] enum UpdateLookup { @@ -303,6 +303,8 @@ where prepared: PreparedControlArguments, context: Option, codec: Arc, + limits: super::memo_limits::StructuralLimits, + external: Option<&super::external_resolution::PreparedExternal>, ) -> Result<(StatusCode, Value)> { // PHP's update_name column holds 255 bytes, one fewer than its // validation allows; native refuses what it cannot store. @@ -330,8 +332,13 @@ where .begin() .await .map_err(|error| at_update_lookup(error, UpdateLookup::AdmissionOwnership))?; - let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1 AND namespace='default'") - .bind(workflow_id).fetch_optional(&mut *tx).await? + // The referenced object now belongs to the update; a refusal rolls + // the claim back with everything else. + if let Some(prepared) = external { + self.claim_external_in(&mut tx, &prepared.reference, &prepared.uri).await?; + } + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; let run_id = DB::string(&instance, "current_run_id")?; if selected_run.is_some_and(|selected| selected != run_id) { @@ -381,13 +388,13 @@ where let contract = DB::document_row(&started, "payload")?; // `WorkflowStub::submitUpdate`'s order: a closed run, an unknown // update, then invalid arguments. Each records a rejection. - let rejection = if !active { - Some(("run_not_active", "rejected_not_active", json!([]))) + let mut rejection = if !active { + Some(("run_not_active", Some("rejected_not_active"), json!([]))) } else if !contract["declared_updates"] .as_array() .is_some_and(|names| names.iter().any(|value| value == name)) { - Some(("unknown_update", "rejected_unknown_update", json!([]))) + Some(("unknown_update", Some("rejected_unknown_update"), json!([]))) } else if contract["declared_update_validators"] .as_array() .is_some_and(|names| names.iter().any(|value| value == name)) @@ -411,16 +418,34 @@ where .. }) => Some(( "invalid_update_arguments", - "rejected_invalid_arguments", + Some("rejected_invalid_arguments"), update_argument_errors(&contract, name, &prepared.values), )), Err(error) => return Err(error), } }; + // `guardPendingUpdates`, then `guardPayloadSize` on the + // serialized arguments: an admissible update over a structural + // ceiling is rejected durably with the limit's message. + let mut exceeded = None; + if rejection.is_none() { + if limits.pending_updates.is_some() { + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_updates WHERE workflow_run_id=$1 AND status='accepted'") + .bind(&run_id).fetch_one(&mut *tx).await + .map_err(|error| at_update_lookup(error.into(), UpdateLookup::Inventory))?; + exceeded = super::memo_limits::Exceeded::pending_updates(limits.pending_updates, pending); + } + if exceeded.is_none() { + exceeded = super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments); + } + if let Some(exceeded) = &exceeded { + rejection = Some(("structural_limit_exceeded", None, json!({"arguments":[exceeded.message]}))); + } + } if let Some((reason, outcome, validation_errors)) = rejection { let update_id = Self::reject_update( &mut tx, - codec, + codec.clone(), UpdateRejection { workflow_id, instance: &instance, @@ -432,6 +457,7 @@ where reason, outcome, validation_errors, + exceeded, }, ) .await?; @@ -442,15 +468,16 @@ where .await .map_err(|error| at_update_lookup(error.into(), UpdateLookup::ReceiptRead))?; return Ok(( - rejection_status(outcome), + rejection_status(outcome.unwrap_or_default()), // A rejected completion wait reports the timeout it // would have used, as `attemptUpdateWithArguments` does. - Self::update_result( + self.rejected_result( + codec, &update, wait_for, - false, (wait_for == "completed").then_some(timeout.unwrap_or(self.update_wait_seconds)), - )?, + ) + .await?, )); } // Published PHP admits an update whenever its run is active. It is @@ -461,18 +488,6 @@ where let open_tasks = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(&run_id).fetch_one(&mut *tx).await .map_err(|error| at_update_lookup(error.into(), UpdateLookup::Inventory))?; - let count = - Self::scalar("SELECT COUNT(*) FROM workflow_updates WHERE workflow_run_id=$1") - .bind(&run_id) - .fetch_one(&mut *tx) - .await - .map_err(|error| at_update_lookup(error.into(), UpdateLookup::Inventory))?; - if count >= MAX_UPDATES_PER_RUN { - return Err(refuse( - StatusCode::SERVICE_UNAVAILABLE, - "update_inventory_exceeds_budget", - )); - } let command_id = id(); let update_id = id(); let command_sequence = DB::number(&run, "last_command_sequence")? @@ -503,14 +518,18 @@ where Some(command) => (json!(DB::string(command, "id")?), json!(DB::number(command, "command_sequence")?)), None => (Value::Null, Value::Null), }; - Self::append(&mut tx, &run_id, "UpdateAccepted", json!({ + let mut accepted = json!({ "workflow_command_id":command_id,"update_id":update_id,"workflow_instance_id":workflow_id, "workflow_run_id":run_id,"update_name":name,"arguments":wire(&prepared.arguments), "ordering_state":if blocking.is_some() {"queued"} else {"ready"}, "queued_behind_command_id":queued_id,"queued_behind_command_sequence":queued_sequence, "queued_behind_command_type":if blocking.is_some() {json!("signal")} else {Value::Null}, - "request_id":request_id - }), None, Some(&command_id)).await + }); + // PHP adds the request id only when the request named one. + if let Some(request_id) = request_id { + accepted["request_id"] = json!(request_id); + } + Self::append(&mut tx, &run_id, "UpdateAccepted", accepted, None, Some(&command_id)).await .map_err(|error| at_update_lookup(error, UpdateLookup::HistoryAppend))?; // A ready update is delivered now when the run has no open // workflow task. Otherwise the open task's completion delivers @@ -545,7 +564,7 @@ where let outcome = DB::optional_string(&update, "outcome")?; return Ok(( rejection_status(outcome.as_deref().unwrap_or_default()), - Self::update_result(&update, wait_for, false, reported)?, + self.rejected_result(codec.clone(), &update, wait_for, reported).await?, )); } let timed_out = wait_for == "completed" @@ -586,14 +605,23 @@ where reason, outcome, validation_errors, + exceeded, } = rejection; let run_id = DB::string(run, "id")?; + let (values, diagnostics, arguments) = match &exceeded { + Some(exceeded) => ( + vec![], + exceeded.fields(), + super::queries::empty_arguments(codec.clone()).await?, + ), + None => (prepared.values.clone(), json!({}), prepared.arguments.clone()), + }; let payload = super::store::prepare_rejected_command( codec, name.to_owned(), - prepared.values.clone(), + values, validation_errors.clone(), - json!({}), + diagnostics, "update_codec_unavailable", ) .await?; @@ -613,7 +641,7 @@ where .bind(DB::document(&context)).bind(request_id).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_updates(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,update_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") .bind(&update_id).bind(&command_id).bind(workflow_id).bind(&run_id).bind(&run_id).bind(name).bind(outcome) - .bind(command_sequence).bind(&prepared.arguments).bind(DB::document(&validation_errors)).bind(reason) + .bind(command_sequence).bind(&arguments).bind(DB::document(&validation_errors)).bind(reason) .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut **tx).await?; Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") .bind(command_sequence) @@ -628,7 +656,7 @@ where // PHP drops null and empty fields from this event. let mut event = json!({"workflow_command_id":command_id,"update_id":update_id, "workflow_instance_id":workflow_id,"workflow_run_id":run_id,"update_name":name, - "arguments":wire(&prepared.arguments)}); + "arguments":wire(&arguments)}); if validation_errors.as_object().is_some_and(|errors| !errors.is_empty()) { event["validation_errors"] = validation_errors; } @@ -638,6 +666,49 @@ where Ok(update_id) } + /// A rejected update's answer. A structural rejection also reports its + /// command's limit fields, as PHP's update result does. + async fn rejected_result( + &self, + codec: Arc, + update: &DB::Row, + wait_for: &str, + timeout: Option, + ) -> Result { + let mut result = Self::update_result(update, wait_for, false, timeout)?; + if DB::optional_string(update, "rejection_reason")?.as_deref() != Some("structural_limit_exceeded") { + return Ok(result); + } + let command = Self::query("SELECT payload FROM workflow_commands WHERE id=$1") + .bind(DB::string(update, "workflow_command_id")?) + .fetch_one(&self.pool) + .await + .map_err(|error| at_update_lookup(error.into(), UpdateLookup::ReceiptRead))?; + let payload = DB::string(&command, "payload")?; + let unavailable = || refuse(StatusCode::SERVICE_UNAVAILABLE, "update_codec_unavailable"); + let permit = codec.acquire_owned().await.map_err(|_| unavailable())?; + let fields = tokio::task::spawn_blocking(move || { + let _permit = permit; + use crate::codec::Value as Payload; + let Ok(Payload::Map(command)) = crate::codec::ValueCodec::new().and_then(|codec| codec.decode(&payload)) else { + return Err(unavailable()); + }; + let mut fields = serde_json::Map::new(); + for key in ["structural_limit_kind", "structural_limit_value", "structural_limit_configured"] { + match command.get(key) { + Some(Payload::String(value)) => fields.insert(key.into(), json!(value)), + Some(Payload::Long(value)) => fields.insert(key.into(), json!(value)), + _ => None, + }; + } + Ok(fields) + }) + .await + .map_err(|_| unavailable())??; + result.as_object_mut().expect("update result").extend(fields); + Ok(result) + } + fn update_result( update: &DB::Row, wait_for: &str, @@ -671,7 +742,7 @@ where let validation_errors = DB::optional_document_row(update, "validation_errors")? .filter(|errors| !errors.is_null()) .unwrap_or_else(|| json!([])); - let response = json!({ + let mut response = json!({ "accepted":!rejected,"workflow_id":DB::string(update,"workflow_instance_id")?,"run_id":DB::string(update,"workflow_run_id")?, "resolved_run_id":DB::string(update,"resolved_workflow_run_id")?,"requested_run_id":null,"target_scope":"instance", "workflow_type":DB::string(update,"workflow_type")?,"command_id":DB::string(update,"workflow_command_id")?, @@ -683,8 +754,21 @@ where "accepted_at":DB::optional_instant(update,"accepted_at")?,"applied_at":DB::optional_instant(update,"applied_at")?, "closed_at":DB::optional_instant(update,"closed_at")?,"reason":if rejected {json!(rejection_reason)} else {Value::Null}, "rejection_reason":rejection_reason,"validation_errors":validation_errors, - "wait_for":wait_for,"wait_timed_out":timed_out,"wait_timeout_seconds":timeout + "wait_for":wait_for,"wait_timed_out":timed_out,"wait_timeout_seconds":timeout, + "rejected_at":DB::optional_instant(update,"rejected_at")? }); + // The requester recorded on the command, as `principalPayload`. + let context = DB::optional_document_row(update, "command_context")?.unwrap_or(Value::Null); + let principal: serde_json::Map = ["type", "id", "label"] + .iter() + .filter_map(|field| { + context["principal"][*field].as_str().filter(|value| !value.trim().is_empty()) + .map(|value| ((*field).to_owned(), json!(value))) + }) + .collect(); + if !principal.is_empty() { + response["principal"] = Value::Object(principal); + } let name = response["update_name"].as_str().unwrap_or_default().to_owned(); Ok(super::control_plane::Operation::Update(&name).response(response)) } @@ -875,6 +959,47 @@ where Ok(()) } + /// `validateUpdateCommands`: each update command in a completion names a + /// different update accepted on this run, the task's own update when the + /// task carries one, with no earlier signal still to apply. A task may + /// complete its update together with other commands, or leave it for a + /// later task, and any task may complete an accepted update. + pub(super) async fn validate_update_commands( + tx: &mut Transaction<'_, DB>, + run_id: &str, + task: &DB::Row, + commands: &[Value], + ) -> Result<()> { + let payload = DB::document_row(task, "payload")?; + let own = payload["workflow_update_id"].as_str().filter(|id| !id.is_empty()); + let carried = payload["workflow_command_id"].as_str().filter(|id| !id.is_empty()); + let invalid = || refuse(StatusCode::CONFLICT, "invalid_commands"); + let mut seen = std::collections::BTreeSet::new(); + for command in commands + .iter() + .filter(|command| matches!(command["type"].as_str(), Some("complete_update" | "fail_update"))) + { + let update_id = command["update_id"].as_str().filter(|id| !id.is_empty()).ok_or_else(invalid)?; + if !seen.insert(update_id) || own.is_some_and(|own| own != update_id) { + return Err(invalid()); + } + let update = Self::query("SELECT status,command_sequence FROM workflow_updates WHERE id=$1 AND workflow_run_id=$2") + .bind(update_id) + .bind(run_id) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(invalid)?; + if DB::string(&update, "status")? != "accepted" + || Self::blocking_signal(tx, run_id, DB::optional_number(&update, "command_sequence")?, carried) + .await? + .is_some() + { + return Err(invalid()); + } + } + Ok(()) + } + /// `UpdateCommandGate::blockingSignal`: the earliest accepted signal not /// yet applied, optionally only one before `before` and other than /// `ignored`. diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index ff2431d29..07c77151a 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -221,7 +221,7 @@ fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, conver /// Published PHP features native does not implement yet. A source holding any /// of their rows is refused, rather than taken over without them. Derived /// projections and PHP's own queue and poll bookkeeping are left in place. -const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 6] = [ ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), ( "upgrade_source_message_streams_unsupported", @@ -239,7 +239,6 @@ const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ ), ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), - ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), ( "upgrade_source_search_attribute_definitions_unsupported", &["search_attribute_definitions"], diff --git a/rust/src/runtime/visibility.rs b/rust/src/runtime/visibility.rs index 53b19157b..a9346da7c 100644 --- a/rust/src/runtime/visibility.rs +++ b/rust/src/runtime/visibility.rs @@ -29,6 +29,13 @@ pub(super) struct VisibilityFilter { offset: i64, } +impl VisibilityFilter { + /// The run list for one workflow type, as a status bucket and page. + pub(super) fn of_type(workflow_type: &str, status: Option, limit: i64, offset: i64) -> Self { + Self { workflow_type: Some(workflow_type.to_owned()), status, pattern: None, visibility: None, limit, offset } + } +} + /// Candidate runs read per page while a structured query filters them. const QUERY_SCAN_PAGE: i64 = 500; @@ -118,18 +125,18 @@ fn validation(field: &'static str, message: &'static str) -> RuntimeError { } } -pub(super) const VISIBILITY_PAGE: &str = "SELECT r.id,r.workflow_instance_id,r.workflow_type,r.queue,r.status,r.compatibility,r.started_at,r.closed_at,i.business_key FROM workflow_runs r JOIN workflow_instances i ON i.id=r.workflow_instance_id WHERE r.namespace=$1 AND i.namespace=$2 AND ($3 IS NULL OR r.workflow_type=$4) AND ($5 IS NULL OR CASE WHEN r.status IN ('pending','running','waiting') THEN 'running' WHEN r.status='completed' THEN 'completed' ELSE 'failed' END=$6 OR r.status=$7) AND ($8 IS NULL OR r.workflow_instance_id LIKE $9 OR i.business_key LIKE $10) ORDER BY r.started_at DESC,r.id DESC LIMIT $11 OFFSET $12"; +pub(super) const VISIBILITY_PAGE: &str = "SELECT r.id,r.workflow_instance_id,r.workflow_type,r.queue,r.status,r.compatibility,r.started_at,r.closed_at,COALESCE(r.business_key,i.business_key) AS business_key FROM workflow_runs r JOIN workflow_instances i ON i.id=r.workflow_instance_id WHERE r.namespace=$1 AND i.namespace=$2 AND ($3 IS NULL OR r.workflow_type=$4) AND ($5 IS NULL OR CASE WHEN r.status IN ('pending','running','waiting') THEN 'running' WHEN r.status='completed' THEN 'completed' ELSE 'failed' END=$6 OR r.status=$7) AND ($8 IS NULL OR r.workflow_instance_id LIKE $9 OR COALESCE(r.business_key,i.business_key) LIKE $10) ORDER BY r.started_at DESC,r.id DESC LIMIT $11 OFFSET $12"; // Preserve decimal micro/nanosecond ordering across PHP SQL UTC and earlier // native RFC3339 UTC strings without SQLite floating-point date conversion. pub(super) const SQLITE_VISIBILITY_PAGE: &str = r#" WITH runs AS ( - SELECT r.id,r.workflow_instance_id,r.workflow_type,r.queue,r.status,r.compatibility,r.started_at,r.closed_at,i.business_key, + SELECT r.id,r.workflow_instance_id,r.workflow_type,r.queue,r.status,r.compatibility,r.started_at,r.closed_at,COALESCE(r.business_key,i.business_key) AS business_key, replace(replace(replace(r.started_at,'T',' '),'Z',''),'+00:00','') AS _dw_started_utc FROM workflow_runs r JOIN workflow_instances i ON i.id=r.workflow_instance_id WHERE r.namespace=$1 AND i.namespace=$2 AND ($3 IS NULL OR r.workflow_type=$4) AND ($5 IS NULL OR CASE WHEN r.status IN ('pending','running','waiting') THEN 'running' WHEN r.status='completed' THEN 'completed' ELSE 'failed' END=$6 OR r.status=$7) - AND ($8 IS NULL OR r.workflow_instance_id LIKE $9 OR i.business_key LIKE $10) + AND ($8 IS NULL OR r.workflow_instance_id LIKE $9 OR COALESCE(r.business_key,i.business_key) LIKE $10) ) SELECT *,rtrim(rtrim(CASE WHEN length(_dw_started_utc)=19 THEN _dw_started_utc||'.0' ELSE _dw_started_utc END,'0'),'.') AS _dw_started_order FROM runs ORDER BY _dw_started_order DESC,id DESC LIMIT $11 OFFSET $12 @@ -147,6 +154,11 @@ where for<'r> i64: Decode<'r, DB>, usize: ColumnIndex, { + /// Up to `limit` runs from the filter's offset, in the list's order. + pub(super) async fn visibility_rows(&self, filter: &VisibilityFilter, limit: i64) -> Result> { + self.visibility_page(filter, limit, filter.offset).await + } + async fn visibility_page(&self, filter: &VisibilityFilter, limit: i64, offset: i64) -> Result> { // PHP's index filters cancelled and terminated runs by their exact // status and the other values by status bucket. diff --git a/rust/src/runtime/worker_incarnations.rs b/rust/src/runtime/worker_incarnations.rs index 45480584f..2a83a3117 100644 --- a/rust/src/runtime/worker_incarnations.rs +++ b/rust/src/runtime/worker_incarnations.rs @@ -223,6 +223,7 @@ where } Self::query("UPDATE workflow_worker_registrations SET status='superseded' WHERE namespace=$1 AND worker_id=$2") .bind(self.namespace.as_ref()).bind(worker_id).execute(&mut *tx).await?; + Self::revoke_worker_activity_leases(&mut tx, self.namespace.as_ref(), worker_id).await?; let tasks = Self::query("SELECT * FROM workflow_tasks WHERE namespace=$1 AND lease_owner=$2 AND task_type='workflow' AND status='leased' ORDER BY available_at,id") .bind(self.namespace.as_ref()).bind(worker_id).fetch_all(&mut *tx).await?; let recovered = i64::try_from(tasks.len()) diff --git a/rust/src/runtime/worker_management.rs b/rust/src/runtime/worker_management.rs index 32d5418a1..198bc8772 100644 --- a/rust/src/runtime/worker_management.rs +++ b/rust/src/runtime/worker_management.rs @@ -466,7 +466,7 @@ where let metrics = process_metrics(&body)?; let interval = bounded(&body, "heartbeat_interval_seconds", 1, 3600)?; let mut tx = self.begin().await?; - let Some(worker) = Self::query("SELECT max_concurrent_workflow_tasks,max_concurrent_activity_tasks,max_concurrent_worker_sessions,process_metrics FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") + let Some(worker) = Self::query("SELECT task_queue,build_id,status,max_concurrent_workflow_tasks,max_concurrent_activity_tasks,max_concurrent_worker_sessions,process_metrics FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2") .bind(worker_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? else { return Err(RuntimeError::Protocol { @@ -480,8 +480,15 @@ where DB::number(&worker, "max_concurrent_activity_tasks")?, DB::number(&worker, "max_concurrent_worker_sessions")?, ]; - Self::query("UPDATE workflow_worker_registrations SET last_heartbeat_at=$1,updated_at=$2 WHERE worker_id=$3 AND namespace=$4") - .bind(DB::bind_time(now())).bind(DB::bind_time(now())).bind(worker_id).bind(self.namespace.as_ref()) + // A heartbeat keeps a draining build's worker draining, and a fenced + // registration fenced. + let status = match DB::optional_string(&worker, "status")?.as_deref() { + Some("superseded") => "superseded", + _ => Self::registration_status(&mut tx, self.namespace.as_ref(), &DB::string(&worker, "task_queue")?, + DB::optional_string(&worker, "build_id")?.as_deref()).await?, + }; + Self::query("UPDATE workflow_worker_registrations SET last_heartbeat_at=$1,updated_at=$2,status=$3 WHERE worker_id=$4 AND namespace=$5") + .bind(DB::bind_time(now())).bind(DB::bind_time(now())).bind(status).bind(worker_id).bind(self.namespace.as_ref()) .execute(&mut *tx).await?; // Only the slots, metrics and cadence the heartbeat reports change. let reported = task_slots(&body, capacity)?; diff --git a/rust/src/runtime/workflow_failures.rs b/rust/src/runtime/workflow_failures.rs index 4d601cae5..f9cfc0c5f 100644 --- a/rust/src/runtime/workflow_failures.rs +++ b/rust/src/runtime/workflow_failures.rs @@ -29,19 +29,8 @@ fn string(value: &Value) -> Option<&str> { } pub(super) fn validate_batch(commands: &[Value]) -> Result<()> { - if commands - .iter() - .filter(|command| { - matches!( - command["type"].as_str(), - Some("complete_workflow" | "fail_workflow" | "continue_as_new") - ) - }) - .count() - > 1 - { - return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_commands")); - } + // A second terminal command is the bridge's 409, after the ownership + // checks (`Store::complete_workflow`). // PHP parseCommands removes terminal commands before group validation. let members: std::borrow::Cow<'_, [Value]> = if commands .iter() diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 2f60a1c43..c3ce551e4 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -133,6 +133,12 @@ mod version_markers; #[path = "execution/activity_lease_renewal.rs"] mod activity_lease_renewal; +#[path = "execution/system_metrics.rs"] +mod system_metrics; + +#[path = "execution/prometheus_metrics.rs"] +mod prometheus_metrics; + #[path = "execution/activity_deadlines.rs"] mod activity_deadlines; @@ -152,6 +158,32 @@ mod workflow_repair; mod history_retention; #[path = "execution/repair_status.rs"] mod repair_status; +#[path = "execution/repair_pass.rs"] +mod repair_pass; +#[path = "execution/activity_lease_expiry.rs"] +mod activity_lease_expiry; +#[path = "execution/structural_limits.rs"] +mod structural_limits; +#[path = "execution/standalone_activities.rs"] +mod standalone_activities; +#[path = "execution/archive.rs"] +mod archive; +#[path = "execution/redrive.rs"] +mod redrive; +#[path = "execution/build_pins.rs"] +mod build_pins; +#[path = "execution/deployments.rs"] +mod deployments; +#[path = "execution/priority_fairness.rs"] +mod priority_fairness; +#[path = "execution/completion_rules.rs"] +mod completion_rules; +#[path = "execution/command_batches.rs"] +mod command_batches; +#[path = "execution/update_routing.rs"] +mod update_routing; +#[path = "execution/command_payloads.rs"] +mod command_payloads; #[path = "execution/activity_timeout_pass.rs"] mod activity_timeout_pass; #[path = "execution/start_request.rs"] @@ -1888,32 +1920,21 @@ async fn updates_preserve_original_receipts_and_fence_recovered_tasks_across_poo StatusCode::CONFLICT ); } + // Published PHP's `validateUpdateCommands` refuses another update in a + // task that carries its own. let mut wrong_update = original.clone(); wrong_update["commands"][0]["update_id"] = json!("another-update"); - assert_eq!( - request(&a, "POST", &complete_path, wrong_update).await.1["reason"], - "update_task_mismatch" - ); + let refused = request(&a, "POST", &complete_path, wrong_update).await; + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + assert_eq!(refused.1["reason"], "invalid_commands", "{}", refused.1); + assert_eq!(refused.1["recorded"], false, "{}", refused.1); + // Published PHP refuses an undecodable result before it applies the + // update, naming the field. let mut malformed = original.clone(); malformed["commands"][0]["result"]["blob"] = json!("not-avro"); - assert_eq!( - request(&a, "POST", &complete_path, malformed).await.1["reason"], - "invalid_update_result" - ); - assert_eq!( - request( - &a, - "POST", - &complete_path, - completion( - &old, - json!([{"type":"complete_workflow","result":envelope(Payload::Null)}]) - ) - ) - .await - .1["reason"], - "update_task_mismatch" - ); + let refused = request(&a, "POST", &complete_path, malformed).await.1; + assert_eq!(refused["reason"], "validation_failed", "{refused}"); + assert!(refused["errors"]["commands.0.result.blob"].is_array(), "{refused}"); assert_eq!( request(&a, "GET", &history_path, Value::Null).await.1, before @@ -2328,7 +2349,7 @@ async fn failed_updates_preserve_terminal_identity_and_parent_continuation_after } #[tokio::test] -async fn updates_bound_durable_inventory_and_preserve_accepted_work_after_wait_timeout() { +async fn updates_have_no_lifetime_cap_and_preserve_accepted_work_after_wait_timeout() { let database = TestDatabase::new().await; let runtime = database.open().await.unwrap(); let app = router(runtime.clone()); @@ -2373,15 +2394,6 @@ async fn updates_bound_durable_inventory_and_preserve_accepted_work_after_wait_t .count(), 64 ); - let over_budget = request(&app, "POST", path, json!({ - "input":envelope(Payload::Array(vec![Payload::Long(65)])),"wait_for":"accepted","request_id":"over-budget" - })).await; - assert_eq!(over_budget.0, StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(over_budget.1["reason"], "update_inventory_exceeds_budget"); - assert_eq!( - request(&app, "GET", &history_path, Value::Null).await.1, - before - ); let duplicate = request(&app, "POST", path, json!({ "input":envelope(Payload::Array(vec![Payload::Long(999)])),"request_id":"bounded:0","wait_for":"completed" })).await; @@ -2391,6 +2403,13 @@ async fn updates_bound_durable_inventory_and_preserve_accepted_work_after_wait_t request(&app, "GET", &history_path, Value::Null).await.1, before ); + // Published PHP bounds only pending updates (DW_V2_LIMIT_PENDING_UPDATES), + // not how many a run receives over its life. + let more = request(&app, "POST", path, json!({ + "input":envelope(Payload::Array(vec![Payload::Long(65)])),"wait_for":"accepted","request_id":"beyond-64" + })).await; + assert_eq!(more.0, StatusCode::ACCEPTED, "{}", more.1); + assert_eq!(more.1["update_status"], "accepted", "{}", more.1); runtime.close().await; database.remove().await; } @@ -4434,14 +4453,15 @@ async fn reported_activity_failure_commits_one_persistent_retry_and_fences_old_o let workflow = poll(&app_a, "worker", "workflow").await; let arguments = envelope(Payload::Array(vec![Payload::Long(9007199254740993)])); let policy = json!({"max_attempts":2,"backoff_seconds":[1],"non_retryable_error_types":[]}); + // A digit string such as "2" is an integer to PHP, and PHP ignores an + // unknown retry policy key. A backoff too large for a timestamp is + // native's own refusal; PHP accepts it. for invalid in [ json!({"max_attempts":0}), - json!({"max_attempts":"2"}), json!({"max_attempts":1.5}), json!({"max_attempts":2,"backoff_seconds":[-1]}), json!({"max_attempts":2,"backoff_seconds":[i64::MAX]}), json!({"max_attempts":2,"non_retryable_error_types":[17]}), - json!({"max_attempts":2,"unsupported":true}), ] { let refused = finish_task(&app_a,&workflow,json!([{"type":"schedule_activity","activity_type":"retry-activity","arguments":arguments,"retry_policy":invalid}])).await; assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); @@ -4652,11 +4672,13 @@ async fn terminal_activity_failures_resume_once_and_preserve_original_failure() false, true, ), + // Laravel's TrimStrings removes the padding before PHP reads the + // policy, so the padded type still matches. ( json!({"max_attempts":2,"backoff_seconds":[0],"non_retryable_error_types":["\u{2003}RuntimeException\u{2003}"]}), - 2, - false, + 1, false, + true, ), ( json!({"max_attempts":3,"backoff_seconds":[1],"non_retryable_error_types":[]}), @@ -4904,12 +4926,18 @@ async fn activity_commit_wakes_replay_and_stale_activity_attempt_is_refused() { StatusCode::UNPROCESSABLE_ENTITY ); assert!(poll(&app, "worker", "activity").await.is_null()); - for delay in [ - json!(-1), - json!(1.5), - json!("1"), - Value::Null, - json!(i64::MAX), + // Published PHP 2.11.1's answers: its transport rules, then its command + // normalizer. A delay too large for a timestamp is native's own refusal; + // PHP wraps it into the past and fires the timer at once. + let normalizer = "Start timer commands require a non-negative integer delay_seconds."; + for (delay, reason, message) in [ + (json!(-1), "validation_failed", Some("The commands.1.delay_seconds field must be at least 0.")), + (json!(1.5), "validation_failed", Some("The commands.1.delay_seconds field must be an integer.")), + (json!("05"), "validation_failed", Some("The commands.1.delay_seconds field must be an integer.")), + (Value::Null, "validation_failed", Some(normalizer)), + (json!(true), "validation_failed", Some(normalizer)), + (json!("+5"), "validation_failed", Some(normalizer)), + (json!(i64::MAX), "invalid_timer_delay", None), ] { let outcome = request( &app, @@ -4922,7 +4950,10 @@ async fn activity_commit_wakes_replay_and_stale_activity_attempt_is_refused() { ) .await; assert_eq!(outcome.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", outcome.1); - assert_eq!(outcome.1["reason"], "invalid_timer_delay"); + assert_eq!(outcome.1["reason"], reason, "{}", outcome.1); + if let Some(message) = message { + assert_eq!(outcome.1["errors"]["commands.1.delay_seconds"], json!([message]), "{}", outcome.1); + } assert!(poll(&app, "worker", "activity").await.is_null()); } assert_eq!( diff --git a/rust/tests/execution/activity_deadlines.rs b/rust/tests/execution/activity_deadlines.rs index b5dd52bc3..ea2d4d3a9 100644 --- a/rust/tests/execution/activity_deadlines.rs +++ b/rust/tests/execution/activity_deadlines.rs @@ -314,7 +314,8 @@ async fn timeout_retry_preserves_full_snapshot_and_global_deadline_across_restar let mut scheduled = command("start_to_close_timeout", 2); scheduled["schedule_to_close_timeout"] = json!(30); scheduled["schedule_to_start_timeout"] = json!(10); - scheduled["heartbeat_timeout"] = json!(5); + // PHP refuses a heartbeat timeout longer than start_to_close. + scheduled["heartbeat_timeout"] = json!(1); assert_eq!( finish_task(&app, &workflow, json!([scheduled])).await.0, StatusCode::OK diff --git a/rust/tests/execution/activity_lease_expiry.rs b/rust/tests/execution/activity_lease_expiry.rs new file mode 100644 index 000000000..4b12b874a --- /dev/null +++ b/rust/tests/execution/activity_lease_expiry.rs @@ -0,0 +1,196 @@ +use super::*; + +const LONG_AGO: &str = "'2020-01-02 03:04:05.000000'"; + +/// Counts rows with a query built for this test. +async fn count(database: &TestDatabase, sql: String) -> i64 { + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await +} + +/// Starts a run whose workflow task schedules `act` with `retry_policy`, and +/// claims the activity. Returns the start response and the claimed task. +async fn claimed(app: &Router, workflow: &str, retry_policy: Value) -> (Value, Value) { + let started = start(app, workflow).await; + let task = poll(app, "worker", "workflow").await; + let mut command = json!({"type":"schedule_activity","activity_type":"act","arguments":envelope(Payload::Array(vec![]))}); + if !retry_policy.is_null() { + command["retry_policy"] = retry_policy; + } + assert_eq!(finish_task(app, &task, json!([command])).await.0, StatusCode::OK); + let activity = poll(app, "worker", "activity").await; + assert!(activity["task_id"].is_string(), "{activity}"); + (started, activity) +} + +/// The run's history once an event of `kind` is recorded. Native's scheduler +/// settles an expired lease within a few of its ticks. +async fn history_with(app: &Router, started: &Value, kind: &str) -> Vec { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; + let events = history.as_array().unwrap().clone(); + if events.iter().any(|event| event["event_type"] == kind) { + return events; + } + assert!(tokio::time::Instant::now() < deadline, "no {kind}: {history}"); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } +} + +async fn expire(database: &TestDatabase, task: &Value) { + let task_id = task["task_id"].as_str().unwrap(); + database + .execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_tasks SET lease_expires_at={LONG_AGO} WHERE id='{task_id}'"))) + .await; +} + +// Published PHP never hands out an activity task again after its lease +// expires. The repair loop closes the attempt as expired, and the activity's +// retry policy schedules the next attempt with the lease as its reason. +#[tokio::test] +async fn an_expired_activity_lease_consumes_the_retry_policy_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + let (started, first) = claimed(&app, "lease-retry", json!({"max_attempts":2,"backoff_seconds":[0]})).await; + let first_id = first["task_id"].as_str().unwrap(); + expire(&database, &first).await; + // A poll racing the repair loop never re-claims the expired task. + let racing = poll(&app, "worker", "activity").await; + assert_ne!(racing["task_id"].as_str(), Some(first_id), "{racing}"); + + let events = history_with(&app, &started, "ActivityRetryScheduled").await; + let retry = events.iter().find(|event| event["event_type"] == "ActivityRetryScheduled").unwrap(); + let payload = &retry["payload"]; + assert_eq!( + (payload["retry_reason"].clone(), payload["failure_reason"].clone(), payload["lease_owner"].clone()), + (json!("lease_expired"), json!("lease_expired"), json!("worker")), + "{retry}" + ); + assert_eq!(payload["lease_expires_at"], "2020-01-02T03:04:05.000000Z", "{retry}"); + assert!(payload.get("timeout_kind").is_none(), "{retry}"); + assert_eq!(payload["message"], "Activity act lost its original task lease before the attempt settled."); + assert_eq!(payload["retry_after_attempt"], 1); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE id='{first_id}' AND status='cancelled' AND last_error='lease_expired'")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM activity_attempts WHERE workflow_task_id='{first_id}' AND status='expired' AND closed_at IS NOT NULL")).await, + 1 + ); + + // The retry is a new task and the second attempt. + let second = if racing["task_id"].is_string() { racing } else { poll(&app, "worker", "activity").await }; + assert!(second["task_id"].is_string(), "{second}"); + assert_ne!(second["task_id"].as_str(), Some(first_id)); + assert_eq!(second["attempt_number"], 2, "{second}"); + let started_events = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let kinds: Vec<_> = started_events.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect(); + let retried_at = kinds.iter().position(|kind| kind == "ActivityRetryScheduled").unwrap(); + assert_eq!(kinds.iter().filter(|kind| *kind == "ActivityStarted").count(), 2, "{kinds:?}"); + assert!(kinds.iter().rposition(|kind| kind == "ActivityStarted").unwrap() > retried_at, "{kinds:?}"); +} + +// With no attempt left, or when the policy names the lease exception as +// non-retryable, the lost lease fails the activity: an internal failure, not +// a timeout, and the workflow runs again to see it. +#[tokio::test] +async fn an_expired_activity_lease_without_retries_fails_the_activity_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + let non_retryable = json!({"max_attempts":3,"non_retryable_error_types":["Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException"]}); + for (workflow, policy, flag) in [("lease-once", Value::Null, false), ("lease-non-retryable", non_retryable, true)] { + let (started, task) = claimed(&app, workflow, policy).await; + expire(&database, &task).await; + let events = history_with(&app, &started, "ActivityFailed").await; + assert!(!events.iter().any(|event| event["event_type"] == "ActivityTimedOut" || event["event_type"] == "ActivityRetryScheduled")); + let failed = events.iter().find(|event| event["event_type"] == "ActivityFailed").unwrap(); + let payload = &failed["payload"]; + assert_eq!( + [&payload["exception_class"], &payload["failure_category"], &payload["non_retryable"], &payload["failure_reason"]], + [&json!("Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException"), &json!("internal"), &json!(flag), &json!("lease_expired")], + "{failed}" + ); + assert!(payload.get("timeout_kind").is_none() && payload.get("close_deadline_at").is_none(), "{failed}"); + let run = started["run_id"].as_str().unwrap(); + let expected = if flag { "TRUE" } else { "FALSE" }; + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_failures WHERE workflow_run_id='{run}' AND propagation_kind='activity' AND failure_category='internal' AND non_retryable={expected}")).await, + 1 + ); + // The workflow sees the failure and finishes. + let resumed = poll(&app, "worker", "workflow").await; + assert!(resumed["task_id"].is_string(), "{resumed}"); + let complete = json!({"type":"complete_workflow","result":envelope(Payload::Long(1))}); + assert_eq!(finish_task(&app, &resumed, json!([complete])).await.0, StatusCode::OK); + } +} + +fn event<'a>(events: &'a [Value], kind: &str) -> &'a Value { + events.iter().find(|event| event["event_type"] == kind).unwrap_or_else(|| panic!("no {kind}: {events:?}")) +} + +// Published PHP settles a deregistered worker's activity leases in the same +// request: the attempt closes as expired, and the activity's retry policy +// records a retry with the worker's departure as its reason. +#[tokio::test] +async fn deregistration_revokes_the_workers_activity_leases_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + let (started, task) = claimed(&app, "revoke-retry", json!({"max_attempts":2,"backoff_seconds":[0]})).await; + let (status, body) = request(&app, "DELETE", "/api/worker/registrations/worker", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["outcome"].clone(), body["recovered_workflow_task_count"].clone()), (json!("deregistered"), json!(0))); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let events = history.as_array().unwrap(); + let retry = &event(events, "ActivityRetryScheduled")["payload"]; + assert_eq!( + (retry["retry_reason"].clone(), retry["failure_reason"].clone(), retry["lease_owner"].clone()), + (json!("worker_deregistered"), json!("worker_deregistered"), json!("worker")), + "{retry}" + ); + assert!(retry["lease_revoked_at"].is_string() && retry["lease_expires_at"].is_string(), "{retry}"); + assert!(retry["lease_revoked_at"].as_str() < retry["lease_expires_at"].as_str(), "{retry}"); + assert!(retry.get("timeout_kind").is_none(), "{retry}"); + let task_id = task["task_id"].as_str().unwrap(); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE id='{task_id}' AND status='cancelled' AND last_error='worker_deregistered'")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM activity_attempts WHERE workflow_task_id='{task_id}' AND status='expired'")).await, + 1 + ); + // A new registration claims the retry as attempt two. + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + let second = poll(&app, "worker", "activity").await; + assert_eq!(second["attempt_number"], 2, "{second}"); +} + +// The fenced deregistration revokes them too. Without a retry policy the +// departure fails the activity. +#[tokio::test] +async fn fenced_deregistration_fails_an_activity_without_retries_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, registered) = request(&app, "POST", "/api/worker/register", json!({"worker_id":"worker","task_queue":"test", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":["act"]})).await; + assert_eq!(status, StatusCode::CREATED, "{registered}"); + let (started, _) = claimed(&app, "revoke-once", Value::Null).await; + let (status, body) = request(&app, "POST", "/api/worker/registrations/worker/deregister", + json!({"registration_token": registered["registration_token"]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let events = history.as_array().unwrap(); + let failed = &event(events, "ActivityFailed")["payload"]; + assert_eq!( + (failed["failure_reason"].clone(), failed["exception_class"].clone(), failed["non_retryable"].clone()), + (json!("worker_deregistered"), json!("Workflow\\V2\\Exceptions\\ActivityLeaseExpiredException"), json!(false)), + "{failed}" + ); + assert!(failed["lease_revoked_at"].is_string(), "{failed}"); + assert!(!events.iter().any(|event| event["event_type"] == "ActivityRetryScheduled")); +} diff --git a/rust/tests/execution/activity_lease_renewal.rs b/rust/tests/execution/activity_lease_renewal.rs index 85849e2b0..ec9bd84c1 100644 --- a/rust/tests/execution/activity_lease_renewal.rs +++ b/rust/tests/execution/activity_lease_renewal.rs @@ -161,13 +161,14 @@ async fn renewal_cannot_revive_an_expired_or_closed_run() { &body["run_closed_reason"], &body["attempt_status"], &body["lease_renewed"]], [&json!(false), &json!(true), &json!("run_terminated"), &json!("terminated"), &json!("terminated"), &json!("cancelled"), &json!(false)]); - // Last: an expired lease is redelivered as a new attempt. + // Last: an expired lease cannot be renewed. The scheduler may already + // have settled it, as PHP's repair loop does, closing the attempt. let (_, _, activity) = claim(&app, "lease-expired").await; database.execute("UPDATE workflow_tasks SET lease_expires_at='2020-01-02 03:04:05.123456' WHERE task_type='activity'").await; let (code, expired) = renew(&app, &activity, own(&activity)).await; assert_eq!(code, StatusCode::CONFLICT, "{expired}"); - assert_eq!([&expired["reason"], &expired["lease_renewed"], &expired["can_continue"]], - [&json!("lease_expired"), &json!(false), &json!(false)]); + assert!(matches!(expired["reason"].as_str(), Some("lease_expired" | "attempt_closed")), "{expired}"); + assert_eq!([&expired["lease_renewed"], &expired["can_continue"]], [&json!(false), &json!(false)]); let status = request_protocol(&app, "POST", &path(&activity, "status"), own(&activity), "1.20").await; assert_eq!(status.1["can_continue"], false, "an expired lease stays expired: {}", status.1); } diff --git a/rust/tests/execution/archive.rs b/rust/tests/execution/archive.rs new file mode 100644 index 000000000..97763e304 --- /dev/null +++ b/rust/tests/execution/archive.rs @@ -0,0 +1,63 @@ +use super::*; + +fn kinds(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +// Published PHP 2.11.1's archive: a closed run records an archive command, +// ArchiveRequested and, the first time, WorkflowArchived. An open run, a +// historical run, an unknown workflow and invalid fields are refused. +#[tokio::test] +async fn archive_records_php_s_command_and_events() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!(["echo"]), json!([])).await; + let started = start(&app, "arc").await; + let (status, open) = request(&app, "POST", "/api/workflows/arc/archive", json!({"reason":"early"})).await; + assert_eq!(status, StatusCode::CONFLICT, "{open}"); + for (field, expected) in [("outcome", json!("rejected_run_not_closed")), ("command_status", json!("rejected")), + ("reason", json!("run_not_closed")), ("rejection_reason", json!("run_not_closed")), ("accepted", json!(false))] { + assert_eq!(open[field], expected, "{field}: {open}"); + } + + let task = poll(&app, "worker", "workflow").await; + let done = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(done.0, StatusCode::OK, "{}", done.1); + + let (status, invalid) = request(&app, "POST", "/api/workflows/arc/archive", json!({"reason":"x".repeat(1001),"request_id":5})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{invalid}"); + assert_eq!(invalid["errors"], json!({"reason":["The reason field must not be greater than 1000 characters."], + "request_id":["The request id field must be a string."]})); + assert_eq!(invalid["workflow_id"], "arc"); + + let (status, archived) = request(&app, "POST", "/api/workflows/arc/archive", json!({"reason":" cleanup "})).await; + assert_eq!(status, StatusCode::OK, "{archived}"); + assert_eq!((archived["outcome"].clone(), archived["reason"].clone(), archived["command_status"].clone()), + (json!("archived"), json!("cleanup"), json!("accepted")), "{archived}"); + assert!(archived.get("accepted").is_none(), "{archived}"); + let path = format!("/api/workflows/arc/runs/{}/archive", started["run_id"].as_str().unwrap()); + let (status, again) = request(&app, "POST", &path, json!({})).await; + assert_eq!(status, StatusCode::OK, "{again}"); + assert_eq!((again["outcome"].clone(), again["reason"].clone()), (json!("archive_not_needed"), Value::Null), "{again}"); + + let history = run_history(&app, &json!("arc"), &started["run_id"]).await; + let tail: Vec<_> = kinds(&history).into_iter().filter(|kind| kind.contains("Archive")).collect(); + assert_eq!(tail, ["ArchiveRequested", "WorkflowArchived", "ArchiveRequested"]); + let archived_event = history.as_array().unwrap().iter().find(|event| event["event_type"] == "WorkflowArchived").unwrap(); + assert_eq!(archived_event["payload"]["archive_command_id"], archived["command_id"]); + assert_eq!(archived_event["payload"]["reason"], "cleanup"); + let last = history.as_array().unwrap().last().unwrap(); + assert!(last["payload"].get("reason").is_none(), "{last}"); + let (_, described) = request(&app, "GET", "/api/workflows/arc", Value::Null).await; + assert_eq!(described["actions"]["can_archive"], false, "{described}"); + + let (status, historical) = request(&app, "POST", "/api/workflows/arc/runs/nope/archive", json!({})).await; + assert_eq!(status, StatusCode::CONFLICT, "{historical}"); + assert_eq!(historical["reason"], "historical_run_command_rejected"); + let (status, missing) = request(&app, "POST", "/api/workflows/nope/archive", json!({})).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{missing}"); + assert_eq!(missing["reason"], "instance_not_found"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/build_pins.rs b/rust/tests/execution/build_pins.rs new file mode 100644 index 000000000..6b09b4e6d --- /dev/null +++ b/rust/tests/execution/build_pins.rs @@ -0,0 +1,128 @@ +use super::*; + +async fn register_build(app: &Router, worker: &str, build: Option<&str>, types: Value) { + let (status, body) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":"test", + "runtime":"php","build_id":build,"supported_workflow_types":types,"supported_activity_types":["a"]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn start_type(app: &Router, workflow: &str, workflow_type: &str) -> Value { + let (status, body) = request(app, "POST", "/api/workflows", json!({"workflow_id":workflow,"workflow_type":workflow_type, + "task_queue":"test","input":envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + body +} + +async fn pin(app: &Router, workflow: &str) -> Value { + let (status, described) = request(app, "GET", &format!("/api/workflows/{workflow}"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + described["compatibility"].clone() +} + +fn schedule_activity() -> Value { + json!([{"type":"schedule_activity","activity_type":"a","arguments":envelope(Payload::Array(vec![]))}]) +} + +// Published PHP 2.11.1's `WorkflowStartVersionPin` pins a new run to the one +// build among the type's live workers, else the queue's, and its pollers +// offer a pinned task only to that build's workers. Native routes an +// unpinned activity to any worker, as it does an unpinned workflow task; +// Workflow #908 records where PHP strands it instead. +#[tokio::test] +async fn runs_are_pinned_to_their_start_cohort_s_build_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_build(&app, "v1-worker", Some("v1"), json!(["echo", "solo"])).await; + let first = start_type(&app, "pinned", "echo").await; + assert_eq!(pin(&app, "pinned").await, "v1"); + let history = run_history(&app, &json!("pinned"), &first["run_id"]).await; + let started = history.as_array().unwrap().iter().find(|event| event["event_type"] == "WorkflowStarted").unwrap(); + assert_eq!(started["payload"]["compatibility"], "v1", "{started}"); + + // With two builds serving the type, a start has no cohort and no pin; a + // type only one build serves is still pinned to it. + register_build(&app, "v2-worker", Some(" v2 "), json!(["echo"])).await; + let open = start_type(&app, "open", "echo").await; + assert_eq!(pin(&app, "open").await, Value::Null); + let history = run_history(&app, &json!("open"), &open["run_id"]).await; + assert!(history[1]["payload"].get("compatibility").is_none(), "{}", history[1]); + start_type(&app, "solo", "solo").await; + assert_eq!(pin(&app, "solo").await, "v1"); + + // The v2 worker skips both v1 runs and takes the unpinned one. + let task = poll(&app, "v2-worker", "workflow").await; + assert_eq!((task["workflow_id"].clone(), task["compatibility"].clone()), (json!("open"), Value::Null), "{task}"); + assert_eq!(poll(&app, "v2-worker", "workflow").await, Value::Null); + let open_task = task; + let task = poll(&app, "v1-worker", "workflow").await; + assert_eq!((task["workflow_id"].clone(), task["compatibility"].clone()), (json!("pinned"), json!("v1")), "{task}"); + let (status, body) = finish_task(&app, &task, schedule_activity()).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + // A pinned run's activity goes only to its build. + assert_eq!(poll(&app, "v2-worker", "activity").await, Value::Null); + let activity = poll(&app, "v1-worker", "activity").await; + assert_eq!(activity["workflow_id"], "pinned", "{activity}"); + + // An unpinned run's activity goes to any worker, here the one that + // scheduled it. + let (status, body) = finish_task(&app, &open_task, schedule_activity()).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let activity = poll(&app, "v2-worker", "activity").await; + assert_eq!(activity["workflow_id"], "open", "{activity}"); + + // An unversioned worker takes only unpinned work. + register_build(&app, "unversioned", None, json!(["echo", "solo"])).await; + assert_eq!(poll(&app, "unversioned", "workflow").await, Value::Null); + start_type(&app, "mixed", "echo").await; + assert_eq!(pin(&app, "mixed").await, Value::Null); + let task = poll(&app, "unversioned", "workflow").await; + assert_eq!(task["workflow_id"], "mixed", "{task}"); + runtime.close().await; + database.remove().await; +} + +// A child keeps its parent's pin, and a child of an unpinned parent is +// unpinned, whichever worker started it, as on published 2.11.1. +#[tokio::test] +async fn children_keep_their_parent_s_build_pin() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_build(&app, "v1-worker", Some("v1"), json!(["echo", "child"])).await; + register_build(&app, "v2-worker", Some("v2"), json!(["echo", "child"])).await; + start_type(&app, "parent", "echo").await; + assert_eq!(pin(&app, "parent").await, Value::Null); + let task = poll(&app, "v2-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"start_child_workflow","workflow_type":"child", + "arguments":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &json!("parent"), &task["run_id"]).await; + let scheduled = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ChildWorkflowScheduled").unwrap(); + let child = scheduled["payload"]["child_workflow_instance_id"].as_str().unwrap().to_owned(); + assert_eq!(pin(&app, &child).await, Value::Null); + let task = poll(&app, "v1-worker", "workflow").await; + assert_eq!(task["workflow_id"], json!(child), "{task}"); + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + // A parent only v1 serves is pinned to v1, and so is its child. + register_build(&app, "v1-worker", Some("v1"), json!(["echo", "child", "solo"])).await; + start_type(&app, "pinned-parent", "solo").await; + assert_eq!(pin(&app, "pinned-parent").await, "v1"); + let mut task = poll(&app, "v1-worker", "workflow").await; + while task["workflow_id"] != "pinned-parent" { + finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Array(vec![]))}])).await; + task = poll(&app, "v1-worker", "workflow").await; + } + let (status, body) = finish_task(&app, &task, json!([{"type":"start_child_workflow","workflow_type":"child", + "arguments":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &json!("pinned-parent"), &task["run_id"]).await; + let scheduled = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ChildWorkflowScheduled").unwrap(); + let child = scheduled["payload"]["child_workflow_instance_id"].as_str().unwrap().to_owned(); + assert_eq!(pin(&app, &child).await, "v1"); + assert_eq!(poll(&app, "v2-worker", "workflow").await, Value::Null); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/child_policies.rs b/rust/tests/execution/child_policies.rs index d764f436f..93fdb46df 100644 --- a/rust/tests/execution/child_policies.rs +++ b/rust/tests/execution/child_policies.rs @@ -33,16 +33,18 @@ async fn child_policies_are_validated_and_recorded_like_php() { let app = router(database.open().await.unwrap()); let run = start(&app, "policy-parent").await; let task = cooperative_parent(&app).await; - for (field, value, message) in [ - ("cancellation_policy", json!("never"), "The selected commands.0.cancellation_policy is invalid."), - ("cancellation_policy", json!(1), "The commands.0.cancellation_policy field must be a string."), + for (field, value, messages) in [ + ("cancellation_policy", json!("never"), json!(["The selected commands.0.cancellation_policy is invalid."])), + // Laravel reports both rules an integer fails. + ("cancellation_policy", json!(1), json!(["The commands.0.cancellation_policy field must be a string.", + "The selected commands.0.cancellation_policy is invalid."])), ("parent_close_policy", json!("orphan"), - "The parent_close_policy must be one of: abandon, request_cancel, request_cancellation, terminate."), + json!(["The parent_close_policy must be one of: abandon, request_cancel, request_cancellation, terminate."])), ] { let (status, refused) = finish_cooperative(&app, &task, json!([with(json!({field: value}))])).await; assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); assert_eq!(refused["reason"], "validation_failed", "{refused}"); - assert_eq!(refused["errors"][format!("commands.0.{field}")], json!([message]), "{refused}"); + assert_eq!(refused["errors"][format!("commands.0.{field}")], messages, "{refused}"); } // Laravel trims the request, so padded values are accepted. diff --git a/rust/tests/execution/command_batches.rs b/rust/tests/execution/command_batches.rs new file mode 100644 index 000000000..4606b3d38 --- /dev/null +++ b/rust/tests/execution/command_batches.rs @@ -0,0 +1,295 @@ +use super::*; + +// Each expectation below is what published Server 2.11.1 answered for the +// same completion (`DefaultWorkflowTaskBridge::complete`). + +fn kinds(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +fn first<'a>(history: &'a Value, kind: &str) -> &'a Value { + history.as_array().unwrap().iter().find(|event| event["event_type"] == kind).unwrap_or_else(|| panic!("no {kind}")) +} + +fn all<'a>(history: &'a Value, kind: &str) -> Vec<&'a Value> { + history.as_array().unwrap().iter().filter(|event| event["event_type"] == kind).collect() +} + +async fn task_for(app: &Router, worker: &str, workflow: &str) -> Value { + for _ in 0..20 { + let task = poll(app, worker, "workflow").await; + if task["workflow_id"] == workflow { + return task; + } + } + panic!("no workflow task for {workflow}"); +} + +async fn current(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +fn activity() -> Value { + json!({"type":"schedule_activity","activity_type":"a","arguments":envelope(Payload::Array(vec![]))}) +} + +fn child() -> Value { + json!({"type":"start_child_workflow","workflow_type":"echo","arguments":envelope(Payload::Array(vec![]))}) +} + +async fn complete_activity(app: &Router, worker: &str) { + let task = poll(app, worker, "activity").await; + let (status, body) = request(app, "POST", &format!("/api/worker/activity-tasks/{}/complete", task["task_id"].as_str().unwrap()), + json!({"lease_owner":task["lease_owner"],"activity_attempt_id":task["activity_attempt_id"],"result":envelope(Payload::Null)})).await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + +// `parseCommands` takes one terminal command anywhere in a batch and applies +// it after every other command. Two terminal commands are refused before +// anything is recorded. +#[tokio::test] +async fn a_terminal_command_applies_after_the_rest_of_its_batch_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["a"])).await; + + let started = start(&app, "terminal-first").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}, + {"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "completed", "{body}"); + // PHP reports the tasks the batch's commands created, and its protocol. + assert_eq!(body["created_task_ids"].as_array().map(Vec::len), Some(1), "{body}"); + assert_eq!(body["protocol_version"], "1.20", "{body}"); + let order = kinds(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + let timer = order.iter().position(|kind| kind == "TimerScheduled").expect("the timer is recorded"); + assert!(order.iter().position(|kind| kind == "WorkflowCompleted").expect("the run completes") > timer, "{order:?}"); + + let started = start(&app, "fail-first").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"fail_workflow","message":"boom"}, activity()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "failed", "{body}"); + let order = kinds(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + let scheduled = order.iter().position(|kind| kind == "ActivityScheduled").expect("the activity is recorded"); + assert!(order.iter().position(|kind| kind == "WorkflowFailed").unwrap() > scheduled, "{order:?}"); + + let started = start(&app, "two-terminals").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}, + {"type":"fail_workflow","message":"x"}])).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "invalid_commands", "{body}"); + assert_eq!(body["outcome"], "completed", "{body}"); + assert_eq!(body["recorded"], false, "{body}"); + assert!(body["run_id"].is_null(), "{body}"); + assert_eq!(run_history(&app, &started["workflow_id"], &started["run_id"]).await, before); +} + +// A task may start several children alongside timers and activities, and a +// child's result wakes its parent even while another resume task waits. +#[tokio::test] +async fn children_start_alongside_other_open_work_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["a"])).await; + let started = start(&app, "mixed-parent").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([child(), child(), {"type":"start_timer","delay_seconds":3600}, activity()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "waiting", "{body}"); + assert_eq!(body["created_task_ids"].as_array().map(Vec::len), Some(4), "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let children: Vec = all(&history, "ChildRunStarted").iter() + .map(|event| event["payload"]["child_workflow_instance_id"].as_str().unwrap().to_owned()).collect(); + assert_eq!(children.len(), 2, "{history}"); + assert_eq!(all(&history, "TimerScheduled").len(), 1); + assert_eq!(all(&history, "ActivityScheduled").len(), 1); + + for id in &children { + let task = task_for(&app, "worker", id).await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(2))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + } + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert_eq!(all(&history, "ChildRunCompleted").len(), 2, "{history}"); + let resumed = task_for(&app, "worker", "mixed-parent").await; + assert!(resumed["task_id"].is_string(), "{resumed}"); +} + +// Continue-as-new leaves the predecessor's open activities, timers and +// commands applied before it in the same batch where they are, and closes it. +#[tokio::test] +async fn continue_as_new_leaves_open_work_with_the_predecessor_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["a"])).await; + let started = start(&app, "continue-open").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":3600}, activity(), activity()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + complete_activity(&app, "worker").await; + let resumed = task_for(&app, "worker", "continue-open").await; + let (status, body) = finish_task(&app, &resumed, json!([activity(), {"type":"continue_as_new"}, + {"type":"start_timer","delay_seconds":60}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "completed", "{body}"); + // The activity's and timer's tasks, then the continued run's first task. + let created = body["created_task_ids"].as_array().unwrap(); + assert_eq!(created.len(), 3, "{body}"); + let order = kinds(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + assert_eq!(order.last().map(String::as_str), Some("WorkflowContinuedAsNew"), "{order:?}"); + assert_eq!(order.iter().filter(|kind| *kind == "ActivityScheduled").count(), 3, "{order:?}"); + assert_eq!(order.iter().filter(|kind| *kind == "TimerScheduled").count(), 2, "{order:?}"); + let described = current(&app, "continue-open").await; + assert_ne!(described["run_id"], started["run_id"], "{described}"); + assert_eq!(described["status"], "pending", "{described}"); + let next = task_for(&app, "worker", "continue-open").await; + assert_eq!(next["run_id"], described["run_id"], "{next}"); + assert_eq!(next["task_id"], created[2], "{next}"); +} + +// A child that continues as new stays its parent's child: the parent records +// `ChildRunStarted` for the new run under the same call and a new link, and +// the new run's result resumes the parent. +#[tokio::test] +async fn a_child_that_continues_as_new_stays_its_parents_child_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!([])).await; + let parent = start(&app, "continuing-parent").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([child()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let first_start = first(&history, "ChildRunStarted")["payload"].clone(); + let kid = first_start["child_workflow_instance_id"].as_str().unwrap().to_owned(); + + let task = task_for(&app, "worker", &kid).await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let continued = current(&app, &kid).await; + assert_ne!(continued["run_id"], first_start["child_workflow_run_id"], "{continued}"); + let kid_history = run_history(&app, &json!(kid), &continued["run_id"]).await; + let started = &first(&kid_history, "WorkflowStarted")["payload"]; + assert_eq!(started["parent_workflow_run_id"], parent["run_id"], "{started}"); + assert_eq!(started["parent_sequence"], first_start["sequence"], "{started}"); + assert_eq!(started["child_call_id"], first_start["child_call_id"], "{started}"); + assert_eq!(started["continued_from_run_id"], first_start["child_workflow_run_id"], "{started}"); + + let history = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let starts = all(&history, "ChildRunStarted"); + assert_eq!(starts.len(), 2, "{history}"); + let second_start = &starts[1]["payload"]; + assert_eq!(second_start["child_workflow_run_id"], continued["run_id"], "{second_start}"); + assert_eq!(second_start["child_run_number"], 2, "{second_start}"); + assert_eq!(second_start["child_call_id"], first_start["child_call_id"], "{second_start}"); + assert_ne!(second_start["workflow_link_id"], first_start["workflow_link_id"], "{second_start}"); + + let task = task_for(&app, "worker", &kid).await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(3))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let completed = &first(&history, "ChildRunCompleted")["payload"]; + assert_eq!(completed["child_workflow_run_id"], continued["run_id"], "{completed}"); + assert_eq!(completed["workflow_link_id"], second_start["workflow_link_id"], "{completed}"); + assert_eq!(completed["child_call_id"], first_start["child_call_id"], "{completed}"); + assert_eq!(completed["child_run_number"], 2, "{completed}"); + let resumed = task_for(&app, "worker", "continuing-parent").await; + assert!(resumed["task_id"].is_string(), "{resumed}"); +} + +// A child that continues as new into another workflow type still resolves +// its parent's call, which keeps the type it started with. +#[tokio::test] +async fn a_child_continued_into_another_type_resolves_its_parent_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo", "other"]), json!([])).await; + let parent = start(&app, "retyped-parent").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([child()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let kid = first(&history, "ChildRunStarted")["payload"]["child_workflow_instance_id"].as_str().unwrap().to_owned(); + let task = task_for(&app, "worker", &kid).await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new","workflow_type":"other"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = task_for(&app, "worker", &kid).await; + assert_eq!(task["workflow_type"], "other", "{task}"); + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(4))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &parent["workflow_id"], &parent["run_id"]).await; + let completed = &first(&history, "ChildRunCompleted")["payload"]; + assert_eq!(completed["child_workflow_type"], "other", "{completed}"); + let resumed = task_for(&app, "worker", "retyped-parent").await; + assert!(resumed["task_id"].is_string(), "{resumed}"); +} + +// `validateUpdateCommands`: an update completes alongside other commands, and +// an update task may leave its update for a later task. +#[tokio::test] +async fn an_update_completes_alongside_other_commands_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + update_worker(&app, "worker").await; + let started = start(&app, "update-batch").await; + open_update_wait(&app, "worker").await; + let (status, body) = request(&app, "POST", "/api/workflows/update-batch/update/set", + json!({"input":[1],"wait_for":"accepted"})).await; + assert!(status.is_success(), "{body}"); + let update = body["update_id"].clone(); + + let task = poll(&app, "worker", "workflow").await; + assert_eq!(task["workflow_update_id"], update, "{task}"); + // The update task leaves its update open. + let (status, body) = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let (status, body) = request(&app, "POST", "/api/workflows/update-batch/signal/finish", + json!({"input":envelope(Payload::Array(vec![Payload::Long(1)]))})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "worker", "workflow").await; + assert!(task["task_id"].is_string(), "{task}"); + let (status, body) = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":60}, + {"type":"complete_update","update_id":update,"result":envelope(Payload::Long(5))}, + {"type":"start_timer","delay_seconds":120}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let order = kinds(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + let completed = order.iter().position(|kind| kind == "UpdateCompleted").expect("the update completes"); + assert!(order[completed..].iter().any(|kind| kind == "TimerScheduled"), "{order:?}"); +} + +// `applyContinueAsNew` takes a new workflow type, which also becomes the +// class, and a new queue from the command. +#[tokio::test] +async fn continue_as_new_takes_a_new_type_and_queue_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo", "other"]), json!([])).await; + let (status, body) = request(&app, "POST", "/api/worker/register", json!({"worker_id":"far","task_queue":"elsewhere", + "runtime":"php","supported_workflow_types":["echo","other"],"supported_activity_types":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + start(&app, "rerouted").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new","workflow_type":"other"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let described = current(&app, "rerouted").await; + assert_eq!(described["workflow_type"], "other", "{described}"); + let next = task_for(&app, "worker", "rerouted").await; + assert_eq!(next["workflow_type"], "other", "{next}"); + let history = run_history(&app, &json!("rerouted"), &described["run_id"]).await; + let started = &first(&history, "WorkflowStarted")["payload"]; + assert_eq!(started["workflow_class"], "other", "{started}"); + + let (status, body) = finish_task(&app, &next, json!([{"type":"continue_as_new","queue":"elsewhere"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (status, polled) = request(&app, "POST", "/api/worker/workflow-tasks/poll", + json!({"worker_id":"far","task_queue":"elsewhere","timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{polled}"); + assert_eq!(polled["task"]["workflow_id"], "rerouted", "{polled}"); +} diff --git a/rust/tests/execution/command_context.rs b/rust/tests/execution/command_context.rs index 4749dfa06..de9e836e3 100644 --- a/rust/tests/execution/command_context.rs +++ b/rust/tests/execution/command_context.rs @@ -250,6 +250,12 @@ async fn cluster_info_publishes_php_s_principal_attribution_contract() { "principal_type":"attacker","actor":"mallory","user":"mallory"})); assert_eq!(guards["request_header_values"]["X-Workflow-Principal-Id"], "mallory"); assert_eq!(guards["request_headers"].as_array().unwrap().len(), 11); + // The PHP SDK reads the service catalog only when this is advertised. + let catalog = &cluster["service_catalog_read_contract"]; + assert_eq!((catalog["schema"].clone(), catalog["version"].clone()), + (json!("durable-workflow.v2.service-catalog-read.contract"), json!(1))); + assert_eq!(catalog["wire_surface"]["calls"], "/api/service-catalog/calls"); + assert_eq!((catalog["pagination"]["default_per_page"].clone(), catalog["pagination"]["max_per_page"].clone()), (json!(50), json!(100))); runtime.close().await; database.remove().await; } diff --git a/rust/tests/execution/command_payloads.rs b/rust/tests/execution/command_payloads.rs new file mode 100644 index 000000000..624ca7a43 --- /dev/null +++ b/rust/tests/execution/command_payloads.rs @@ -0,0 +1,43 @@ +use super::*; + +const UNDECODABLE: &str = "The payload must contain one complete Avro Value encoded with the official codec. JSON is only the HTTP document transport."; + +// Published PHP 2.11.1 refuses a workflow task completion whose command +// payloads are not complete Avro values, naming the field, before it applies +// any command; the task stays leased to its worker. +#[tokio::test] +async fn completion_payloads_must_decode_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "payload-worker", json!(["echo"]), json!(["a"])).await; + let started = start(&app, "payloads").await; + let task = poll(&app, "payload-worker", "workflow").await; + let cases = [ + (json!([{"type":"complete_workflow","result":{"codec":"avro","blob":"AAAA"}}]), "commands.0.result.blob", UNDECODABLE), + (json!([{"type":"record_side_effect","result":"AAAA"}]), "commands.0.result", UNDECODABLE), + (json!([{"type":"schedule_activity","activity_type":"a","arguments":envelope(Payload::Array(vec![]))}, + {"type":"schedule_activity","activity_type":"a","arguments":{"codec":"avro","blob":"AAAA"}}]), "commands.1.arguments.blob", UNDECODABLE), + (json!([{"type":"complete_workflow","result":[1, 2]}]), "commands.0.result", + "Workflow task command field [result] must be a string or a payload envelope when provided."), + (json!([{"type":"complete_workflow","result":{"codec":"","blob":"AAAA"}}]), "commands.0.result.codec", + "The payload envelope codec must be a non-empty string."), + (json!([{"type":"complete_workflow","result":{"codec":"avro","blob":5}}]), "commands.0.result.blob", + "The payload envelope blob must be a string."), + ]; + for (commands, field, message) in cases { + let (status, refused) = finish_task(&app, &task, commands.clone()).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{commands}: {refused}"); + assert_eq!((refused["reason"].clone(), refused["errors"][field].clone()), (json!("validation_failed"), json!([message])), "{commands}: {refused}"); + } + let (status, refused) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":{"codec":"json","blob":"[]"}}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert!(refused["errors"]["commands.0.result.codec"][0].as_str().unwrap().starts_with("unsupported_payload_codec:"), "{refused}"); + // Nothing was recorded; the same task still completes. + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert_eq!(history.as_array().unwrap().len(), 2, "{history}"); + let (status, done) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}])).await; + assert_eq!((status, done["run_status"].clone()), (StatusCode::OK, json!("completed")), "{done}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/completion_rules.rs b/rust/tests/execution/completion_rules.rs new file mode 100644 index 000000000..10660971f --- /dev/null +++ b/rust/tests/execution/completion_rules.rs @@ -0,0 +1,129 @@ +use super::*; + +fn path(task: &Value) -> String { + format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()) +} + +/// The refusal's first message and its field errors. +async fn refused(app: &Router, task: &Value, body: Value) -> (String, Value) { + let (status, response) = request(app, "POST", &path(task), body).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{response}"); + assert_eq!(response["reason"], "validation_failed", "{response}"); + assert_eq!(response["protocol_version"], "1.20", "{response}"); + (response["message"].as_str().unwrap().to_owned(), response["errors"].clone()) +} + +// Published PHP 2.11.1 checks a workflow task completion against +// `WorkflowTaskCompletionValidation::rules` before any guard, ownership +// included, and turns digit strings in its integer fields into integers. Each +// expectation below is what the published image answered. +#[tokio::test] +async fn completions_follow_php_s_transport_rules() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!(["echo"]), json!(["a"])).await; + start(&app, "rules").await; + let task = poll(&app, "worker", "workflow").await; + let timer = json!({"type":"start_timer","delay_seconds":3600}); + + // The rules come before the lease owner is checked. + let (message, _) = refused(&app, &task, json!({"lease_owner":"someone-else", + "workflow_task_attempt":task["workflow_task_attempt"],"commands":[{"type":5}]})).await; + assert_eq!(message, "The commands.0.type field must be a string."); + let (message, errors) = refused(&app, &task, json!({"workflow_task_attempt":"x","commands":[timer]})).await; + assert_eq!(message, "The lease owner field is required. (and 1 more error)"); + assert_eq!(errors["workflow_task_attempt"], json!(["The workflow task attempt field must be an integer."])); + for (commands, expected) in [ + (json!([]), "The commands field is required."), + (json!("x"), "The commands field must be an array."), + (json!(["x"]), "The commands.0.type field is required."), + (json!([{"type":"schedule_activity","activity_type":"a","retry_policy":{"max_attempts":0,"backoff_seconds":[1,"x",-1], + "non_retryable_error_types":[5]}}]), "The commands.0.retry_policy.max_attempts field must be at least 1. (and 3 more errors)"), + (json!([{"type":"complete_workflow","principal_subject":"x","principal_roles":null}]), + "The commands.0.principal_subject field is prohibited."), + (json!([{"type":"start_timer","delay_seconds":1,"cancellation_cleanup":{"scope_id":"a"}}]), + "The commands.0.cancellation_cleanup.request_id field is required when commands.0.cancellation_cleanup is present. (and 1 more error)"), + (json!([{"type":"schedule_activity","activity_type":"a","execution_mode":"remote"}]), + "The selected commands.0.execution_mode is invalid."), + (json!([{"type":"start_timer","delay_seconds":"x"},{"type":"complete_workflow","message":[]}]), + "The commands.0.delay_seconds field must be an integer. (and 1 more error)"), + ] { + let (message, _) = refused(&app, &task, completion(&task, commands)).await; + assert_eq!(message, expected); + } + let (message, errors) = refused(&app, &task, json!({"lease_owner":task["lease_owner"], + "workflow_task_attempt":task["workflow_task_attempt"],"commands":[timer],"sticky_cache":{"worker_id":"w"}})).await; + assert_eq!(message, "The sticky cache.workflow id field is required when sticky cache is present. (and 3 more errors)"); + assert_eq!(errors["sticky_cache.ttl_seconds"], json!(["The sticky cache.ttl seconds field is required when sticky cache is present."])); + + // Digit strings are integers once the rules pass. + let (status, body) = request(&app, "POST", &path(&task), json!({"lease_owner":task["lease_owner"], + "workflow_task_attempt":task["workflow_task_attempt"].to_string(),"commands":[{"type":"start_timer","delay_seconds":"7"}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + runtime.close().await; + database.remove().await; +} + +// PHP's command normalizer takes a bare string payload as the Avro blob +// itself, and continue-as-new arguments that are plain values as no +// arguments, so the next run keeps the current input. +#[tokio::test] +async fn bare_payload_strings_and_plain_continue_as_new_arguments_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!(["echo"]), json!(["a"])).await; + let blob = envelope(Payload::Array(vec![Payload::String("x".into())]))["blob"].clone(); + + start(&app, "bare-result").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":blob}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (_, described) = request(&app, "GET", "/api/workflows/bare-result", Value::Null).await; + assert_eq!(described["output_envelope"], json!({"codec":"avro","blob":blob}), "{described}"); + + start(&app, "bare-arguments").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, + json!([{"type":"schedule_activity","activity_type":"a","arguments":blob}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let activity = poll(&app, "worker", "activity").await; + assert_eq!(activity["arguments"], json!({"codec":"avro","blob":blob}), "{activity}"); + + let started = start(&app, "plain-continue").await; + let (_, original) = request(&app, "GET", "/api/workflows/plain-continue", Value::Null).await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new","arguments":[1,"two"]}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (_, continued) = request(&app, "GET", "/api/workflows/plain-continue", Value::Null).await; + assert_ne!(continued["run_id"], started["run_id"], "{continued}"); + assert_eq!(continued["input_envelope"], original["input_envelope"], "{continued}"); + let task = poll(&app, "worker", "workflow").await; + let (status, refusal) = finish_task(&app, &task, json!([{"type":"continue_as_new","arguments":5}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refusal}"); + assert_eq!(refusal["errors"]["commands.0.arguments"], + json!(["Workflow task command field [arguments] must be a string or a payload envelope when provided."])); + runtime.close().await; + database.remove().await; +} + +// PHP's executor leaves a run that a completed task did not close waiting, +// even when the task recorded only a version marker. +#[tokio::test] +async fn a_task_that_records_only_markers_leaves_the_run_waiting_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!(["echo"]), json!([])).await; + start(&app, "marker-only").await; + let task = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &task, + json!([{"type":"record_version_marker","change_id":"only","version":1,"min_supported":1,"max_supported":1}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "waiting", "{body}"); + let (_, described) = request(&app, "GET", "/api/workflows/marker-only", Value::Null).await; + assert_eq!(described["status"], "waiting", "{described}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/continuations.rs b/rust/tests/execution/continuations.rs index 56f47c722..cd5101aab 100644 --- a/rust/tests/execution/continuations.rs +++ b/rust/tests/execution/continuations.rs @@ -51,52 +51,8 @@ async fn continuation_inherits_committed_upserts_before_successor_commands() { {"type":"continue_as_new"} ]), ); - let initial = request( - &app, - "GET", - "/api/workflows/continuation-authored-memos", - Value::Null, - ) - .await - .1; - let mut refused_routing = body.clone(); - refused_routing["commands"][1]["queue"] = json!("different"); - let refused = request(&app, "POST", &path, refused_routing).await; - assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!(refused.1["reason"], "continue_as_new_routing_unqualified"); - assert_eq!( - request( - &app, - "GET", - "/api/workflows/continuation-authored-memos", - Value::Null - ) - .await - .1, - initial, - "rejected continuation rolls back its preceding memo mutation" - ); - let wrong_order = completion( - &task, - json!([ - {"type":"continue_as_new"}, body["commands"][0] - ]), - ); - assert_eq!( - request(&app, "POST", &path, wrong_order).await.0, - StatusCode::UNPROCESSABLE_ENTITY - ); - assert_eq!( - request( - &app, - "GET", - "/api/workflows/continuation-authored-memos", - Value::Null - ) - .await - .1, - initial - ); + // Published PHP applies the memo before the continuation wherever the + // continuation sits in the batch (`command_batches`). let committed = request(&app, "POST", &path, body.clone()).await; assert_eq!(committed.0, StatusCode::OK, "{}", committed.1); let parent_path = format!("/api/workflows/continuation-authored-memos/runs/{old}"); @@ -290,17 +246,16 @@ async fn malformed_or_unsupported_continuation_refuses_without_a_successor() { original["run_id"].as_str().unwrap() ); let before = request(&app, "GET", &history_path, Value::Null).await.1; - for commands in [ - json!([{"type":"continue_as_new","arguments":envelope(Payload::Long(7))}]), - json!([{"type":"continue_as_new","workflow_type":"different"}]), - json!([{"type":"continue_as_new","queue":"different"}]), - json!([{"type":"continue_as_new"},{"type":"complete_workflow","result":envelope(Payload::Null)}]), + // Two terminal commands are the bridge's 409 `invalid_commands`, as PHP's. + for (commands, expected) in [ + (json!([{"type":"continue_as_new","arguments":envelope(Payload::Long(7))}]), StatusCode::UNPROCESSABLE_ENTITY), + (json!([{"type":"continue_as_new"},{"type":"complete_workflow","result":envelope(Payload::Null)}]), StatusCode::CONFLICT), ] { assert_eq!( request(&app, "POST", &path, completion(&task, commands)) .await .0, - StatusCode::UNPROCESSABLE_ENTITY + expected ); assert_eq!( request(&app, "GET", &history_path, Value::Null).await.1, @@ -449,7 +404,9 @@ async fn continuation_commits_one_successor_and_preserves_original_receipts_afte &task, json!([{"type":"continue_as_new","arguments":envelope(Payload::Array(vec![Payload::Long(7)])),"workflow_type":null,"queue":"test"}]), ); - body["sticky_cache"] = json!({"worker_id":"continuation-worker","ttl_seconds":300}); + // PHP requires every sticky cache field once one is sent. + body["sticky_cache"] = json!({"worker_id":"continuation-worker","workflow_id":"continuation-original", + "run_id":task["run_id"],"build_id":"build-1","ttl_seconds":300}); let committed = request(&app, "POST", &path, body.clone()).await; assert_eq!(committed.0, StatusCode::OK, "{}", committed.1); assert_eq!(committed.1["recorded"], true); diff --git a/rust/tests/execution/deployments.rs b/rust/tests/execution/deployments.rs new file mode 100644 index 000000000..5b5d3afcb --- /dev/null +++ b/rust/tests/execution/deployments.rs @@ -0,0 +1,105 @@ +use super::*; + +const QUEUE: &str = "/api/task-queues/test/build-ids"; + +async fn register_build(app: &Router, worker: &str, build: &str) -> Value { + let (status, body) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":"test", + "runtime":"php","build_id":build,"supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + body +} + +async fn pin_of(app: &Router, workflow: &str) -> Value { + request(app, "GET", &format!("/api/workflows/{workflow}"), Value::Null).await.1["compatibility"].clone() +} + +// Published PHP 2.11.1's build-id lifecycle: promotion pins new starts, a +// drained build's workers are refused polls, a queue whose workers all drain +// refuses starts, and each refusal names its blockage. +#[tokio::test] +async fn build_ids_promote_drain_resume_and_roll_back_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register_build(&app, "v1-worker", "v1").await; + register_build(&app, "v2-worker", "v2").await; + + let (status, missing) = request(&app, "POST", &format!("{QUEUE}/promote"), json!({})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{missing}"); + assert_eq!(missing["errors"]["build_id"], json!(["The build id field must be present."])); + let (status, unknown) = request(&app, "POST", &format!("{QUEUE}/promote"), json!({"build_id":"v3"})).await; + assert_eq!(status, StatusCode::CONFLICT, "{unknown}"); + assert_eq!(unknown["blockages"], json!([{"reason":"unknown_deployment","message":"No deployment is registered for default/test@v3.", + "scope":{"namespace":"default","task_queue":"test","build_id":"v3"}, + "expected_resolution":"Roll a worker that heartbeats this build id, then retry promotion."}])); + assert_eq!((unknown["outcome"].clone(), unknown["deployment"].clone()), (json!("rejected_unknown_deployment"), Value::Null)); + + // Promotion pins new starts to the build, though two builds are live. + let (status, promoted) = request(&app, "POST", &format!("{QUEUE}/promote"), json!({"build_id":" v1 "})).await; + assert_eq!(status, StatusCode::OK, "{promoted}"); + assert_eq!((promoted["build_id"].clone(), promoted["new_start_selected"].clone(), promoted["deployment"]["state"].clone()), + (json!("v1"), json!(true), json!("promoted")), "{promoted}"); + assert!(promoted["promoted_at"].as_str().unwrap().ends_with('Z'), "{promoted}"); + start(&app, "first").await; + assert_eq!(pin_of(&app, "first").await, "v1"); + let (_, listed) = request(&app, "GET", "/api/deployments", Value::Null).await; + let states: Vec<_> = listed["deployments"].as_array().unwrap().iter() + .map(|deployment| (deployment["name"].clone(), deployment["state"].clone(), deployment["new_start_selected"].clone())).collect(); + assert_eq!(states, vec![(json!("default/test@v1"), json!("promoted"), json!(true)), (json!("default/test@v2"), json!("active"), json!(false))]); + let (status, shown) = request(&app, "GET", "/api/deployments/default/test@v1", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{shown}"); + assert!(shown["promoted_at"].as_str().unwrap().ends_with("+00:00"), "{shown}"); + assert_eq!(request(&app, "GET", "/api/deployments/default/test@v9", Value::Null).await, + (StatusCode::NOT_FOUND, json!({"error":"deployment_not_found"}))); + assert_eq!(request(&app, "GET", "/api/deployments/nonsense", Value::Null).await.0, StatusCode::BAD_REQUEST); + + // A draining build's workers claim nothing, and stay draining when they + // register again; new starts go to the build still active. + let (status, drained) = request(&app, "POST", &format!("{QUEUE}/drain"), json!({"build_id":"v1"})).await; + assert_eq!((status, drained["drain_intent"].clone(), drained["new_start_selected"].clone()), (StatusCode::OK, json!("draining"), json!(false))); + let (status, refused) = request(&app, "POST", "/api/worker/workflow-tasks/poll", json!({"worker_id":"v1-worker","task_queue":"test","timeout_seconds":0})).await; + assert_eq!((status, refused["reason"].clone(), refused["poll_status"].clone()), (StatusCode::CONFLICT, json!("worker_draining"), json!("draining")), "{refused}"); + assert_eq!(register_build(&app, "v1-worker", "v1").await["status"], "draining"); + start(&app, "second").await; + assert_eq!(pin_of(&app, "second").await, "v2"); + + // With every live worker draining, the queue refuses starts. + request(&app, "POST", &format!("{QUEUE}/drain"), json!({"build_id":"v2"})).await; + let (status, closed) = request(&app, "POST", "/api/workflows", json!({"workflow_id":"third","workflow_type":"echo","task_queue":"test", + "input":envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CONFLICT, "{closed}"); + for (field, expected) in [("reason", json!("task_queue_draining")), ("outcome", json!("rejected_task_queue_draining")), + ("active_worker_count", json!(0)), ("draining_worker_count", json!(2)), ("draining_build_ids", json!(["v1", "v2"]))] { + assert_eq!(closed[field], expected, "{field}: {closed}"); + } + + // Resuming v1 lets its worker claim its pinned run again. + let (_, resumed) = request(&app, "POST", &format!("{QUEUE}/resume"), json!({"build_id":"v1"})).await; + assert_eq!((resumed["drain_intent"].clone(), resumed["new_start_selected"].clone()), (json!("active"), json!(true)), "{resumed}"); + assert_eq!(poll(&app, "v1-worker", "workflow").await["workflow_id"], "first"); + + // A draining deployment cannot be promoted until it resumes; a rollback + // returns new starts to the earlier promotion. + let (status, blocked) = request(&app, "POST", "/api/deployments/default/test@v2/promote", Value::Null).await; + assert_eq!(status, StatusCode::CONFLICT, "{blocked}"); + assert_eq!(blocked["blockages"].as_array().unwrap().len(), 1, "{blocked}"); + assert_eq!(blocked["reason"], "fleet_is_draining"); + assert_eq!(request(&app, "POST", "/api/deployments/default/test@v2/resume", Value::Null).await.1["state"], "active"); + let (status, promoted) = request(&app, "POST", "/api/deployments/default/test@v2/promote", Value::Null).await; + assert_eq!((status, promoted["state"].clone(), promoted["new_start_selected"].clone()), (StatusCode::OK, json!("promoted"), json!(true)), "{promoted}"); + start(&app, "fourth").await; + assert_eq!(pin_of(&app, "fourth").await, "v2"); + let (_, rolled) = request(&app, "POST", "/api/deployments/default/test@v2/rollback", Value::Null).await; + assert_eq!((rolled["state"].clone(), rolled["accepts_new_work"].clone(), rolled["promoted_at"].clone()), + (json!("rolled_back"), json!(false), Value::Null), "{rolled}"); + let (status, again) = request(&app, "POST", "/api/deployments/default/test@v2/rollback", Value::Null).await; + assert_eq!((status, again["reason"].clone()), (StatusCode::CONFLICT, json!("incompatible_policy")), "{again}"); + start(&app, "fifth").await; + assert_eq!(pin_of(&app, "fifth").await, "v1"); + let (_, builds) = request(&app, "GET", QUEUE, Value::Null).await; + let selected: Vec<_> = builds["build_ids"].as_array().unwrap().iter() + .map(|entry| (entry["build_id"].clone(), entry["new_start_selected"].clone(), entry["rolled_back_at"].is_string())).collect(); + assert_eq!(selected, vec![(json!("v1"), json!(true), false), (json!("v2"), json!(false), true)], "{builds}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/external_payloads.rs b/rust/tests/execution/external_payloads.rs index 785bea3c8..c3771c42e 100644 --- a/rust/tests/execution/external_payloads.rs +++ b/rust/tests/execution/external_payloads.rs @@ -1,4 +1,8 @@ use super::*; +#[path = "external_payloads/cleanup.rs"] +mod cleanup; +#[path = "external_payloads/commands.rs"] +mod commands; #[path = "external_payloads/completion_uploads.rs"] mod completion_uploads; use axum::{body::Bytes, http::HeaderMap}; @@ -125,7 +129,11 @@ async fn json_request( } async fn unclaimed(database: &TestDatabase) -> i64 { - let query = "SELECT COUNT(*) FROM runtime_external_payloads WHERE retained_at IS NULL AND expires_at IS NOT NULL"; + count(database, "SELECT COUNT(*) FROM runtime_external_payloads WHERE retained_at IS NULL AND expires_at IS NOT NULL").await +} + +/// One integer the database answers. +async fn count(database: &TestDatabase, query: &'static str) -> i64 { match database { TestDatabase::Sqlite(dir) => { let pool = sqlx::SqlitePool::connect(&format!( @@ -463,8 +471,10 @@ async fn external_workflow_completion_fences_before_retention_and_keeps_opaque_r .1, before ); + // PHP ignores an unknown field but refuses a timer delay on a + // complete_workflow command. let mut unsupported = body.clone(); - unsupported["commands"][0]["unknown"] = json!(true); + unsupported["commands"][0]["delay_seconds"] = json!(1); assert_eq!( json_request(&app, "POST", &path, "worker-token", unsupported) .await @@ -637,12 +647,21 @@ async fn external_activity_result_retains_only_after_live_authority_and_survives StatusCode::CONFLICT ); assert_eq!(unclaimed(&database).await, 1); - assert_eq!( - json_request(&app, "GET", &history_path, "operator-token", Value::Null) - .await - .1, - before - ); + // The refused completion publishes nothing. The only later event + // is the scheduler settling the expired lease, as PHP's repair + // loop does. + let after = json_request(&app, "GET", &history_path, "operator-token", Value::Null) + .await + .1; + let (before, after) = (before["events"].as_array().unwrap(), after["events"].as_array().unwrap()); + assert_eq!(after[..before.len()], before[..], "{after:?}"); + for event in &after[before.len()..] { + assert_eq!( + (event["event_type"].as_str(), event["payload"]["failure_reason"].as_str()), + (Some("ActivityFailed"), Some("lease_expired")), + "{event}" + ); + } runtime.close().await; } else { // Fail after retention and activity-row changes, at history publication. diff --git a/rust/tests/execution/external_payloads/cleanup.rs b/rust/tests/execution/external_payloads/cleanup.rs new file mode 100644 index 000000000..4b4e32bb0 --- /dev/null +++ b/rust/tests/execution/external_payloads/cleanup.rs @@ -0,0 +1,135 @@ +use super::*; +use sqlx::AssertSqlSafe; + +async fn upload_bytes(router: &Router, bytes: &[u8]) -> Value { + let (status, _, body) = request(router, "POST", "/api/external-payloads/v1", "worker-token", "default", + Some(&metadata(bytes)), Body::from(bytes.to_vec())).await; + assert_eq!(status, StatusCode::CREATED, "{}", String::from_utf8_lossy(&body)); + serde_json::from_slice::(&body).unwrap()["reference"].clone() +} + +fn id(reference: &Value) -> &str { + reference["reference_id"].as_str().unwrap() +} + +/// The local driver's content-addressed object path. +fn object(root: &std::path::Path, reference: &Value) -> std::path::PathBuf { + let sha = reference["sha256"].as_str().unwrap(); + root.join("avro").join(&sha[..2]).join(sha) +} + +async fn expire(database: &TestDatabase, ids: &[&str]) { + let list = ids.iter().map(|id| format!("'{id}'")).collect::>().join(","); + database.execute(AssertSqlSafe(format!( + "UPDATE runtime_external_payloads SET expires_at='2000-01-01 00:00:00' WHERE id IN ({list})"))).await; +} + +async fn pass(router: &Router, body: Value) -> (StatusCode, Value) { + json_request(router, "POST", "/api/system/external-payload-cleanup/pass", "admin-token", body).await +} + +async fn status(router: &Router) -> Value { + let (status, body) = json_request(router, "GET", "/api/system/external-payload-cleanup", "admin-token", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["namespace"], "default"); + body["cleanup"].clone() +} + +fn report(values: [i64; 7], backlog: i64, limit: i64, pressure: bool) -> Value { + json!({"processed": values[0], "deleted_references": values[1], "deleted_backing_objects": values[2], + "shared_objects_preserved": values[3], "skipped": values[4], "blocked": values[5], + "storage_driver_failures": values[6], "backlog": backlog, "scan_limit": limit, "scan_pressure": pressure}) +} + +#[tokio::test] +async fn cleanup_reclaims_expired_objects_and_keeps_shared_live_and_retained_ones() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("objects"); + let (database, _runtime, router) = setup(ExternalPayloadConfig::default(), &root).await; + let fresh = status(&router).await; + assert_eq!([&fresh["totals"]["passes"], &fresh["last_pass"]["status"]], [&json!(0), &Value::Null]); + let expired = upload_bytes(&router, b"expired object").await; + let shared = upload_bytes(&router, b"shared object").await; + let live = upload_bytes(&router, b"live object").await; + let retained = upload_bytes(&router, b"retained object").await; + // Another namespace's reference to the shared object. + database.execute(AssertSqlSafe(format!("INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes,upload_status,retained_at,expires_at,created_at,updated_at) SELECT 'zz-shared-twin','other',storage_uri,storage_uri_sha256,codec,sha256,size_bytes,upload_status,NULL,expires_at,created_at,updated_at FROM runtime_external_payloads WHERE id='{}'", id(&shared)))).await; + expire(&database, &[id(&expired), id(&shared), "zz-shared-twin", id(&retained)]).await; + database.execute(AssertSqlSafe(format!( + "UPDATE runtime_external_payloads SET retained_at='2026-01-01 00:00:00' WHERE id='{}'", id(&retained)))).await; + + // The pass reclaims only this namespace's references. The shared one + // loses its row and keeps the object the other namespace still holds. + let (code, body) = pass(&router, json!({})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + assert_eq!(body, json!({"namespace": "default", "cleanup": report([2, 2, 1, 1, 0, 0, 0], 0, 100, false)})); + assert!(!object(&root, &expired).exists()); + assert!(object(&root, &shared).exists()); + assert!(object(&root, &live).exists()); + assert!(object(&root, &retained).exists()); + assert_eq!(count(&database, "SELECT COUNT(*) FROM runtime_external_payloads").await, 3); + assert_eq!(count(&database, "SELECT COUNT(*) FROM runtime_external_payloads WHERE id='zz-shared-twin'").await, 1); + + let after = status(&router).await; + assert_eq!( + after["totals"], + json!({"passes": 1, "deleted_references": 2, "deleted_backing_objects": 1, "shared_objects_preserved": 1, + "blocked_outcomes": 0, "storage_driver_failures": 0}) + ); + assert_eq!([&after["last_pass"]["status"], &after["last_pass"]["processed"]], [&json!("completed"), &json!(2)]); + let completed = after["last_pass"]["completed_at"].as_str().unwrap(); + assert!(completed.ends_with(".000000Z"), "{completed}"); + assert_eq!(after["backlog"], json!({"expired_unclaimed": 0, "failed_registration_objects": 0})); +} + +#[tokio::test] +async fn cleanup_respects_its_limit_a_backup_hold_and_a_disabled_policy() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("objects"); + let (database, _runtime, router) = setup(ExternalPayloadConfig::default(), &root).await; + let first = upload_bytes(&router, b"first object").await; + let second = upload_bytes(&router, b"second object").await; + expire(&database, &[id(&first), id(&second)]).await; + assert_eq!(status(&router).await["backlog"]["expired_unclaimed"], 2); + + // A one-reference pass reports the rest as backlog under pressure. + let (code, body) = pass(&router, json!({"limit": "1"})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + assert_eq!(body["cleanup"], report([1, 1, 1, 0, 0, 0, 0], 1, 1, true)); + + // An unexpired online-backup hold pauses deletion: 207, object kept. + database.execute("INSERT INTO runtime_external_payload_backup_hold(id,owner,acquired_at,expires_at) VALUES (1,'7f1c3a1e-0000-4000-8000-000000000001','2026-01-01 00:00:00','2037-01-01 00:00:00')").await; + let (code, body) = pass(&router, json!({})).await; + assert_eq!(code, StatusCode::MULTI_STATUS, "{body}"); + assert_eq!(body["cleanup"], report([1, 0, 0, 0, 0, 1, 0], 1, 100, false)); + let remaining = if object(&root, &first).exists() { &first } else { &second }; + assert!(object(&root, remaining).exists()); + let held = status(&router).await; + assert_eq!([&held["last_pass"]["status"], &held["totals"]["blocked_outcomes"]], [&json!("blocked"), &json!(1)]); + + // With the policy disabled there is no driver to delete through. + database.execute("UPDATE runtime_external_payload_backup_hold SET expires_at='2000-01-01 00:00:00'").await; + assert_eq!(policy(&router, "default", &root, false).await, StatusCode::OK); + let (code, body) = pass(&router, json!({})).await; + assert_eq!(code, StatusCode::MULTI_STATUS, "{body}"); + assert_eq!(body["cleanup"]["blocked"], 1); + + // Once the hold has expired and the policy is back, the object goes. + assert_eq!(policy(&router, "default", &root, true).await, StatusCode::OK); + let (code, body) = pass(&router, json!({"limit": null})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + assert_eq!(body["cleanup"], report([1, 1, 1, 0, 0, 0, 0], 0, 100, false)); + assert!(!object(&root, remaining).exists()); + assert_eq!(status(&router).await["totals"]["passes"], 4); + + for (limit, message) in [ + (json!(0), "The limit field must be at least 1."), + (json!(1001), "The limit field must not be greater than 1000."), + (json!(2.5), "The limit field must be an integer."), + ] { + let (code, body) = pass(&router, json!({"limit": limit})).await; + assert_eq!(code, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"]["limit"], json!([message])); + } + assert_eq!(status(&router).await["totals"]["passes"], 4); +} diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs new file mode 100644 index 000000000..82237259c --- /dev/null +++ b/rust/tests/execution/external_payloads/commands.rs @@ -0,0 +1,153 @@ +//! Runtime external references on signal, update and query input, as +//! published PHP 2.11.1 resolves them: a signal keeps the reference, an update +//! and a query carry the referenced bytes inline, and each claims the object. +use super::*; + +async fn upload(router: &Router, blob: &str) -> Value { + let (status, _, body) = request(router, "POST", "/api/external-payloads/v1", "worker-token", "default", + Some(&metadata(blob.as_bytes())), Body::from(blob.to_owned())).await; + assert_eq!(status, StatusCode::CREATED, "{}", String::from_utf8_lossy(&body)); + serde_json::from_slice::(&body).unwrap()["reference"].clone() +} + +fn argument(value: &str) -> String { + durable_workflow_server::codec::ValueCodec::new() + .unwrap() + .encode(&Payload::Array(vec![Payload::String(value.into())])) + .unwrap() +} + +fn string_parameter() -> Value { + json!([{"name":"value","type":"string","required":true,"variadic":false,"default_available":false,"allows_null":false}]) +} + +/// Starts `workflow_id` on its own queue and leaves it waiting on a timer. +async fn waiting_run(app: &Router, workflow_id: &str, worker: Value) { + let queue = worker["task_queue"].clone(); + let worker_id = worker["worker_id"].clone(); + let (status, body) = json_request(app, "POST", "/api/worker/register", "worker-token", worker).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + let (status, body) = json_request(app, "POST", "/api/workflows", "operator-token", + json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":queue,"input":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + let (status, polled) = json_request(app, "POST", "/api/worker/workflow-tasks/poll", "worker-token", + json!({"worker_id":worker_id,"task_queue":queue,"timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{polled}"); + let task = &polled["task"]; + let (status, body) = json_request(app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()), + "worker-token", json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "commands":[{"type":"start_timer","delay_seconds":3600}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + +fn typed(body: &Value, status: u16, reason: &str, message: &str, workflow_id: &str) { + assert_eq!(body["schema"], "durable-workflow.v2.runtime-external-payload-error.v1", "{body}"); + assert_eq!((body["status"].as_u64(), body["reason"].as_str(), body["message"].as_str()), + (Some(u64::from(status)), Some(reason), Some(message)), "{body}"); + assert_eq!(body["retryable"], false, "{body}"); + assert_eq!(body["workflow_id"], workflow_id, "{body}"); +} + +#[tokio::test] +async fn signal_update_and_query_input_resolve_runtime_references_like_php() { + let temp = tempfile::tempdir().unwrap(); + let (database, runtime, app) = setup(ExternalPayloadConfig::default(), &temp.path().join("objects")).await; + + // A signal keeps the reference; its history shows the external envelope. + waiting_run(&app, "external-signal", json!({"worker_id":"signal-worker","task_queue":"external-signal", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + let signal = upload(&app, &argument("signal")).await; + assert_eq!(unclaimed(&database).await, 1); + let (status, body) = json_request(&app, "POST", "/api/workflows/external-signal/signal/s", "operator-token", + json!({"input":{"codec":"avro","external_payload":signal}})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + assert_eq!(unclaimed(&database).await, 0); + let (_, history) = json_request(&app, "GET", + &format!("/api/workflows/external-signal/runs/{}/history", body["run_id"].as_str().unwrap()), "operator-token", Value::Null).await; + let received = history["events"].as_array().unwrap().iter().find(|event| event["event_type"] == "SignalReceived").expect("signal event"); + assert_eq!(received["payload"]["arguments"], json!({"codec":"avro","external_payload":signal}), "{received}"); + + // An update resolves the reference to its bytes and keeps them inline. + waiting_run(&app, "external-update", json!({"worker_id":"update-worker","task_queue":"external-update", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":["workflow_updates"], + "workflow_command_contracts":{"echo":{"updates":["u"],"update_contracts":[{"name":"u","parameters":string_parameter()}]}}})).await; + let blob = argument("update"); + let update = upload(&app, &blob).await; + let (status, body) = json_request(&app, "POST", "/api/workflows/external-update/update/u", "operator-token", + json!({"input":{"codec":"avro","external_payload":update},"wait_for":"accepted"})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + assert_eq!(body["update_status"], "accepted", "{body}"); + assert_eq!(unclaimed(&database).await, 0); + let (status, polled) = json_request(&app, "POST", "/api/worker/workflow-tasks/poll", "worker-token", + json!({"worker_id":"update-worker","task_queue":"external-update","timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{polled}"); + let accepted = polled["task"]["history_events"].as_array().unwrap().iter() + .find(|event| event["event_type"] == "UpdateAccepted").expect("update event"); + assert_eq!(accepted["payload"]["arguments"], json!({"codec":"avro","blob":blob}), "{accepted}"); + + // A query carries the bytes to its worker. + waiting_run(&app, "external-query", json!({"worker_id":"query-worker","task_queue":"external-query", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":["query_tasks"], + "workflow_command_contracts":{"echo":{"queries":["look"],"query_contracts":[{"name":"look","parameters":string_parameter()}]}}})).await; + let blob = argument("query"); + let query = upload(&app, &blob).await; + let asked = { + let app = app.clone(); + let query = query.clone(); + tokio::spawn(async move { + json_request(&app, "POST", "/api/workflows/external-query/query/look", "operator-token", + json!({"input":{"codec":"avro","external_payload":query}})).await + }) + }; + let task = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let (status, body) = json_request(&app, "POST", "/api/worker/query-tasks/poll", "worker-token", + json!({"worker_id":"query-worker","task_queue":"external-query","timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + if !body["task"].is_null() { + break body["task"].clone(); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("query task"); + assert_eq!(task["query_arguments"], json!({"codec":"avro","blob":blob}), "{task}"); + let (status, body) = json_request(&app, "POST", &format!("/api/worker/query-tasks/{}/complete", task["query_task_id"].as_str().unwrap()), + "worker-token", json!({"lease_owner":"query-worker","query_task_attempt":task["query_task_attempt"], + "result_envelope":{"codec":"avro","blob":blob}})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (status, answer) = asked.await.unwrap(); + assert_eq!(status, StatusCode::OK, "{answer}"); + assert_eq!(answer["result_envelope"], json!({"codec":"avro","blob":blob}), "{answer}"); + assert_eq!(unclaimed(&database).await, 0); + + // Refusals are PHP's typed external payload errors, named for the + // workflow, and claim nothing. + let spare = upload(&app, &argument("spare")).await; + assert_eq!(unclaimed(&database).await, 1); + let mut missing = spare.clone(); + missing["reference_id"] = json!("ep_01M5KMWZFKJPWN8PZTTXMC4VH0"); + for (path, input, status, reason, message) in [ + ("/api/workflows/external-signal/signal/s", json!({"codec":"avro","external_payload":spare,"extra":1}), 422, + "external_payload_unsupported", "External payload envelopes must contain exactly codec and external_payload."), + ("/api/workflows/external-update/update/u", json!({"codec":"json","external_payload":spare}), 422, + "external_payload_integrity_mismatch", "External payload envelope codec does not match the runtime reference."), + // The lookup comes before the codec comparison. + ("/api/workflows/external-query/query/look", json!({"codec":"json","external_payload":missing}), 404, + "external_payload_not_found", "External payload reference was not found in this namespace."), + ("/api/workflows/external-signal/signal/s", json!({"codec":"avro","external_storage":{"uri":"file:///elsewhere"}}), 422, + "external_payload_unsupported", "Provider-specific external_storage references are not accepted by the runtime transport."), + ("/api/workflows/external-update/update/u", json!({"codec":"avro","external_payload":{"reference_id":"ep_1"}}), 422, + "external_payload_unsupported", + "Runtime external payload references must contain exactly schema, reference_id, codec, size_bytes, and sha256."), + ] { + let (code, body) = json_request(&app, "POST", path, "operator-token", json!({"input":input})).await; + assert_eq!(code.as_u16(), status, "{path} {body}"); + typed(&body, status, reason, message, path.split('/').nth(3).unwrap()); + assert_eq!(body["control_plane"]["reason"], reason, "{body}"); + } + assert_eq!(unclaimed(&database).await, 1); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/external_payloads/completion_uploads.rs b/rust/tests/execution/external_payloads/completion_uploads.rs index 2c4a80878..0a9591d9b 100644 --- a/rust/tests/execution/external_payloads/completion_uploads.rs +++ b/rust/tests/execution/external_payloads/completion_uploads.rs @@ -446,3 +446,30 @@ async fn activity_completion_during_upload_cannot_reopen_its_closed_allowance() runtime.close().await; database.remove().await; } + +#[tokio::test] +async fn cleanup_renews_a_live_completion_budget_and_drops_a_closed_one() { + let (_provider, database, _runtime, app, task, bytes) = original(120).await; + let uploaded = upload(&app, "worker-token", "default", &context(&task), bytes, None).await; + assert_eq!(uploaded.0, StatusCode::CREATED, "{}", uploaded.2); + assert_eq!(count(&database, "SELECT COUNT(*) FROM runtime_payload_completion_budgets").await, 1); + database.execute("UPDATE runtime_payload_completion_budgets SET expires_at='2000-01-01 00:00:00'").await; + let path = "/api/system/external-payload-cleanup/pass"; + let (code, body) = json_request(&app, "POST", path, "admin-token", json!({})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + // The activity can still complete, so its allowance gets another period. + assert_eq!( + count(&database, "SELECT COUNT(*) FROM runtime_payload_completion_budgets WHERE expires_at>'2001-01-01'").await, + 1 + ); + let (code, body) = json_request(&app, "POST", + &format!("/api/worker/activity-tasks/{}/complete", task["task_id"].as_str().unwrap()), "worker-token", + json!({"activity_attempt_id": task["activity_attempt_id"], "lease_owner": task["lease_owner"], + "result": envelope(Payload::Array(vec![]))})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + database.execute("UPDATE runtime_payload_completion_budgets SET expires_at='2000-01-01 00:00:00'").await; + let (code, body) = json_request(&app, "POST", path, "admin-token", json!({})).await; + assert_eq!(code, StatusCode::OK, "{body}"); + // A closed activity cannot resume its upload: the allowance is dropped. + assert_eq!(count(&database, "SELECT COUNT(*) FROM runtime_payload_completion_budgets").await, 0); +} diff --git a/rust/tests/execution/named_signal_timeouts.rs b/rust/tests/execution/named_signal_timeouts.rs index 7f928d09b..d4d4cfd1f 100644 --- a/rust/tests/execution/named_signal_timeouts.rs +++ b/rust/tests/execution/named_signal_timeouts.rs @@ -286,10 +286,13 @@ async fn named_invalid_signal_timeouts_leave_original_attempt_usable() { let run = namespaces::start_scoped(&app, namespace, "named-invalid-timeout").await; let task = signals::claim(&app, namespace).await; let before = signals::read(&app, namespace, &run, "history").await; - for timeout in [json!(-1), json!(1.5), json!("1"), json!(i64::MAX)] { + // PHP's rule refusals, then native's own for a timeout too large for a + // timestamp, which PHP accepts. "1" is an integer to PHP. + for (timeout, reason) in [(json!(-1), "validation_failed"), (json!(1.5), "validation_failed"), + (json!(i64::MAX), "invalid_signal_timeout")] { let refused = signals::finish(&app, namespace, &task, wait(timeout)).await; assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!(refused.1["reason"], "invalid_signal_timeout"); + assert_eq!(refused.1["reason"], reason, "{}", refused.1); assert_eq!( signals::read(&app, namespace, &run, "history").await, before diff --git a/rust/tests/execution/named_timers.rs b/rust/tests/execution/named_timers.rs index e12530739..aa4fab816 100644 --- a/rust/tests/execution/named_timers.rs +++ b/rust/tests/execution/named_timers.rs @@ -197,7 +197,8 @@ async fn named_timer_invalid_batch_is_atomic_and_zero_delay_keeps_its_identity() ) .await; assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!(refused.1["reason"], "invalid_timer_delay"); + assert_eq!(refused.1["errors"]["commands.1.delay_seconds"], + json!(["The commands.1.delay_seconds field must be at least 0."]), "{}", refused.1); assert_eq!( scoped(&app, "alpha", "GET", &path, Value::Null).await, before diff --git a/rust/tests/execution/namespaces.rs b/rust/tests/execution/namespaces.rs index 933665b6a..2ffcf717f 100644 --- a/rust/tests/execution/namespaces.rs +++ b/rust/tests/execution/namespaces.rs @@ -524,6 +524,8 @@ async fn same_worker_id_isolated_claims_completion_and_deregistration_by_namespa first["run_id"].as_str().unwrap() ); let before = scoped(&app, "alpha", "GET", &history_path, Value::Null).await; + // A refused completion in the namespace changes nothing. (Published PHP + // serves every workflow command in a named namespace: `named_namespace_commands`.) let unsupported = scoped( &app, "alpha", @@ -531,18 +533,13 @@ async fn same_worker_id_isolated_claims_completion_and_deregistration_by_namespa &path, completion( &first_task, - // Timers and children have separate positive namespace coverage. - // Side effects remain outside this namespace slice and must fail - // atomically. - json!([{"type":"record_side_effect","result":envelope(Payload::Long(9007199254740993))}]), + json!([{"type":"record_side_effect","result":envelope(Payload::Long(1))}, + {"type":"complete_workflow","result":envelope(Payload::Null)},{"type":"fail_workflow","message":"x"}]), ), ) .await; - assert_eq!(unsupported.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!( - unsupported.1["reason"], - "development_namespace_command_unqualified" - ); + assert_eq!(unsupported.0, StatusCode::CONFLICT, "{}", unsupported.1); + assert_eq!(unsupported.1["reason"], "invalid_commands", "{}", unsupported.1); assert_eq!( scoped(&app, "alpha", "GET", &history_path, Value::Null).await, before @@ -615,6 +612,39 @@ async fn same_worker_id_isolated_claims_completion_and_deregistration_by_namespa database.remove().await; } +#[tokio::test] +async fn named_namespace_workflow_task_failure_retries_only_in_its_own_namespace() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + for name in ["alpha", "beta"] { + create(&app, name).await; + let (status, body) = scoped(&app, name, "POST", "/api/worker/register", json!({ + "worker_id":"shared-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + } + let run = start_scoped(&app, "alpha", "retry-alpha").await; + let poll = json!({"worker_id":"shared-worker","task_queue":"test","timeout_seconds":0}); + let task = scoped(&app, "alpha", "POST", "/api/worker/workflow-tasks/poll", poll.clone()).await.1["task"].clone(); + let path = format!("/api/worker/workflow-tasks/{}/fail", task["task_id"].as_str().unwrap()); + let body = json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "failure":{"message":"named failure","type":"RuntimeException"}}); + let refused = scoped(&app, "beta", "POST", &path, body.clone()).await; + assert_eq!(refused.0, StatusCode::NOT_FOUND, "{}", refused.1); + assert_eq!(refused.1["reason"], "task_not_found"); + + let (status, failed) = scoped(&app, "alpha", "POST", &path, body).await; + assert_eq!(status, StatusCode::OK, "{failed}"); + assert_eq!([&failed["outcome"], &failed["recorded"]], [&json!("failed"), &json!(true)]); + let foreign = scoped(&app, "beta", "POST", "/api/worker/workflow-tasks/poll", poll.clone()).await; + assert_eq!(foreign.0, StatusCode::OK, "{}", foreign.1); + assert!(foreign.1["task"].is_null(), "{}", foreign.1); + let retry = scoped(&app, "alpha", "POST", "/api/worker/workflow-tasks/poll", poll).await.1["task"].clone(); + assert_eq!(retry["task_id"], failed["next_task_id"], "{retry}"); + assert_eq!([&retry["run_id"], &retry["workflow_task_attempt"]], [&run["run_id"], &json!(2)]); +} + #[tokio::test] async fn namespace_updates_follow_published_retention_rules() { let database = TestDatabase::new().await; @@ -812,3 +842,76 @@ async fn namespace_delete_refuses_while_external_payloads_remain() { runtime.close().await; database.remove().await; } + +async fn claim_named(app: &Router, namespace: &str) -> Value { + let (status, body) = scoped(app, namespace, "POST", "/api/worker/workflow-tasks/poll", + json!({"worker_id":"named-worker","task_queue":"test","timeout_seconds":0})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body["task"].clone() +} + +async fn complete_named(app: &Router, namespace: &str, task: &Value, commands: Value) -> (StatusCode, Value) { + let path = format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()); + scoped(app, namespace, "POST", &path, completion(task, commands)).await +} + +// Published PHP serves every workflow command and operation in a named +// namespace: version markers, side effects, updates and their completion, +// repair, cooperative cancellation requests and continue-as-new +// (`named_namespace_commands`). +#[tokio::test] +async fn named_namespace_commands_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + create(&app, "gamma").await; + let (status, body) = scoped(&app, "gamma", "POST", "/api/worker/register", json!({"worker_id":"named-worker", + "task_queue":"test","runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[], + "capabilities":["workflow_updates","cooperative_cancellation"],"workflow_command_contracts":{"echo":{ + "signals":[],"queries":[],"updates":["set"],"update_validators":[],"update_contracts":[{"name":"set","parameters":[ + {"name":"value","type":"int","required":true,"variadic":false,"default_available":false,"allows_null":false}]}]}}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + + let started = start_scoped(&app, "gamma", "named-commands").await; + let task = claim_named(&app, "gamma").await; + let (status, body) = complete_named(&app, "gamma", &task, json!([ + {"type":"record_version_marker","change_id":"c","version":1,"min_supported":1,"max_supported":1}, + {"type":"record_side_effect","result":envelope(Payload::Long(1))}, + {"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let (status, body) = scoped(&app, "gamma", "POST", "/api/workflows/named-commands/update/set", + json!({"input":[1],"wait_for":"accepted"})).await; + assert!(status.is_success(), "{status} {body}"); + let update = body["update_id"].clone(); + let task = claim_named(&app, "gamma").await; + assert_eq!(task["workflow_update_id"], update, "{task}"); + let (status, body) = complete_named(&app, "gamma", &task, + json!([{"type":"complete_update","update_id":update,"result":envelope(Payload::Long(2))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let (status, body) = scoped(&app, "gamma", "POST", "/api/workflows/named-commands/repair", json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (status, body) = scoped(&app, "gamma", "POST", "/api/workflows/named-commands/request-cancellation", + json!({"reason":"stop"})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let history_path = format!("/api/workflows/named-commands/runs/{}/history", started["run_id"].as_str().unwrap()); + let (_, history) = scoped(&app, "gamma", "GET", &history_path, Value::Null).await; + let kinds: Vec<_> = history["events"].as_array().unwrap().iter().map(|event| event["event_type"].clone()).collect(); + for kind in ["VersionMarkerRecorded", "SideEffectRecorded", "UpdateCompleted"] { + assert!(kinds.contains(&json!(kind)), "{kind}: {history}"); + } + + let continued = start_scoped(&app, "gamma", "named-continue").await; + let mut task = claim_named(&app, "gamma").await; + while task["workflow_id"] != "named-continue" { + task = claim_named(&app, "gamma").await; + } + let (status, body) = complete_named(&app, "gamma", &task, json!([{"type":"continue_as_new"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["run_status"], "completed", "{body}"); + let (_, described) = scoped(&app, "gamma", "GET", "/api/workflows/named-continue", Value::Null).await; + assert_ne!(described["run_id"], continued["run_id"], "{described}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/priority_fairness.rs b/rust/tests/execution/priority_fairness.rs new file mode 100644 index 000000000..230ba8a90 --- /dev/null +++ b/rust/tests/execution/priority_fairness.rs @@ -0,0 +1,41 @@ +use super::*; + +// Published PHP 2.11.1's priority-fairness snapshot: ready tasks by priority +// tier and fairness class, and a score of 0 for each class, since PHP's +// fairness state starts empty on every request. A poll takes the oldest +// ready task of the most urgent tier. +#[tokio::test] +async fn priority_fairness_reports_tiers_and_classes_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (status, empty) = request(&app, "GET", "/api/task-queues/test/priority-fairness", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{empty}"); + assert_eq!(empty, json!({"namespace":"default","queue":"test", + "workflow_task":{"ready_tasks":0,"priority_tiers":[],"recent_dispatch":[]}, + "activity_task":{"ready_tasks":0,"priority_tiers":[],"recent_dispatch":[]}})); + for (workflow, extra) in [("a1", json!({"fairness_key":"Tenant-A"})), ("a2", json!({"fairness_key":"tenant-a","fairness_weight":3})), + ("b1", json!({"fairness_key":"tenant-b","fairness_weight":2})), ("n1", json!({})), ("u1", json!({"priority":1,"fairness_key":"urgent"}))] { + let mut body = json!({"workflow_id":workflow,"workflow_type":"echo","task_queue":"test","input":envelope(Payload::Array(vec![]))}); + body.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + let (status, started) = request(&app, "POST", "/api/workflows", body).await; + assert_eq!(status, StatusCode::CREATED, "{started}"); + } + let (_, snapshot) = request(&app, "GET", "/api/task-queues/test/priority-fairness", Value::Null).await; + assert_eq!(snapshot["workflow_task"], json!({"ready_tasks":5,"priority_tiers":[ + {"priority":1,"count":1,"classes":[{"fairness_key":"urgent","count":1,"fairness_weight":1}]}, + {"priority":5,"count":4,"classes":[{"fairness_key":null,"count":1,"fairness_weight":1}, + {"fairness_key":"tenant-a","count":2,"fairness_weight":1},{"fairness_key":"tenant-b","count":1,"fairness_weight":2}]}], + "recent_dispatch":[{"fairness_key":"tenant-a","score":0},{"fairness_key":"tenant-b","score":0}, + {"fairness_key":null,"score":0},{"fairness_key":"urgent","score":0}]})); + let (status, body) = request(&app, "POST", "/api/worker/register", json!({"worker_id":"fair-worker","task_queue":"test", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + let mut order = Vec::new(); + for _ in 0..5 { + order.push(poll(&app, "fair-worker", "workflow").await["workflow_id"].clone()); + } + assert_eq!(order, ["u1", "a1", "a2", "b1", "n1"]); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/prometheus_metrics.rs b/rust/tests/execution/prometheus_metrics.rs new file mode 100644 index 000000000..16add676b --- /dev/null +++ b/rust/tests/execution/prometheus_metrics.rs @@ -0,0 +1,170 @@ +use super::*; + +const PROMETHEUS: &str = "/api/system/prometheus-metrics"; + +async fn summary(app: &Router) -> Value { + let (status, body) = request(app, "GET", PROMETHEUS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + [&body["schema"], &body["namespace"]], + [&json!("durable-workflow.prometheus-metrics.v1"), &json!("default")] + ); + body +} + +async fn start_on(app: &Router, workflow: &str, workflow_type: &str, queue: &str) { + let (status, body) = request(app, "POST", "/api/workflows", json!({"workflow_id": workflow, + "workflow_type": workflow_type, "task_queue": queue, "input": envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +fn buckets(values: [i64; 10]) -> Value { + let bounds = ["0.1", "0.5", "1", "2.5", "5", "10", "30", "60", "300", "900"]; + Value::Object(bounds.iter().zip(values).map(|(bound, value)| ((*bound).to_owned(), json!(value))).collect()) +} + +#[tokio::test] +async fn prometheus_summary_counts_runs_activities_and_queues_with_cumulative_latency() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let empty = summary(&app).await; + assert_eq!(empty["series"], json!({"workflows": [], "activities": [], "task_queues": []})); + assert_eq!(empty["latency_buckets_seconds"].to_string(), "[0.1,0.5,1,2.5,5,10,30,60,300,900]"); + assert_eq!( + [&empty["series_count"], &empty["observed_series_count_precision"]], + [&json!(0), &json!("exact")] + ); + assert_eq!(empty["cardinality"]["series_limits"]["workflows"]["limit"], 100); + + let (status, body) = request(&app, "POST", "/api/worker/register", json!({"worker_id": "prom-worker", + "task_queue": "test", "runtime": "php", "supported_workflow_types": ["echo", "other"], + "supported_activity_types": ["act"], "max_concurrent_workflow_tasks": 4, "max_concurrent_activity_tasks": 3})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + start_on(&app, "prom-completed", "echo", "test").await; + let task = poll(&app, "prom-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type": "schedule_activity", "activity_type": "act", + "arguments": envelope(Payload::Array(vec![])), "retry_policy": {"max_attempts": 1, "backoff_seconds": []}, + "start_to_close_timeout": 300}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let activity = poll(&app, "prom-worker", "activity").await; + let (status, body) = request(&app, "POST", + &format!("/api/worker/activity-tasks/{}/complete", activity["task_id"].as_str().unwrap()), + json!({"activity_attempt_id": activity["activity_attempt_id"], "lease_owner": activity["lease_owner"], + "result": envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "prom-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, + json!([{"type": "complete_workflow", "result": envelope(Payload::Array(vec![]))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + start_on(&app, "prom-terminated", "other", "test").await; + let (status, body) = request(&app, "POST", "/api/workflows/prom-terminated/terminate", json!({"reason": "prom"})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + start_on(&app, "prom-waiting", "echo", "second").await; + // Fixed durations: the completed run 2 s, the terminated run 50 ms and the + // activity 1.5 s, so each falls in known buckets on every backend. + database.execute("UPDATE workflow_run_summaries SET duration_ms=2000 WHERE workflow_type='echo' AND status='completed'").await; + database.execute("UPDATE workflow_run_summaries SET duration_ms=50 WHERE workflow_type='other'").await; + database.execute("UPDATE activity_executions SET started_at='2026-10-22 10:00:00', closed_at='2026-10-22 10:00:01.5'").await; + // As in PHP, a task made available within the current second still counts + // as delayed; this one became available earlier. + database.execute("UPDATE workflow_tasks SET available_at='2026-10-22 09:00:00' WHERE queue='second'").await; + + let read = summary(&app).await; + let workflows = read["series"]["workflows"].as_array().unwrap(); + assert_eq!(workflows.len(), 3, "{read}"); + assert_eq!( + workflows[0], + json!({"task_queue": "second", "workflow_type": "echo", "started_total": 1, "completed_total": 0, + "failed_total": 0, "cancelled_total": 0, "terminated_total": 0, "running": 1, + "latency_seconds": {"count": 0, "sum": 0, "buckets": buckets([0; 10])}}) + ); + assert_eq!( + workflows[1], + json!({"task_queue": "test", "workflow_type": "echo", "started_total": 1, "completed_total": 1, + "failed_total": 0, "cancelled_total": 0, "terminated_total": 0, "running": 0, + "latency_seconds": {"count": 1, "sum": 2, "buckets": buckets([0, 0, 0, 1, 1, 1, 1, 1, 1, 1])}}) + ); + assert_eq!( + [&workflows[2]["terminated_total"], &workflows[2]["latency_seconds"]], + [&json!(1), &json!({"count": 1, "sum": 0.05, "buckets": buckets([1; 10])})] + ); + // SQLite measures activity durations with PHP's julianday difference, a + // float accurate to about a millisecond. + let mut activities = read["series"]["activities"].clone(); + let sum = activities[0]["latency_seconds"]["sum"].take().as_f64().unwrap(); + assert!((sum - 1.5).abs() < 0.001, "{sum}"); + assert_eq!( + activities, + json!([{"task_queue": "test", "workflow_type": "echo", "activity_type": "act", "started_total": 1, + "completed_total": 1, "failed_total": 0, "cancelled_total": 0, "running": 0, + "latency_seconds": {"count": 1, "sum": null, "buckets": buckets([0, 0, 0, 1, 1, 1, 1, 1, 1, 1])}}]) + ); + let queues = read["series"]["task_queues"].as_array().unwrap(); + assert_eq!( + queues.iter().map(|queue| queue["task_queue"].as_str().unwrap()).collect::>(), + ["second", "test"] + ); + assert_eq!( + [&queues[0]["workflow_ready_tasks"], &queues[0]["workflow_ready_due_tasks"], &queues[0]["active_pollers"]], + [&json!(1), &json!(1), &json!(0)] + ); + assert_eq!( + [ + &queues[1]["workflow_ready_tasks"], + &queues[1]["activity_tasks_added_last_minute"], + &queues[1]["active_pollers"], + &queues[1]["workflow_slot_capacity"], + &queues[1]["activity_slot_capacity"], + &queues[1]["available_workflow_slots"], + &queues[1]["available_activity_slots"], + ], + [&json!(0), &json!(1), &json!(1), &json!(4), &json!(3), &json!(4), &json!(3)] + ); + assert_eq!( + [&read["series_count"], &read["observed_series_count"], &read["suppressed_series_count"]], + [&json!(6), &json!(6), &json!(0)] + ); + + // The summary is namespace-scoped: a named namespace sees only its own. + super::namespaces::create(&app, "prom-named").await; + let (status, named) = super::namespaces::scoped(&app, "prom-named", "GET", PROMETHEUS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{named}"); + assert_eq!( + [&named["namespace"], &named["series_count"]], + [&json!("prom-named"), &json!(0)] + ); +} + +#[tokio::test] +async fn prometheus_summary_bounds_workflow_series_and_discloses_the_suppressed_starts() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + for index in 0..101 { + start_on(&app, &format!("bounded-{index:03}"), &format!("type-{index:03}"), "test").await; + } + let read = summary(&app).await; + let workflows = read["series"]["workflows"].as_array().unwrap(); + assert_eq!(workflows.len(), 100); + assert_eq!( + [&workflows[0]["workflow_type"], &workflows[99]["workflow_type"]], + [&json!("type-000"), &json!("type-099")] + ); + assert_eq!( + read["cardinality"]["series_limits"]["workflows"], + json!({"label_dimensions": ["task_queue", "workflow_type"], "limit": 100, "observed_series_count": 101, + "observed_series_count_precision": "lower_bound", "reported_series_count": 100, + "suppressed_series_count": 1, "suppressed_started_total": 1, "suppressed_counts_precision": "lower_bound", + "truncated": true, "selection": "bounded_task_queue_and_workflow_type_ascending"}) + ); + assert_eq!( + [ + &read["observed_series_count"], + &read["observed_series_count_precision"], + &read["suppressed_series_count"], + &read["suppressed_series_count_precision"], + ], + [&json!(102), &json!("lower_bound"), &json!(1), &json!("lower_bound")] + ); +} diff --git a/rust/tests/execution/redrive.rs b/rust/tests/execution/redrive.rs new file mode 100644 index 000000000..233e14fc1 --- /dev/null +++ b/rust/tests/execution/redrive.rs @@ -0,0 +1,113 @@ +use super::*; + +const FINGERPRINT: &str = "sha256:abababababababababababababababababababababababababababababababab"; + +fn kinds(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +async fn complete_activity(app: &Router, task: &Value) { + let path = format!("/api/worker/activity-tasks/{}/complete", task["task_id"].as_str().unwrap()); + let done = request(app, "POST", &path, json!({"lease_owner":task["lease_owner"], + "activity_attempt_id":task["activity_attempt_id"],"result":envelope(Payload::Array(vec![]))})).await; + assert_eq!(done.0, StatusCode::OK, "{}", done.1); +} + +/// A run whose second activity failed and whose workflow then failed at +/// that step, as the PHP SDK reports it. +async fn failed_at_second_activity(app: &Router, workflow: &str) -> Value { + let started = start(app, workflow).await; + let task = poll(app, "worker", "workflow").await; + finish_task(app, &task, json!([{"type":"schedule_activity","activity_type":"a","arguments":envelope(Payload::Array(vec![]))}])).await; + complete_activity(app, &poll(app, "worker", "activity").await).await; + let task = poll(app, "worker", "workflow").await; + finish_task(app, &task, json!([{"type":"schedule_activity","activity_type":"b","arguments":envelope(Payload::Array(vec![]))}])).await; + let activity = poll(app, "worker", "activity").await; + let path = format!("/api/worker/activity-tasks/{}/fail", activity["task_id"].as_str().unwrap()); + let failed = request(app, "POST", &path, json!({"lease_owner":activity["lease_owner"], + "activity_attempt_id":activity["activity_attempt_id"],"failure":{"class":"App\\Boom","message":"boom"}})).await; + assert_eq!(failed.0, StatusCode::OK, "{}", failed.1); + let task = poll(app, "worker", "workflow").await; + let done = finish_task(app, &task, json!([{"type":"fail_workflow","message":"gave up","failed_step_sequence":2, + "failed_activity_execution_id":activity["activity_execution_id"]}])).await; + assert_eq!(done.1["run_status"], "failed", "{}", done.1); + started +} + +// Published PHP 2.11.1's redrive continues a run that failed at an activity +// step in a new run: its history reuses the completed activities before the +// step, so the worker reschedules only the failed one. +#[tokio::test] +async fn redrive_resumes_a_failed_run_at_its_failed_activity_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let registered = request(&app, "POST", "/api/worker/register", json!({"worker_id":"worker","task_queue":"test", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":["a","b"], + "workflow_definition_fingerprints":{"echo":FINGERPRINT}})).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let source = failed_at_second_activity(&app, "rd").await; + let source_run = source["run_id"].as_str().unwrap(); + let path = format!("/api/workflows/rd/runs/{source_run}/redrive"); + + let (status, invalid) = request(&app, "POST", &path, json!({"request_id":"x".repeat(192)})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{invalid}"); + assert_eq!(invalid["errors"]["request_id"], json!(["The request id field must not be greater than 191 characters."])); + let (status, missing) = request(&app, "POST", "/api/workflows/rd/runs/nope/redrive", json!({})).await; + assert_eq!((status, missing["reason"].clone()), (StatusCode::NOT_FOUND, json!("run_not_found")), "{missing}"); + + let (status, redriven) = request(&app, "POST", &path, json!({"request_id":"rq-1"})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{redriven}"); + for (field, expected) in [("continued_from_run_id", json!(source_run)), ("resume_step_sequence", json!(2)), + ("command_status", json!("accepted")), ("outcome", json!("redriven")), ("reason", Value::Null)] { + assert_eq!(redriven[field], expected, "{field}: {redriven}"); + } + let run = redriven["run_id"].as_str().unwrap().to_owned(); + assert_eq!(redriven["workflow_run_id"], json!(run)); + let (status, again) = request(&app, "POST", &path, json!({"request_id":"rq-1"})).await; + assert_eq!((status, again["run_id"].clone(), again["task_id"].clone()), (StatusCode::OK, json!(run), Value::Null), "{again}"); + let (status, current) = request(&app, "POST", &format!("/api/workflows/rd/runs/{run}/redrive"), json!({})).await; + assert_eq!((status, current["reason"].clone()), (StatusCode::CONFLICT, json!("run_not_failed")), "{current}"); + + let history = run_history(&app, &json!("rd"), &json!(run)).await; + assert_eq!(kinds(&history), ["StartAccepted", "WorkflowStarted", "ActivityCompleted"]); + let started = &history[1]["payload"]; + for (field, expected) in [("continued_from_run_id", json!(source_run)), ("resume_step_sequence", json!(2)), + ("recovery_kind", json!("redrive")), ("workflow_definition_fingerprint", json!(FINGERPRINT))] { + assert_eq!(started[field], expected, "{field}: {started}"); + } + let reused = &history[2]["payload"]; + assert_eq!((reused["sequence"].clone(), reused["activity_type"].clone(), reused["reused_from_run_id"].clone()), + (json!(1), json!("a"), json!(source_run)), "{reused}"); + + // The worker replays the reused step and reschedules only the failed one. + let task = poll(&app, "worker", "workflow").await; + assert_eq!(task["run_id"], json!(run), "{task}"); + let resumed = finish_task(&app, &task, json!([{"type":"schedule_activity","activity_type":"b","arguments":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(resumed.0, StatusCode::OK, "{}", resumed.1); + let activity = poll(&app, "worker", "activity").await; + assert_eq!(activity["activity_type"], "b", "{activity}"); + complete_activity(&app, &activity).await; + let task = poll(&app, "worker", "workflow").await; + let done = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Array(vec![]))}])).await; + assert_eq!(done.1["run_status"], "completed", "{}", done.1); + let history = run_history(&app, &json!("rd"), &json!(run)).await; + let scheduled = history.as_array().unwrap().iter().find(|event| event["event_type"] == "ActivityScheduled").unwrap(); + assert_eq!(scheduled["payload"]["sequence"], 2, "{scheduled}"); + let (_, described) = request(&app, "GET", "/api/workflows/rd", Value::Null).await; + assert_eq!((described["run_number"].clone(), described["continued_from_run_id"].clone(), described["recovery_kind"].clone()), + (json!(2), json!(source_run), json!("redrive")), "{described}"); + + // A run that failed without an activity boundary cannot be redriven. + let plain = start(&app, "rd2").await; + let task = poll(&app, "worker", "workflow").await; + finish_task(&app, &task, json!([{"type":"fail_workflow","message":"x"}])).await; + let (status, refused) = request(&app, "POST", &format!("/api/workflows/rd2/runs/{}/redrive", plain["run_id"].as_str().unwrap()), json!({})).await; + assert_eq!(status, StatusCode::CONFLICT, "{refused}"); + for (field, expected) in [("reason", json!("unrecorded_failure_boundary")), ("command_status", json!("rejected")), + ("outcome", json!("rejected")), ("accepted", json!(false)), ("workflow_run_id", Value::Null)] { + assert_eq!(refused[field], expected, "{field}: {refused}"); + } + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/repair_pass.rs b/rust/tests/execution/repair_pass.rs new file mode 100644 index 000000000..e4597687c --- /dev/null +++ b/rust/tests/execution/repair_pass.rs @@ -0,0 +1,175 @@ +use super::*; + +const PASS: &str = "/api/system/repair/pass"; +const LONG_AGO: &str = "'2020-01-02 03:04:05.000000'"; + +async fn started_on(app: &Router, workflow: &str, queue: &str) -> Value { + let (status, body) = request(app, "POST", "/api/workflows", json!({"workflow_id":workflow,"workflow_type":"echo","task_queue":queue,"input":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + body +} + +/// Counts rows with a query built for this test. +async fn count(database: &TestDatabase, sql: String) -> i64 { + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await +} + +async fn execute(database: &TestDatabase, sql: String) { + database.execute(sqlx::AssertSqlSafe(sql)).await; +} + +fn counts(report: &Value, fields: &[&str]) -> Vec { + fields.iter().map(|field| report[*field].clone()).collect() +} + +// Published PHP's repair pass redispatches overdue ready tasks, returns tasks +// whose lease expired, and creates a lost workflow task. Its filters and loop +// throttle shape what one pass selects. +#[tokio::test] +async fn repair_pass_recovers_tasks_and_runs_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, empty) = request(&app, "POST", PASS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{empty}"); + assert_eq!( + counts(&empty, &["throttled", "selected_total_candidates", "dispatched_tasks", "service_call_timeout_candidates"]), + [json!(false), json!(0), json!(0), json!(0)] + ); + assert_eq!(empty["command_contract_failures"], json!([])); + + let overdue = started_on(&app, "overdue", "repair-a").await; + let leased = started_on(&app, "leased", "repair-b").await; + let stalled = started_on(&app, "stalled", "repair-c").await; + let (overdue, leased, stalled) = + (overdue["run_id"].as_str().unwrap(), leased["run_id"].as_str().unwrap(), stalled["run_id"].as_str().unwrap()); + // Each task was created and dispatched long before the redispatch cutoff. + execute(&database, format!("UPDATE workflow_tasks SET created_at={LONG_AGO},available_at={LONG_AGO},last_dispatched_at={LONG_AGO}")).await; + execute(&database, format!("UPDATE workflow_tasks SET status='leased',lease_owner='gone',lease_expires_at={LONG_AGO} WHERE workflow_run_id='{leased}'")).await; + // The stalled run lost its task. + execute(&database, format!("DELETE FROM workflow_tasks WHERE workflow_run_id='{stalled}'")).await; + execute(&database, format!("UPDATE workflow_run_summaries SET liveness_state='repair_needed',next_task_id=NULL WHERE id='{stalled}'")).await; + + // A pass scoped to one queue selects only its candidates. + let (status, scoped) = request(&app, "POST", PASS, json!({"queue": " repair-a "})).await; + assert_eq!(status, StatusCode::OK, "{scoped}"); + assert_eq!(scoped["queue"], "repair-a"); + assert_eq!( + counts(&scoped, &["selected_existing_task_candidates", "selected_missing_task_candidates", "repaired_existing_tasks", "dispatched_tasks"]), + [json!(1), json!(0), json!(1), json!(1)], + "{scoped}" + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{overdue}' AND status='ready' AND repair_count=1 AND last_dispatched_at>'2021-01-01 00:00:00'")).await, + 1 + ); + + // A pass that respects the loop throttle runs nothing right after another. + let (status, throttled) = request(&app, "POST", PASS, json!({"respect_throttle": true})).await; + assert_eq!(status, StatusCode::OK, "{throttled}"); + assert_eq!(counts(&throttled, &["throttled", "selected_total_candidates"]), [json!(true), json!(0)]); + + // An unthrottled pass returns the expired lease and creates the lost task. + let (status, full) = request(&app, "POST", PASS, json!({"respect_throttle": "0"})).await; + assert_eq!(status, StatusCode::OK, "{full}"); + assert_eq!( + counts(&full, &["throttled", "selected_existing_task_candidates", "selected_missing_task_candidates", "repaired_existing_tasks", "repaired_missing_tasks", "dispatched_tasks"]), + [json!(false), json!(1), json!(1), json!(1), json!(1), json!(2)], + "{full}" + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{leased}' AND status='ready' AND lease_owner IS NULL AND lease_expires_at IS NULL AND repair_count=1")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{stalled}' AND task_type='workflow' AND status='ready' AND repair_count=1 AND last_dispatched_at IS NOT NULL")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_run_summaries WHERE id='{stalled}' AND next_task_id IS NOT NULL")).await, + 1 + ); + + // Nothing is left to repair. + let (status, quiet) = request(&app, "POST", PASS, json!({})).await; + assert_eq!(status, StatusCode::OK, "{quiet}"); + assert_eq!(quiet["selected_total_candidates"], 0, "{quiet}"); +} + +// A run past its deadline gets the task that closes it. An expired lease on +// a running activity is settled by its retry policy, by the scheduler or the +// pass, whichever runs first. A candidate whose recovery fails is reported +// with 207 and keeps the failure as its last error. +#[tokio::test] +async fn repair_pass_creates_deadline_tasks_settles_leases_and_reports_failures() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + + let deadline = started_on(&app, "deadline", "repair-d").await; + let deadline = deadline["run_id"].as_str().unwrap(); + execute(&database, format!("UPDATE workflow_tasks SET status='completed' WHERE workflow_run_id='{deadline}'")).await; + execute(&database, format!("UPDATE workflow_runs SET execution_deadline_at={LONG_AGO} WHERE id='{deadline}'")).await; + + // A running activity whose lease expired. + let started = start(&app, "activity").await; + let task = poll(&app, "worker", "workflow").await; + let command = json!({"type":"schedule_activity","activity_type":"act","arguments":envelope(Payload::Array(vec![]))}); + assert_eq!(finish_task(&app, &task, json!([command])).await.0, StatusCode::OK); + let activity = poll(&app, "worker", "activity").await; + assert!(activity["task_id"].is_string(), "{activity}"); + let run = started["run_id"].as_str().unwrap(); + execute(&database, format!("UPDATE workflow_tasks SET lease_expires_at={LONG_AGO} WHERE workflow_run_id='{run}' AND task_type='activity'")).await; + + // An overdue ready task whose repair counter cannot count another repair. + let exhausted = started_on(&app, "exhausted", "repair-e").await; + let exhausted = exhausted["run_id"].as_str().unwrap(); + let most = match &database { + TestDatabase::Sqlite(_) => i64::MAX, + TestDatabase::Postgres { .. } => i64::from(i32::MAX), + TestDatabase::MySql { .. } => i64::from(u32::MAX), + }; + execute(&database, format!("UPDATE workflow_tasks SET created_at={LONG_AGO},available_at={LONG_AGO},last_dispatched_at={LONG_AGO},repair_count={most} WHERE workflow_run_id='{exhausted}'")).await; + + let (status, report) = request(&app, "POST", PASS, json!({})).await; + assert_eq!(status, StatusCode::MULTI_STATUS, "{report}"); + assert_eq!( + counts(&report, &["deadline_expired_candidates", "deadline_expired_tasks_created"]), + [json!(1), json!(1)], + "{report}" + ); + let failures = report["existing_task_failures"].as_array().unwrap(); + assert_eq!(failures.len(), 1, "{report}"); + assert_eq!(failures[0]["message"], "repair_count_exhausted", "{report}"); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{exhausted}' AND status='ready' AND last_error='repair_count_exhausted'")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{deadline}' AND task_type='workflow' AND status='ready' AND repair_count=1")).await, + 1 + ); + // The lost lease failed the activity once, as its policy allows no retry. + let settled = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let failed = count(&database, format!("SELECT COUNT(*) FROM workflow_history_events WHERE workflow_run_id='{run}' AND event_type='ActivityFailed'")).await; + if failed == 1 { + break; + } + assert!(failed == 0 && tokio::time::Instant::now() < settled, "{failed} lease failures"); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + let task_id = activity["task_id"].as_str().unwrap(); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE id='{task_id}' AND status='cancelled' AND last_error='lease_expired'")).await, + 1 + ); + + // The deadline task is open now, so the run is no longer a candidate. + let (_, again) = request(&app, "POST", PASS, json!({"run_ids": [deadline]})).await; + assert_eq!(again["deadline_expired_candidates"], 0, "{again}"); + + let (status, invalid) = request(&app, "POST", PASS, json!({"run_ids": [""], "queue": "q".repeat(129)})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{invalid}"); + assert_eq!(invalid["errors"]["run_ids.0"], json!(["The run_ids.0 field must be a string.", "The run_ids.0 field must be at least 1 characters."])); + assert_eq!(invalid["errors"]["queue"], json!(["The queue field must not be greater than 128 characters."])); +} diff --git a/rust/tests/execution/run_observation_signals.rs b/rust/tests/execution/run_observation_signals.rs index bc46f7f2d..ca7665c66 100644 --- a/rust/tests/execution/run_observation_signals.rs +++ b/rust/tests/execution/run_observation_signals.rs @@ -313,11 +313,10 @@ async fn run_observation_signals_named_namespace_retains_batched_admission_gate( ]), ) .await; - assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!( - refused.1["reason"], - "development_namespace_command_unqualified" - ); + // A named namespace accepts every command, but native still opens one + // wait at a time. + assert_eq!(refused.0, StatusCode::CONFLICT, "{}", refused.1); + assert_eq!(refused.1["reason"], "pending_durable_operations", "{}", refused.1); assert_eq!( read(&app, "signal-observation", &run, "history").await, before diff --git a/rust/tests/execution/search_attribute_upserts.rs b/rust/tests/execution/search_attribute_upserts.rs index 1c3fd5967..5558b4011 100644 --- a/rust/tests/execution/search_attribute_upserts.rs +++ b/rust/tests/execution/search_attribute_upserts.rs @@ -209,3 +209,23 @@ async fn workflow_lists_filter_and_show_typed_attributes_like_php() { runtime.close().await; database.remove().await; } + +// Published PHP fills in an upsert's registered types before its second +// capability check, so a worker without `typed_search_attributes` cannot +// upsert search attributes even when it sends no `attribute_types`. +#[tokio::test] +async fn an_upsert_needs_the_typed_capability_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "plain-worker", json!(["echo"]), json!([])).await; + define(&app, "region", "keyword").await; + let started = start(&app, "untyped-upsert").await; + let before = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let task = poll(&app, "plain-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, + json!([{"type":"upsert_search_attributes","attributes":{"region":"eu"}}])).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "workflow_metadata_capability_not_advertised", "{body}"); + assert_eq!(body["capability"], "typed_search_attributes", "{body}"); + assert_eq!(run_history(&app, &started["workflow_id"], &started["run_id"]).await, before); +} diff --git a/rust/tests/execution/standalone_activities.rs b/rust/tests/execution/standalone_activities.rs new file mode 100644 index 000000000..6d6063530 --- /dev/null +++ b/rust/tests/execution/standalone_activities.rs @@ -0,0 +1,200 @@ +use super::*; + +fn arguments(values: Vec) -> Value { + envelope(Payload::Array(values)) +} + +async fn start_activity(app: &Router, body: Value) -> (StatusCode, Value) { + request(app, "POST", "/api/activities", body).await +} + +fn kinds(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +fn last<'a>(history: &'a Value, kind: &str) -> &'a Value { + &history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == kind).unwrap()["payload"] +} + +/// Waits for the native scheduler to settle what it owns. +async fn activity_until(app: &Router, id: &str, status: &str) -> Value { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let (_, shown) = request(app, "GET", &format!("/api/activities/{id}"), Value::Null).await; + if shown["status"] == status { + return shown; + } + assert!(tokio::time::Instant::now() < deadline, "{shown}"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } +} + +// Published PHP 2.11.1's `POST /api/activities` writes a host run of type +// `dw.standalone_activity` with StartAccepted, WorkflowStarted and +// ActivityScheduled and a ready activity task. A worker claims it like any +// activity, and its result completes the host run with the same output. +#[tokio::test] +async fn a_standalone_activity_runs_in_its_own_host_run_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!([]), json!(["greet"])).await; + let input = arguments(vec![Payload::String("x".into())]); + let (status, started) = start_activity(&app, json!({"activity_id":"sa-1","activity_type":"greet","task_queue":"test", + "input":input,"business_key":"bk-1","retry_policy":{"max_attempts":3,"backoff_seconds":[0]}, + "start_to_close_timeout_seconds":60})).await; + assert_eq!(status, StatusCode::CREATED, "{started}"); + for (field, expected) in [("activity_id", json!("sa-1")), ("workflow_id", json!("sa-1")), + ("workflow_type", json!("dw.standalone_activity")), ("activity_type", json!("greet")), + ("activity_class", json!("greet")), ("task_queue", json!("test")), ("namespace", json!("default")), + ("status", json!("running")), ("payload_codec", json!("avro")), ("schedule_to_start_deadline_at", Value::Null), + ("command_status", json!("accepted")), ("command_source", json!("control_plane"))] { + assert_eq!(started[field], expected, "{field}: {started}"); + } + let (status, running) = start_activity(&app, json!({"activity_id":"sa-1","activity_type":"greet"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{running}"); + assert_eq!(running["errors"], json!({"activity_id":["Standalone activity [sa-1] is already running."]})); + + let (_, pending) = request(&app, "GET", "/api/activities/sa-1", Value::Null).await; + assert_eq!((pending["activity_status"].clone(), pending["attempt_count"].clone(), pending["attempts"].clone()), + (json!("pending"), json!(0), json!([])), "{pending}"); + assert_eq!(pending["business_key"], "bk-1"); + + let task = poll(&app, "worker", "activity").await; + assert_eq!(task["arguments"], input, "{task}"); + assert_eq!(task["retry_policy"]["max_attempts"], 3, "{task}"); + let (_, claimed) = request(&app, "GET", "/api/activities/sa-1", Value::Null).await; + assert_eq!(claimed["current_attempt"]["activity_attempt_id"], task["activity_attempt_id"], "{claimed}"); + assert_eq!((claimed["current_attempt"]["is_current"].clone(), claimed["current_attempt"]["can_continue"].clone()), + (json!(true), json!(true)), "{claimed}"); + + let path = format!("/api/worker/activity-tasks/{}/complete", task["task_id"].as_str().unwrap()); + let (status, completed) = request(&app, "POST", &path, json!({"lease_owner":task["lease_owner"], + "activity_attempt_id":task["activity_attempt_id"],"result":input})).await; + assert_eq!(status, StatusCode::OK, "{completed}"); + assert_eq!((completed["outcome"].clone(), completed["next_task_id"].clone()), (json!("completed"), Value::Null), "{completed}"); + + let (_, done) = request(&app, "GET", "/api/activities/sa-1", Value::Null).await; + assert_eq!((done["status"].clone(), done["closed_reason"].clone(), done["activity_status"].clone()), + (json!("completed"), json!("completed"), json!("completed")), "{done}"); + assert_eq!(done["result"], input, "{done}"); + assert_eq!(done["current_attempt"]["stop_reason"], "attempt_closed", "{done}"); + let history = run_history(&app, &json!("sa-1"), &started["workflow_run_id"]).await; + assert_eq!(kinds(&history), ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityStarted", + "ActivityCompleted", "WorkflowCompleted"]); + assert_eq!(last(&history, "WorkflowCompleted")["output"], input); + assert_eq!(last(&history, "WorkflowStarted")["business_key"], "bk-1"); + // No workflow task is ever created for a host run. + assert_eq!(poll(&app, "worker", "workflow").await, Value::Null); + + // A closed activity's id starts a new run. + let (status, again) = start_activity(&app, json!({"activity_id":"sa-1","activity_type":"greet","task_queue":"test"})).await; + assert_eq!(status, StatusCode::CREATED, "{again}"); + assert_ne!(again["workflow_run_id"], started["workflow_run_id"]); + let (_, runs) = request(&app, "GET", "/api/workflows/sa-1/runs", Value::Null).await; + assert_eq!(runs["run_count"], 2, "{runs}"); + // Without input the activity has no arguments, as in PHP. + let task = poll(&app, "worker", "activity").await; + assert!(task.get("arguments").is_none(), "{task}"); + let (status, described) = request(&app, "GET", "/api/workflows/sa-1", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!(described["input_envelope"], Value::Null, "{described}"); + runtime.close().await; + database.remove().await; +} + +// A terminal failure or timeout fails the host run with the activity's +// failure, as `closeStandaloneHostRun` and its timeout variant do. +#[tokio::test] +async fn a_standalone_activity_failure_or_timeout_fails_its_host_run_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "worker", json!([]), json!(["greet"])).await; + let (_, started) = start_activity(&app, json!({"activity_id":"sa-2","activity_type":"greet","task_queue":"test"})).await; + let task = poll(&app, "worker", "activity").await; + let path = format!("/api/worker/activity-tasks/{}/fail", task["task_id"].as_str().unwrap()); + let (status, failed) = request(&app, "POST", &path, json!({"lease_owner":task["lease_owner"], + "activity_attempt_id":task["activity_attempt_id"],"failure":{"class":"App\\Boom","message":"boom","type":"Boom"}})).await; + assert_eq!(status, StatusCode::OK, "{failed}"); + assert_eq!(failed["next_task_id"], Value::Null, "{failed}"); + let (_, shown) = request(&app, "GET", "/api/activities/sa-2", Value::Null).await; + assert_eq!((shown["status"].clone(), shown["closed_reason"].clone(), shown["activity_status"].clone()), + (json!("failed"), json!("failed"), json!("failed")), "{shown}"); + let history = run_history(&app, &json!("sa-2"), &started["workflow_run_id"]).await; + assert_eq!(kinds(&history).last().unwrap(), "WorkflowFailed"); + let activity_failed = last(&history, "ActivityFailed"); + let workflow_failed = last(&history, "WorkflowFailed"); + for (field, expected) in [("failure_id", activity_failed["failure_id"].clone()), ("source_kind", json!("activity_execution")), + ("source_id", started["activity_execution_id"].clone()), ("failure_category", json!("activity")), + ("non_retryable", json!(false)), ("exception_type", json!("Boom")), ("exception_class", json!("App\\Boom")), + ("message", json!("boom")), ("exception", activity_failed["exception"].clone())] { + assert_eq!(workflow_failed[field], expected, "{field}: {workflow_failed}"); + } + + let (_, timed) = start_activity(&app, json!({"activity_id":"sa-3","activity_type":"greet","task_queue":"nobody", + "schedule_to_start_timeout_seconds":1})).await; + assert!(timed["schedule_to_start_deadline_at"].is_string(), "{timed}"); + let shown = activity_until(&app, "sa-3", "failed").await; + assert_eq!(shown["closed_reason"], "timed_out", "{shown}"); + let history = run_history(&app, &json!("sa-3"), &timed["workflow_run_id"]).await; + assert_eq!(kinds(&history), ["StartAccepted", "WorkflowStarted", "ActivityScheduled", "ActivityTimedOut", "WorkflowFailed"]); + let workflow_failed = last(&history, "WorkflowFailed"); + assert_eq!(workflow_failed["exception_class"], "Workflow\\V2\\Exceptions\\ActivityTimeoutException"); + assert_eq!((workflow_failed["failure_category"].clone(), workflow_failed["exception_type"].clone()), + (json!("timeout"), Value::Null), "{workflow_failed}"); + assert_eq!(workflow_failed["exception"]["trace"], json!([]), "{workflow_failed}"); + runtime.close().await; + database.remove().await; +} + +// `ActivityController`'s refusals, identifiers and listing. +#[tokio::test] +async fn standalone_activity_requests_are_validated_and_listed_like_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let (status, refused) = start_activity(&app, json!({"activity_id":"bad id!","task_queue":"q", + "retry_policy":{"max_attempts":0,"backoff_seconds":[-1]},"heartbeat_timeout_seconds":0})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert_eq!(refused["errors"], json!({ + "activity_id":["The activity id field format is invalid."], + "activity_type":["The activity type field is required."], + "retry_policy.max_attempts":["The retry policy.max attempts field must be at least 1."], + "heartbeat_timeout_seconds":["The heartbeat timeout seconds field must be at least 1."], + "retry_policy.backoff_seconds.0":["The retry_policy.backoff_seconds.0 field must be at least 0."]}), "{refused}"); + + start(&app, "wf-x").await; + let (status, reserved) = start_activity(&app, json!({"activity_id":"wf-x","activity_type":"greet"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{reserved}"); + assert_eq!(reserved["errors"]["activity_id"], + json!(["Identifier [wf-x] is already reserved for workflow type [echo] and cannot be reused for a standalone activity."])); + for missing in ["wf-x", "nope"] { + let (status, absent) = request(&app, "GET", &format!("/api/activities/{missing}"), Value::Null).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{absent}"); + assert_eq!(absent, json!({"message":"Standalone activity not found.","reason":"activity_not_found"})); + } + + let mut started = Vec::new(); + for _ in 0..3 { + let (status, generated) = start_activity(&app, json!({"activity_type":"greet"})).await; + assert_eq!(status, StatusCode::CREATED, "{generated}"); + assert_eq!(generated["task_queue"], "default", "{generated}"); + started.push(generated["activity_id"].clone()); + } + let (_, page) = request(&app, "GET", "/api/activities?page_size=2", Value::Null).await; + assert_eq!(page["activity_count"], 2, "{page}"); + assert_eq!(page["activities"][0]["activity_id"], started[2], "newest first: {page}"); + assert_eq!(page["next_page_token"], "Mg==", "{page}"); + let (_, rest) = request(&app, "GET", "/api/activities?page_size=2&next_page_token=Mg%3D%3D", Value::Null).await; + assert_eq!((rest["activity_count"].clone(), rest["next_page_token"].clone()), (json!(1), Value::Null), "{rest}"); + assert_eq!(rest["activities"][0]["status_bucket"], "running", "{rest}"); + let (_, completed) = request(&app, "GET", "/api/activities?status=completed", Value::Null).await; + assert_eq!(completed["activities"], json!([]), "{completed}"); + let (status, invalid) = request(&app, "GET", "/api/activities?status=cancelled&page_size=500", Value::Null).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{invalid}"); + assert_eq!(invalid["errors"], json!({"status":["The selected status is invalid."], + "page_size":["The page size field must not be greater than 200."]})); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/structural_limits.rs b/rust/tests/execution/structural_limits.rs new file mode 100644 index 000000000..42534b31e --- /dev/null +++ b/rust/tests/execution/structural_limits.rs @@ -0,0 +1,223 @@ +use super::*; +use durable_workflow_server::runtime::StructuralLimits; + +/// Counts rows with a query built for this test. +async fn count(database: &TestDatabase, sql: String) -> i64 { + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await +} + +async fn limited(database: &TestDatabase, signals: usize, updates: usize, payload: usize) -> (Runtime, Router) { + let limits = StructuralLimits { + pending_signals: Some(signals), + pending_updates: Some(updates), + payload_bytes: Some(payload), + }; + let runtime = database.open().await.unwrap().with_structural_limits(limits); + (runtime.clone(), router(runtime)) +} + +/// A worker whose `note` signal and update take one string. +async fn note_worker(app: &Router) { + let note = json!([{"name":"note","parameters":[{"name":"value","type":"string","required":true,"variadic":false,"default_available":false,"allows_null":false}]}]); + let (status, body) = request(app, "POST", "/api/worker/register", json!({ + "worker_id":"worker","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["workflow_updates"],"workflow_command_contracts":{"echo":{ + "signals":["note"],"signal_contracts":note,"updates":["note"],"update_validators":[],"update_contracts":note + }}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +fn note(value: &str) -> Value { + arguments(vec![Payload::String(value.into())]) +} + +fn arguments(values: Vec) -> Value { + envelope(Payload::Array(values)) +} + +/// A string argument whose serialized blob exceeds a 300-byte ceiling. +fn oversized() -> Value { + arguments(vec![Payload::String("x".repeat(400))]) +} + +fn events_of(history: &Value, kind: &str) -> Vec { + history.as_array().unwrap().iter().filter(|event| event["event_type"] == kind).cloned().collect() +} + +// Published PHP 2.11.1, `DW_V2_LIMIT_PENDING_SIGNALS=2` and +// `DW_V2_LIMIT_PAYLOAD_SIZE_BYTES=300`: a signal past either ceiling answers +// 409 `structural_limit_exceeded` with no outcome and records a rejected +// command and signal, but no history. An oversized signal keeps no arguments. +#[tokio::test] +async fn signals_over_a_structural_limit_are_rejected_durably_like_php() { + let database = TestDatabase::new().await; + let (runtime, app) = limited(&database, 2, 500, 300).await; + note_worker(&app).await; + let started = start(&app, "limited-signals").await; + let path = "/api/workflows/limited-signals/signal/note"; + + let (status, big) = request(&app, "POST", path, json!({"input":oversized()})).await; + assert_eq!(status, StatusCode::CONFLICT, "{big}"); + for (field, expected) in [ + ("outcome", Value::Null), + ("command_status", json!("rejected")), + ("reason", json!("structural_limit_exceeded")), + ("rejection_reason", json!("structural_limit_exceeded")), + ("validation_errors", json!([])), + ("accepted", json!(false)), + ] { + assert_eq!(big[field], expected, "{field}: {big}"); + } + let empty = arguments(vec![])["blob"].as_str().unwrap().to_owned(); + let command = big["command_id"].as_str().unwrap(); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{command}' AND status='rejected' AND outcome IS NULL AND rejection_reason='structural_limit_exceeded' AND arguments='{empty}'")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_commands WHERE id='{command}' AND status='rejected' AND outcome IS NULL")).await, + 1 + ); + + for value in ["a", "b"] { + let (status, accepted) = request(&app, "POST", path, json!({"input":note(value)})).await; + assert!(status.is_success(), "{accepted}"); + assert_eq!(accepted["outcome"], "signal_received", "{accepted}"); + } + let (status, pending) = request(&app, "POST", path, json!({"input":note("c")})).await; + assert_eq!(status, StatusCode::CONFLICT, "{pending}"); + assert_eq!((pending["reason"].clone(), pending["outcome"].clone()), (json!("structural_limit_exceeded"), Value::Null), "{pending}"); + assert_eq!(pending["validation_errors"], json!([]), "{pending}"); + let command = pending["command_id"].as_str().unwrap(); + // A count refusal keeps the signal's arguments. + let three = note("c")["blob"].as_str().unwrap().to_owned(); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{command}' AND status='rejected' AND arguments='{three}'")).await, + 1 + ); + + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert_eq!(events_of(&history, "SignalReceived").len(), 2, "{history}"); + assert!(events_of(&history, "SignalRejected").is_empty(), "{history}"); + runtime.close().await; + database.remove().await; +} + +// The same reference with `DW_V2_LIMIT_PENDING_UPDATES=1`: an oversized +// update and one past the accepted-update ceiling are rejected durably, with +// the limit's message, no arguments, and the limit fields in the answer. +#[tokio::test] +async fn updates_over_a_structural_limit_are_rejected_durably_like_php() { + let database = TestDatabase::new().await; + let (runtime, app) = limited(&database, 5000, 1, 300).await; + note_worker(&app).await; + let started = start(&app, "limited-updates").await; + let task = poll(&app, "worker", "workflow").await; + let timer = json!([{"type":"start_timer","delay_seconds":3600}]); + assert_eq!(finish_task(&app, &task, timer).await.0, StatusCode::OK); + let path = "/api/workflows/limited-updates/update/note"; + + let blob = oversized()["blob"].as_str().unwrap().len(); + let (status, big) = request(&app, "POST", path, json!({"input":oversized(),"request_id":"big"})).await; + assert_eq!(status, StatusCode::CONFLICT, "{big}"); + let message = format!("Structural limit exceeded: payload size {blob} bytes (limit 300 bytes)."); + for (field, expected) in [ + ("outcome", Value::Null), + ("command_status", json!("rejected")), + ("update_status", json!("rejected")), + ("reason", json!("structural_limit_exceeded")), + ("validation_errors", json!({"arguments":[message]})), + ("structural_limit_kind", json!("payload_size")), + ("structural_limit_value", json!(blob)), + ("structural_limit_configured", json!(300)), + ("accepted", json!(false)), + ] { + assert_eq!(big[field], expected, "{field}: {big}"); + } + // Like PHP, the answer names the requester and when it was rejected. + assert_eq!(big["principal"]["type"], "auth:token", "{big}"); + assert!(big["principal"]["id"].is_string(), "{big}"); + assert_eq!(big["rejected_at"], big["closed_at"], "{big}"); + // A retried request returns the original rejection with its limit. + let (status, retried) = request(&app, "POST", path, json!({"input":oversized(),"request_id":"big"})).await; + assert_eq!(status, StatusCode::CONFLICT, "{retried}"); + assert_eq!(retried, big); + + let (status, accepted) = request(&app, "POST", path, json!({"input":note("a"),"wait_for":"accepted"})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{accepted}"); + let (status, pending) = request(&app, "POST", path, json!({"input":note("b")})).await; + assert_eq!(status, StatusCode::CONFLICT, "{pending}"); + assert_eq!(pending["validation_errors"], json!({"arguments":["Structural limit exceeded: 1 pending updates (limit 1)."]}), "{pending}"); + assert_eq!( + (pending["structural_limit_kind"].clone(), pending["structural_limit_value"].clone(), pending["structural_limit_configured"].clone()), + (json!("pending_update_count"), json!(1), json!(1)), + "{pending}" + ); + + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let rejected = events_of(&history, "UpdateRejected"); + assert_eq!(rejected.len(), 2, "{history}"); + assert_eq!(rejected[0]["payload"]["arguments"], arguments(vec![]), "{history}"); + assert_eq!(rejected[0]["payload"]["validation_errors"], big["validation_errors"]); + assert_eq!(rejected[1]["payload"]["arguments"], arguments(vec![]), "{history}"); + let accepted = events_of(&history, "UpdateAccepted"); + assert_eq!(accepted.len(), 1, "{history}"); + // PHP records a request id only when the request named one. + assert!(accepted[0]["payload"].get("request_id").is_none(), "{history}"); + runtime.close().await; + database.remove().await; +} + +// `ActivityOutcomeRecorder` measures the result it would store. Over the +// payload ceiling the activity fails, without a retry, with PHP's +// structural-limit exception, and the worker is told it completed. +#[tokio::test] +async fn an_oversized_activity_result_fails_the_activity_like_php() { + let database = TestDatabase::new().await; + let (runtime, app) = limited(&database, 5000, 500, 300).await; + register(&app, "worker", json!(["echo"]), json!(["act"])).await; + let started = start(&app, "limited-result").await; + let task = poll(&app, "worker", "workflow").await; + let command = json!({"type":"schedule_activity","activity_type":"act","arguments":arguments(vec![]), + "retry_policy":{"max_attempts":3,"backoff_seconds":[0]}}); + assert_eq!(finish_task(&app, &task, json!([command])).await.0, StatusCode::OK); + let activity = poll(&app, "worker", "activity").await; + let (status, answer) = request(&app, "POST", + &format!("/api/worker/activity-tasks/{}/complete", activity["task_id"].as_str().unwrap()), + json!({"lease_owner":activity["lease_owner"],"activity_attempt_id":activity["activity_attempt_id"],"result":oversized()})).await; + assert_eq!(status, StatusCode::OK, "{answer}"); + assert_eq!((answer["outcome"].clone(), answer["recorded"].clone()), (json!("completed"), json!(true)), "{answer}"); + assert!(answer["next_task_id"].is_string(), "{answer}"); + + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(events_of(&history, "ActivityCompleted").is_empty(), "{history}"); + assert!(events_of(&history, "ActivityRetryScheduled").is_empty(), "{history}"); + let failed = &events_of(&history, "ActivityFailed")[0]["payload"]; + let blob = oversized()["blob"].as_str().unwrap().len(); + let message = format!("Structural limit exceeded: payload size {blob} bytes (limit 300 bytes)."); + for (field, expected) in [ + ("failure_category", json!("structural_limit")), + ("non_retryable", json!(true)), + ("exception_type", Value::Null), + ("exception_class", json!("Workflow\\V2\\Exceptions\\StructuralLimitExceededException")), + ("message", json!(message)), + ("code", json!(0)), + ("exception", json!({"message":message,"code":0})), + ("structural_limit_kind", json!("payload_size")), + ("structural_limit_value", json!(blob)), + ("structural_limit_configured", json!(300)), + ] { + assert_eq!(failed[field], expected, "{field}: {failed}"); + } + let run = started["run_id"].as_str().unwrap(); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM workflow_failures WHERE workflow_run_id='{run}' AND failure_category='structural_limit' AND non_retryable=TRUE")).await, + 1 + ); + assert_eq!( + count(&database, format!("SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id='{run}' AND status='failed' AND result IS NULL")).await, + 1 + ); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/system_metrics.rs b/rust/tests/execution/system_metrics.rs new file mode 100644 index 000000000..2e1fbcfa3 --- /dev/null +++ b/rust/tests/execution/system_metrics.rs @@ -0,0 +1,138 @@ +use super::*; + +const METRICS: &str = "/api/system/metrics"; + +async fn system_metrics(app: &Router, method: &str) -> Value { + let (status, body) = request(app, method, METRICS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["namespace"], "default"); + let families: Vec<&String> = body["metrics"].as_object().unwrap().keys().collect(); + assert_eq!(families.len(), 5, "{body}"); + for (name, family) in body["metrics"].as_object().unwrap() { + assert_eq!( + body["cardinality"]["metric_label_sets"][name], family["label_cardinality_policy"], + "{name}" + ); + } + body["metrics"].clone() +} + +async fn start_type(app: &Router, workflow: &str, workflow_type: &str) { + let (status, body) = request(app, "POST", "/api/workflows", json!({"workflow_id": workflow, + "workflow_type": workflow_type, "task_queue": "test", + "input": envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +#[tokio::test] +async fn system_metrics_count_a_failed_workflow_task_and_the_namespace_state_like_published_php() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let empty = system_metrics(&app, "GET").await; + assert_eq!( + empty["dw_workflow_task_consecutive_failures"], + json!({"max_consecutive_failures": 0, "failed_task_count": 0, "workflow_type_count": 0, + "workflow_type_limit": 20, "workflow_types_truncated": false, "suppressed_workflow_type_count": 0, + "suppressed_failed_task_count": 0, "label_cardinality_policy": {"namespace": "request_scope_not_label", + "workflow_type": {"cardinality_class": "bounded_series", "limit": 20, + "selection": "top_by_max_consecutive_failures_then_name"}}, + "by_workflow_type": []}) + ); + assert_eq!(empty["dw_projection_drift_total"]["total"], 0); + let usage = &empty["dw_namespace_durable_state_usage"]; + assert_eq!(usage["usage"].as_object().unwrap().len(), 16); + assert!(usage["usage"].as_object().unwrap().values().all(|count| count == 0), "{usage}"); + assert_eq!(usage["rejections_by_reason"].as_object().unwrap().len(), 17); + assert_eq!(usage["rejections_by_reason"]["namespace_durable_state_quota_unavailable"], 0); + assert!(usage["limits"].as_object().unwrap().values().all(Value::is_null)); + assert_eq!(empty["dw_runtime_external_payload_namespace_usage"]["used_objects"], 0); + assert_eq!(empty["dw_namespace_request_admission_rejections"]["max_requests_per_minute"], Value::Null); + + register(&app, "metrics-worker", json!(["echo", "other"]), json!([])).await; + start_type(&app, "metrics-a", "echo").await; + start_type(&app, "metrics-b", "other").await; + let task = poll(&app, "metrics-worker", "workflow").await; + assert_eq!(task["workflow_type"], "echo", "{task}"); + let (status, body) = request(&app, "POST", + &format!("/api/worker/workflow-tasks/{}/fail", task["task_id"].as_str().unwrap()), + json!({"lease_owner": task["lease_owner"], "workflow_task_attempt": task["workflow_task_attempt"], + "failure": {"message": "metrics failure", "type": "RuntimeException"}})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + let read = system_metrics(&app, "GET").await; + assert_eq!(read, system_metrics(&app, "POST").await); + let failures = &read["dw_workflow_task_consecutive_failures"]; + assert_eq!( + [&failures["max_consecutive_failures"], &failures["failed_task_count"], &failures["workflow_type_count"]], + [&json!(1), &json!(1), &json!(1)] + ); + assert_eq!( + failures["by_workflow_type"], + json!([{"workflow_type": "echo", "max_consecutive_failures": 1, "failed_task_count": 1}]) + ); + let usage = &read["dw_namespace_durable_state_usage"]["usage"]; + for (resource, count) in [ + ("workflow_instances", 2), + ("workflow_runs", 2), + ("open_workflow_runs", 2), + ("worker_registrations", 1), + ("workflow_tasks", 3), + ("pending_workflow_tasks", 2), + ("workflow_commands", 2), + ("schedules", 0), + ("workflow_timers", 0), + ("workflow_streams", 0), + ] { + assert_eq!(usage[resource], count, "{resource}: {usage}"); + } +} + +#[tokio::test] +async fn system_metrics_bound_the_failure_series_and_report_server_wide_summary_drift() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + for index in 0..21 { + start_type(&app, &format!("bounded-{index:02}"), &format!("type-{index:02}")).await; + } + database.execute("UPDATE workflow_tasks SET status='failed', attempt_count=1 WHERE task_type='workflow'").await; + database.execute("UPDATE workflow_tasks SET attempt_count=3 WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE workflow_type='type-20')").await; + database.execute("DELETE FROM workflow_run_summaries WHERE id IN (SELECT id FROM workflow_runs WHERE workflow_type='type-05')").await; + + let metrics = system_metrics(&app, "GET").await; + let failures = &metrics["dw_workflow_task_consecutive_failures"]; + assert_eq!( + [ + &failures["max_consecutive_failures"], + &failures["failed_task_count"], + &failures["workflow_type_count"], + &failures["workflow_types_truncated"], + &failures["suppressed_workflow_type_count"], + &failures["suppressed_failed_task_count"], + ], + [&json!(3), &json!(21), &json!(21), &json!(true), &json!(1), &json!(1)] + ); + let shown: Vec<&str> = failures["by_workflow_type"] + .as_array() + .unwrap() + .iter() + .map(|row| row["workflow_type"].as_str().unwrap()) + .collect(); + let mut expected = vec!["type-20".to_owned()]; + expected.extend((0..19).map(|index| format!("type-{index:02}"))); + assert_eq!(shown, expected); + assert_eq!(failures["by_workflow_type"][0]["max_consecutive_failures"], 3); + + let drift = &metrics["dw_projection_drift_total"]; + assert_eq!( + [&drift["total"], &drift["tables_with_drift"], &drift["table_count"], &drift["scope"]], + [&json!(1), &json!(1), &json!(5), &json!("server")] + ); + assert_eq!( + drift["by_table"], + json!([{"table": "run_summaries", "needs_rebuild": 1}, {"table": "run_waits", "needs_rebuild": 0}, + {"table": "run_timeline_entries", "needs_rebuild": 0}, {"table": "run_timer_entries", "needs_rebuild": 0}, + {"table": "run_lineage_entries", "needs_rebuild": 0}]) + ); +} diff --git a/rust/tests/execution/task_queue_build_ids.rs b/rust/tests/execution/task_queue_build_ids.rs index 6b922fcc1..4beb3de2b 100644 --- a/rust/tests/execution/task_queue_build_ids.rs +++ b/rust/tests/execution/task_queue_build_ids.rs @@ -161,15 +161,9 @@ async fn build_ids_group_workers_and_pending_tasks_as_php_does() { assert_eq!(status, StatusCode::OK, "{body}"); assert_eq!(body, expected); - // Promote, drain and resume change nothing: native keeps no rollouts. - for action in ["promote", "drain", "resume"] { - let (status, refused) = request(&app, "POST", &format!("{PATH}/{action}"), json!({"build_id":"v2"})).await; - assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); - assert_eq!( - refused, - json!({"reason":"task_queue_build_id_rollout_unqualified","message":"task_queue_build_id_rollout_unqualified"}) - ); - } + // Promoting a build no worker or rollout knows is refused and changes nothing. + let (status, refused) = request(&app, "POST", &format!("{PATH}/promote"), json!({"build_id":"v404"})).await; + assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("unknown_deployment")), "{refused}"); assert_eq!(request(&app, "GET", PATH, Value::Null).await.1, expected); runtime.close().await; database.remove().await; @@ -190,8 +184,7 @@ async fn an_empty_queue_has_no_build_ids() { } // The route is namespace-scoped and needs an operator or administrator, as -// the other task queue routes do; rollout changes are refused in every -// namespace. +// the other task queue routes do; rollouts belong to their namespace. #[tokio::test] async fn build_ids_follow_task_queue_roles_and_namespaces() { let database = TestDatabase::new().await; @@ -217,7 +210,8 @@ async fn build_ids_follow_task_queue_roles_and_namespaces() { .collect(); assert_eq!( cohorts, - vec![(json!("nv1"), json!("active"), json!("idle")), (Value::Null, json!("no_workers"), json!("pending"))], + // The start is pinned to the queue's one build, as on published 2.11.1. + vec![(json!("nv1"), json!("active"), json!("pending"))], "{named}" ); let (status, default) = call(&app, "admin-token", "default", "GET", PATH, Value::Null).await; @@ -236,14 +230,22 @@ async fn build_ids_follow_task_queue_roles_and_namespaces() { assert_eq!(unknown.0, StatusCode::NOT_FOUND, "{}", unknown.1); assert_eq!(unknown.1["reason"], "namespace_not_found"); assert_eq!(call(&app, "operator-token", "missing", "GET", PATH, Value::Null).await, unknown); + // nv1 is only registered in `builds`, so only there can it be promoted. + let (status, promoted) = call(&app, "operator-token", "builds", "POST", &format!("{PATH}/promote"), json!({"build_id":"nv1"})).await; + assert_eq!((status, promoted["new_start_selected"].clone()), (StatusCode::OK, json!(true)), "{promoted}"); + let (status, refused) = call(&app, "operator-token", "default", "POST", &format!("{PATH}/promote"), json!({"build_id":"nv1"})).await; + assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("unknown_deployment")), "{refused}"); for namespace in ["default", "builds"] { - for action in ["promote", "drain", "resume"] { - let (status, refused) = + for (action, intent) in [("drain", "draining"), ("resume", "active")] { + let (status, changed) = call(&app, "operator-token", namespace, "POST", &format!("{PATH}/{action}"), json!({"build_id":"nv1"})).await; - assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); - assert_eq!(refused["reason"], "task_queue_build_id_rollout_unqualified"); + assert_eq!((status, changed["drain_intent"].clone(), changed["namespace"].clone()), (StatusCode::OK, json!(intent), json!(namespace)), "{changed}"); } } + let (_, named) = call(&app, "operator-token", "builds", "GET", PATH, Value::Null).await; + assert_eq!(named["build_ids"][0]["new_start_selected"], true, "{named}"); + let (_, default) = call(&app, "operator-token", "default", "GET", PATH, Value::Null).await; + assert!(default["build_ids"].as_array().unwrap().iter().all(|entry| entry["new_start_selected"] == false), "{default}"); runtime.close().await; database.remove().await; } diff --git a/rust/tests/execution/timed_condition_waits.rs b/rust/tests/execution/timed_condition_waits.rs index 34edd1a76..bc04cbef9 100644 --- a/rust/tests/execution/timed_condition_waits.rs +++ b/rust/tests/execution/timed_condition_waits.rs @@ -677,17 +677,18 @@ async fn invalid_condition_timeouts_preserve_the_original_batch_and_lease() { let run = namespaces::start_scoped(&app, "default", "invalid-condition-timeout").await; let task = claim(&app, "default", "condition-worker", 0).await; let prefix = read(&app, "default", &run, "history").await; - for seconds in [ - json!(-1), - json!(1.5), - json!(true), - json!("1"), - json!([]), - json!(i64::MAX), + // PHP's rule and normalizer refusals, then native's own for a timeout too + // large for a timestamp, which PHP accepts. "1" is an integer to PHP. + for (seconds, reason) in [ + (json!(-1), "validation_failed"), + (json!(1.5), "validation_failed"), + (json!(true), "validation_failed"), + (json!([]), "validation_failed"), + (json!(i64::MAX), "invalid_condition_timeout"), ] { let result=finish(&app,"default",&task,json!([{"type":"record_version_marker","change_id":"condition-atomic","version":1,"min_supported":1,"max_supported":1},condition(seconds,"logical:0")])).await; assert_eq!(result.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", result.1); - assert_eq!(result.1["reason"], "invalid_condition_timeout"); + assert_eq!(result.1["reason"], reason, "{}", result.1); assert_eq!(read(&app, "default", &run, "history").await, prefix); } let mut untimed = condition(Value::Null, "logical:0"); diff --git a/rust/tests/execution/timed_signal_waits.rs b/rust/tests/execution/timed_signal_waits.rs index 013b273aa..9622c9d96 100644 --- a/rust/tests/execution/timed_signal_waits.rs +++ b/rust/tests/execution/timed_signal_waits.rs @@ -422,13 +422,14 @@ async fn invalid_timeouts_refuse_the_whole_batch_and_preserve_the_lease() { let run = start(&app, "invalid-signal-timeout").await; let task = poll(&app, "invalid-signal-timeout", "workflow").await; let prefix = run_history(&app, &run["workflow_id"], &run["run_id"]).await; - for timeout in [ - json!(-1), - json!(1.5), - json!(true), - json!("1"), - json!([]), - json!(i64::MAX), + // PHP's rule and normalizer refusals, then native's own for a timeout too + // large for a timestamp, which PHP accepts. "1" is an integer to PHP. + for (timeout, reason) in [ + (json!(-1), "validation_failed"), + (json!(1.5), "validation_failed"), + (json!(true), "validation_failed"), + (json!([]), "validation_failed"), + (json!(i64::MAX), "invalid_signal_timeout"), ] { let response=finish_task(&app,&task,json!([{"type":"record_version_marker","change_id":"atomic-timeout","version":1,"min_supported":1,"max_supported":1},{"type":"open_signal_wait","signal_name":"payload","timeout_seconds":timeout}])).await; assert_eq!( @@ -437,7 +438,7 @@ async fn invalid_timeouts_refuse_the_whole_batch_and_preserve_the_lease() { "{}", response.1 ); - assert_eq!(response.1["reason"], "invalid_signal_timeout"); + assert_eq!(response.1["reason"], reason, "{}", response.1); assert_eq!( run_history(&app, &run["workflow_id"], &run["run_id"]).await, prefix @@ -826,7 +827,7 @@ async fn missing_or_changed_wait_projection_cannot_gain_summary_authority() { } #[tokio::test] -async fn named_batched_timeout_wait_keeps_its_existing_admission_gate() { +async fn named_batched_timeout_wait_is_admitted_like_the_default_namespace() { let database = TestDatabase::new().await; let runtime = database.open().await.unwrap(); let app = router(runtime.clone()); @@ -855,19 +856,14 @@ async fn named_batched_timeout_wait_keeps_its_existing_admission_gate() { {"type":"open_signal_wait","signal_name":"payload","timeout_seconds":1} ]), ); - let refused = namespaces::scoped(&app, "timed-signals", "POST", &path, command).await; - assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!( - refused.1["reason"], - "development_namespace_command_unqualified" - ); - assert_eq!( - task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_timers").await, - 0 - ); + // A named namespace takes the batch as the default namespace does: a + // wait whose timeout has already passed does not hold the next one. + let accepted = namespaces::scoped(&app, "timed-signals", "POST", &path, command).await; + assert_eq!(accepted.0, StatusCode::OK, "{}", accepted.1); + assert_eq!(accepted.1["run_status"], "waiting", "{}", accepted.1); assert_eq!( task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_run_waits").await, - 0 + 2 ); assert_eq!(task["run_id"], run["run_id"]); runtime.close().await; diff --git a/rust/tests/execution/update_routing.rs b/rust/tests/execution/update_routing.rs new file mode 100644 index 000000000..7d67f63e1 --- /dev/null +++ b/rust/tests/execution/update_routing.rs @@ -0,0 +1,44 @@ +use super::*; + +// Native delivers an accepted update only to a worker that advertises +// `workflow_updates` and declares the update for the run's type, as the PHP, +// Python and Rust SDKs do exactly when they register its handler. Published +// PHP 2.11.1 hands the update to any worker of the type, so during a rolling +// deploy a worker without the handler fails it (`UnknownUpdate`). Server #690 +// records that defect. +#[tokio::test] +async fn accepted_updates_reach_only_workers_that_declare_them() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let contract = json!([{"name":"u","parameters":[{"name":"value","type":"string","required":true,"variadic":false, + "default_available":false,"allows_null":false}]}]); + for (worker, capabilities, updates, contracts) in [ + ("old-worker", json!([]), json!([]), json!([])), + ("new-worker", json!(["workflow_updates"]), json!(["u"]), contract), + ] { + let (status, body) = request(&app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":"test", + "runtime":"python","supported_workflow_types":["echo"],"supported_activity_types":[],"capabilities":capabilities, + "workflow_command_contracts":{"echo":{"signals":[],"queries":[],"updates":updates,"update_validators":[], + "update_contracts":contracts}}})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + } + let (status, started) = request(&app, "POST", "/api/workflows", json!({"workflow_id":"rolling","workflow_type":"echo", + "task_queue":"test","input":envelope(Payload::Array(vec![]))})).await; + assert_eq!(status, StatusCode::CREATED, "{started}"); + let task = poll(&app, "new-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":3600}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (status, update) = request(&app, "POST", "/api/workflows/rolling/update/u", + json!({"input":envelope(Payload::Array(vec![Payload::String("x".into())])),"wait_for":"accepted"})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{update}"); + assert_eq!(update["update_status"], "accepted", "{update}"); + + // The worker without the handler is never offered the update; the one + // that declares it is. + assert_eq!(poll(&app, "old-worker", "workflow").await, Value::Null); + let task = poll(&app, "new-worker", "workflow").await; + assert_eq!(task["workflow_update_id"], update["update_id"], "{task}"); + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/version_markers.rs b/rust/tests/execution/version_markers.rs index 934520fb8..784a24827 100644 --- a/rust/tests/execution/version_markers.rs +++ b/rust/tests/execution/version_markers.rs @@ -58,7 +58,8 @@ async fn markers_survive_restart_and_advance_later_authored_turns_once() { let started = start(&app, "version-markers").await; let task = poll(&app, "marker-worker", "workflow").await; let mut before = marker(" \tfirst λ\n"); - before["payload_codec"] = json!("newer-ignored-codec"); + // PHP refuses an unknown codec on any command, a marker included. + before["payload_codec"] = json!("avro"); before["future_metadata"] = json!({"ignored":true}); let commands = json!([before, activity_command(), marker("after-activity")]); assert_eq!( @@ -193,30 +194,26 @@ async fn invalid_marker_batches_preserve_original_lease_and_full_history() { let started = start(&app, "invalid-version-markers").await; let task = poll(&app, "marker-worker", "workflow").await; let prefix = run_history(&app, &started["workflow_id"], &started["run_id"]).await; - for (field, value) in [ - ("change_id", json!(" \t\n")), - ("change_id", json!(1)), - ("version", json!(1.0)), - ("version", json!("1")), - ("version", Value::Null), - ("version", json!(2)), - ("version", json!(-2)), - ("min_supported", json!(2)), - ("max_supported", json!(-2)), - ("min_supported", json!("-1")), - ("max_supported", Value::Null), - ("retry_policy", json!({"max_attempts":2})), - ("delay_seconds", json!(1)), + // Published PHP 2.11.1's answers: a malformed marker or option is a 422, + // and a version outside its range is the bridge's 409. Digit strings such + // as "1" and "-1" are integers to PHP. + for (field, value, status) in [ + ("change_id", json!(" \t\n"), StatusCode::UNPROCESSABLE_ENTITY), + ("change_id", json!(1), StatusCode::UNPROCESSABLE_ENTITY), + ("version", json!(1.0), StatusCode::UNPROCESSABLE_ENTITY), + ("version", Value::Null, StatusCode::UNPROCESSABLE_ENTITY), + ("version", json!(2), StatusCode::CONFLICT), + ("version", json!(-2), StatusCode::CONFLICT), + ("min_supported", json!(2), StatusCode::CONFLICT), + ("max_supported", json!(-2), StatusCode::CONFLICT), + ("max_supported", Value::Null, StatusCode::UNPROCESSABLE_ENTITY), + ("retry_policy", json!({"max_attempts":2}), StatusCode::UNPROCESSABLE_ENTITY), + ("delay_seconds", json!(1), StatusCode::UNPROCESSABLE_ENTITY), ] { let mut bad = marker("invalid"); bad[field] = value; let result = finish_task(&app, &task, json!([marker("must-not-commit"), bad])).await; - assert_eq!( - result.0, - StatusCode::UNPROCESSABLE_ENTITY, - "{field}: {}", - result.1 - ); + assert_eq!(result.0, status, "{field}: {}", result.1); assert_eq!( run_history(&app, &started["workflow_id"], &started["run_id"]).await, prefix, diff --git a/rust/tests/execution/waiting_history.rs b/rust/tests/execution/waiting_history.rs index 293d55863..d44c15691 100644 --- a/rust/tests/execution/waiting_history.rs +++ b/rust/tests/execution/waiting_history.rs @@ -63,8 +63,14 @@ async fn waiting_reports_preserve_original_activity_and_history_across_restart() assert_eq!(prefix.as_array().unwrap().len(), 5); assert_eq!(prefix[4]["event_type"], "TimerFired"); let body = waiting(&replay, failure); - let (status, acknowledgement) = report(&app, &replay, body.clone()).await; + let (status, mut acknowledgement) = report(&app, &replay, body.clone()).await; assert_eq!(status, StatusCode::OK, "{acknowledgement}"); + // Every worker response carries PHP's worker envelope. + assert_eq!(acknowledgement["protocol_version"], "1.20", "{acknowledgement}"); + assert!(acknowledgement["server_capabilities"].is_object(), "{acknowledgement}"); + for field in ["protocol_version", "server_capabilities"] { + acknowledgement.as_object_mut().unwrap().remove(field); + } assert_eq!( acknowledgement, json!({"task_id":replay["task_id"],"workflow_task_attempt":replay["workflow_task_attempt"], diff --git a/rust/tests/execution/workflow_failures.rs b/rust/tests/execution/workflow_failures.rs index 4a5751d3e..f27d69fea 100644 --- a/rust/tests/execution/workflow_failures.rs +++ b/rust/tests/execution/workflow_failures.rs @@ -67,7 +67,8 @@ async fn workflow_failure_is_terminal_atomic_and_retains_one_receipt_after_cold_ assert_eq!(payload["exception_class"], "DomainException"); assert_eq!(payload["exception_type"], "domain-v1"); assert_eq!(payload["non_retryable"], true); - assert_eq!(payload["exception"]["message"], " inner failure λ "); + // Laravel's TrimStrings trims every string in the request. + assert_eq!(payload["exception"]["message"], "inner failure λ"); assert_eq!(payload["exception"]["code"], 9007199254740993_i64); assert_eq!( payload["exception"]["properties"], @@ -302,17 +303,18 @@ async fn workflow_failure_rejects_malformed_batches_and_stale_attempts_without_m let run = start(&app, "workflow-failure-invalid").await; let task = poll(&app, "failure-worker", "workflow").await; let original = history(&app, &run).await; - for commands in [ - json!([{"type":"fail_workflow","message":" \t\u{000b}"}]), - json!([{"type":"fail_workflow","message":"failure","non_retryable":"true"}]), - json!([{"type":"start_timer","delay_seconds":5},{"type":"fail_workflow","message":"failure","failed_step_sequence":1}]), - json!([{"type":"fail_workflow","message":"one"},{"type":"fail_workflow","message":"two"}]), - json!([{"type":"fail_workflow","message":"one"},{"type":"complete_workflow","result":envelope(Payload::Null)}]), - json!([{"type":"fail_workflow","message":"one"},{"type":"unknown"}]), + // Two terminal commands are the bridge's 409 `invalid_commands`, as PHP's. + for (commands, expected) in [ + (json!([{"type":"fail_workflow","message":" \t\u{000b}"}]), StatusCode::UNPROCESSABLE_ENTITY), + (json!([{"type":"fail_workflow","message":"failure","non_retryable":"true"}]), StatusCode::UNPROCESSABLE_ENTITY), + (json!([{"type":"start_timer","delay_seconds":5},{"type":"fail_workflow","message":"failure","failed_step_sequence":1}]), StatusCode::UNPROCESSABLE_ENTITY), + (json!([{"type":"fail_workflow","message":"one"},{"type":"fail_workflow","message":"two"}]), StatusCode::CONFLICT), + (json!([{"type":"fail_workflow","message":"one"},{"type":"complete_workflow","result":envelope(Payload::Null)}]), StatusCode::CONFLICT), + (json!([{"type":"fail_workflow","message":"one"},{"type":"unknown"}]), StatusCode::UNPROCESSABLE_ENTITY), ] { assert_eq!( finish_task(&app, &task, commands).await.0, - StatusCode::UNPROCESSABLE_ENTITY + expected ); assert_eq!(history(&app, &run).await, original); assert_eq!( diff --git a/scripts/conformance/server-parity/ACTIVITY-RETRY-RECEIPTS.md b/scripts/conformance/server-parity/ACTIVITY-RETRY-RECEIPTS.md index e58f1b4dc..1ed138cf2 100644 --- a/scripts/conformance/server-parity/ACTIVITY-RETRY-RECEIPTS.md +++ b/scripts/conformance/server-parity/ACTIVITY-RETRY-RECEIPTS.md @@ -78,3 +78,24 @@ IDs and active activities. It cannot repair the retained parent task or touch neighboring marker workflows. Resource limits and individual budgets remain unchanged. Other-database, final-head and default integration remain pending. These controls do not establish release or migration readiness. + +## Contract decision: the deadline runs from the attempt start + +**October 23, 2026.** The retry must not precede the first attempt's start plus +its start-to-close timeout. The contract reads that start from the +`ActivityStarted` snapshot (`payload.activity.started_at`), not from the +event's own timestamp. + +Published PHP 2.11.1 takes one clock reading for the attempt's `started_at` and +close deadline. `WorkflowHistoryEvent::record` then takes a later one for the +event. Its timeout enforcer is correct against its own deadline, but it can run +in the few milliseconds between those readings. Measured against the event time, +that retry looks early. This happened once on Server main's SQLite job +([run 39961767541](https://github.com/durable-workflow/server/actions/runs/39961767541)). +That job records the PHP target first and stops at its first failure, so the +failing target was PHP. Native stamps both from the same millisecond. + +The contract still refuses a retry before the attempt's deadline. It also +requires the event to be stamped at or after the attempt start, and less than +one second after it. The failure message names the transport and how early the +retry was. diff --git a/scripts/conformance/server-parity/activity-deadline-receipts-contract.mjs b/scripts/conformance/server-parity/activity-deadline-receipts-contract.mjs index 5bd65bded..299ea3d89 100644 --- a/scripts/conformance/server-parity/activity-deadline-receipts-contract.mjs +++ b/scripts/conformance/server-parity/activity-deadline-receipts-contract.mjs @@ -150,7 +150,16 @@ export function checkActivityDeadlineReceipts(fixture, observation, workflowId, assert.equal(retry.payload.retry_task_id, second.task_id); assert.equal(retry.payload.retry_of_task_id, first.task_id); assert.equal(retry.payload.retry_backoff_seconds, spec.backoff_seconds[0]); - assert(instant(retry.timestamp) >= instant(started.timestamp) + BigInt(spec.start_to_close_timeout) * 1_000_000_000n); + // The deadline runs from the attempt's own start, which the event snapshot + // carries. Published PHP stamps the event a few milliseconds after that start. + const attemptStart = started.payload.activity.started_at; + assert.equal(typeof attemptStart, 'string', 'ActivityStarted carries the attempt start'); + const stampLag = instant(started.timestamp) - instant(attemptStart); + assert(stampLag >= 0n && stampLag < 1_000_000_000n, + `ActivityStarted is stamped ${stampLag}ns after the attempt start; expected 0 to under 1s`); + const closeDeadline = instant(attemptStart) + BigInt(spec.start_to_close_timeout) * 1_000_000_000n; + assert(instant(retry.timestamp) >= closeDeadline, + `${observation.mode} retry ${retry.timestamp} precedes the start-to-close deadline by ${closeDeadline - instant(retry.timestamp)}ns`); assert(instant(restarted.timestamp) >= instant(retry.payload.retry_available_at)); assert(instant(completed.timestamp) < instant(restarted.timestamp) + BigInt(spec.start_to_close_timeout) * 1_000_000_000n); assert.deepEqual(completed.typed_decoded.result, fixture.typed_value); diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 794ad66a9..85f39f1ac 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -14,6 +14,10 @@ import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; +import {checkSystemMetrics} from './system-metrics-contract.mjs'; +import {checkPrometheusMetrics} from './prometheus-metrics-contract.mjs'; +import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; +import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; @@ -146,6 +150,10 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; + const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; + const prometheusMetrics = fixture.prometheus_metrics ? checkPrometheusMetrics(fixture, observation, workflowId) : null; + const payloadCleanup = fixture.payload_cleanup ? checkPayloadCleanup(fixture, observation, workflowId) : null; + const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; @@ -695,6 +703,10 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(storageTest ? {storage_test: storageTest} : {}), ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), + ...(systemMetrics ? {system_metrics: systemMetrics} : {}), + ...(prometheusMetrics ? {prometheus_metrics: prometheusMetrics} : {}), + ...(payloadCleanup ? {payload_cleanup: payloadCleanup} : {}), + ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), diff --git a/scripts/conformance/server-parity/payload-cleanup-contract.mjs b/scripts/conformance/server-parity/payload-cleanup-contract.mjs new file mode 100644 index 000000000..0240e61b2 --- /dev/null +++ b/scripts/conformance/server-parity/payload-cleanup-contract.mjs @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; + +const WHOLE_SECOND = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.000000Z$/; + +// Reviewed external payload cleanup answers (SystemController:: +// externalPayloadCleanupStatus and externalPayloadCleanupPass). Every answer is +// exact except the namespace placeholder and a pass's whole-second time. +export function checkPayloadCleanup(fixture, observation, workflowId) { + const definition = fixture.payload_cleanup; + const state = observation.payload_cleanup; + const projection = {scope: 'payload_cleanup', cases: definition.requests.length}; + assert.ok(state && typeof state === 'object', 'actual payload cleanup observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_system_routes'}, 'embedded cleanup explicitly inapplicable'); + return projection; + } + const namespace = `${workflowId}-cleanup`.toLowerCase(); + assert.deepStrictEqual([state.applicable, state.namespace], [true, namespace], 'cleanup namespace'); + assert.deepStrictEqual(Object.keys(state.requests), definition.requests.map(([label]) => label), 'every request recorded in order'); + for (const [label] of definition.requests) { + const recorded = state.requests[label]; + const [status, body] = definition.expected[label]; + const completed = recorded.response?.cleanup?.last_pass?.completed_at; + if (body.cleanup?.last_pass?.completed_at === '@second') { + assert.match(String(completed), WHOLE_SECOND, `${label}: whole-second completion time`); + } + const expected = JSON.parse(JSON.stringify(body).split('"@own"').join(JSON.stringify(namespace)) + .split('"@second"').join(JSON.stringify(completed ?? null))); + assert.deepStrictEqual([recorded.status, recorded.response], [status, expected], label); + } + return projection; +} diff --git a/scripts/conformance/server-parity/payload-cleanup-probe.php b/scripts/conformance/server-parity/payload-cleanup-probe.php new file mode 100644 index 000000000..2a7df99aa --- /dev/null +++ b/scripts/conformance/server-parity/payload-cleanup-probe.php @@ -0,0 +1,38 @@ +createNamespace($namespace, 'payload cleanup λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new PayloadCleanupHttp(['base_uri' => rtrim($url, '/').'/api/system/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, mixed $body = null) use ($http, $token, $namespace): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), ...($body === null ? [] : ['json' => $body])]); + + return ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + $state = ['applicable' => true, 'namespace' => $namespace, 'requests' => []]; + foreach ($definition['requests'] as [$label, $method, $path, $body]) { + $state['requests'][$label] = $raw($method, $path, $body === [] ? (object) [] : $body); + } + + return $state; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 5ccb00c47..b6339da20 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -432,6 +432,10 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; + $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; + $prometheusMetrics = isset($fixture['prometheus_metrics']) ? finishHttpPrometheusMetrics($client, $fixture, $workflowId, $url) : null; + $payloadCleanup = isset($fixture['payload_cleanup']) ? finishHttpPayloadCleanup($client, $fixture, $workflowId, $url) : null; + $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) @@ -534,6 +538,10 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'storage_test' => $storageTest, 'activity_lease_renewal' => $activityLeaseRenewal, 'service_catalog' => $serviceCatalog, + 'system_metrics' => $systemMetrics, + 'prometheus_metrics' => $prometheusMetrics, + 'payload_cleanup' => $payloadCleanup, + 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, @@ -840,6 +848,10 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/storage-test-probe.php'; require __DIR__.'/activity-lease-renewal-probe.php'; require __DIR__.'/service-catalog-probe.php'; + require __DIR__.'/system-metrics-probe.php'; + require __DIR__.'/prometheus-metrics-probe.php'; + require __DIR__.'/payload-cleanup-probe.php'; + require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; @@ -1005,6 +1017,18 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } + if ($mode === 'embedded' && isset($fixture['system_metrics'])) { + $observation['system_metrics'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_metrics']; + } + if ($mode === 'embedded' && isset($fixture['prometheus_metrics'])) { + $observation['prometheus_metrics'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_metrics']; + } + if ($mode === 'embedded' && isset($fixture['payload_cleanup'])) { + $observation['payload_cleanup'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_system_routes']; + } + if ($mode === 'embedded' && isset($fixture['repair_pass'])) { + $observation['repair_pass'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_system_routes']; + } if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } diff --git a/scripts/conformance/server-parity/prometheus-metrics-contract.mjs b/scripts/conformance/server-parity/prometheus-metrics-contract.mjs new file mode 100644 index 000000000..c4e72f1ab --- /dev/null +++ b/scripts/conformance/server-parity/prometheus-metrics-contract.mjs @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; + +const MICROS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; +const BOUNDS = ['0.1', '0.5', '1', '2.5', '5', '10', '30', '60', '300', '900']; +const LIMIT = 100; +const SELECTIONS = { + workflows: 'bounded_task_queue_and_workflow_type_ascending', + activities: 'bounded_task_queue_workflow_type_and_activity_type_ascending', + task_queues: 'task_queue_name_ascending', +}; +const DIMENSIONS = { + workflows: ['task_queue', 'workflow_type'], + activities: ['task_queue', 'workflow_type', 'activity_type'], + task_queues: ['task_queue'], +}; + +const zeros = keys => Object.fromEntries(keys.map(key => [key, 0])); + +function labelSet(family) { + return {namespace: 'request_scope_not_label', ...Object.fromEntries(DIMENSIONS[family] + .map(dimension => [dimension, {cardinality_class: 'bounded_series', limit: LIMIT, selection: SELECTIONS[family]}]))}; +} + +function cardinality(counts) { + const disclosure = family => ({ + label_dimensions: DIMENSIONS[family], limit: LIMIT, observed_series_count: counts[family], + observed_series_count_precision: 'exact', reported_series_count: counts[family], suppressed_series_count: 0, + suppressed_started_total: 0, suppressed_counts_precision: 'exact', truncated: false, selection: SELECTIONS[family], + }); + return { + series_limits: {workflows: disclosure('workflows'), activities: disclosure('activities'), task_queues: disclosure('task_queues')}, + metric_label_sets: { + dw_workflow_runs_total: labelSet('workflows'), dw_workflow_run_latency_seconds: labelSet('workflows'), + dw_activity_executions_total: labelSet('activities'), dw_activity_execution_latency_seconds: labelSet('activities'), + dw_task_queue_runtime_state: labelSet('task_queues'), + }, + }; +} + +// A latency histogram for durations that each last at least `minimum` +// seconds. Its bounds are cumulative. Published PHP on SQLite compares each +// duration with a bound bound as text, which SQLite orders after every number, +// so every bucket holds every duration; that recorded reference defect is +// accepted beside the cumulative histogram (see the fixture's contract). +function checkLatency(latency, count, minimum, label) { + assert.deepStrictEqual(Object.keys(latency).sort(), ['buckets', 'count', 'sum'], `${label} latency fields`); + assert.equal(latency.count, count, `${label} latency count`); + assert.deepStrictEqual(Object.keys(latency.buckets).sort(), [...BOUNDS].sort(), `${label} bucket bounds`); + const buckets = BOUNDS.map(bound => latency.buckets[bound]); + assert.ok(buckets.every(Number.isInteger), `${label} integer buckets`); + if (count === 0) { + assert.deepStrictEqual([latency.sum, buckets], [0, BOUNDS.map(() => 0)], `${label} empty histogram`); + return; + } + assert.ok(typeof latency.sum === 'number' && latency.sum >= count * minimum, `${label} latency sum covers each duration`); + assert.ok(buckets.every((value, index) => index === 0 || value >= buckets[index - 1]), `${label} cumulative buckets`); + assert.equal(buckets.at(-1), count, `${label} every duration within 900 seconds`); + const cumulative = BOUNDS.filter(bound => Number(bound) < minimum).every(bound => latency.buckets[bound] === 0); + const textBounds = buckets.every(value => value === count); + assert.ok(cumulative || textBounds, `${label} buckets below the minimum duration are empty`); +} + +function queueEntry(queue, values = {}) { + const fields = []; + for (const prefix of ['workflow', 'activity']) { + for (const field of ['ready_tasks', 'ready_due_tasks', 'delayed_tasks', 'leased_tasks', 'expired_leases', + 'tasks_added_last_minute', 'tasks_dispatched_last_minute']) { + fields.push(`${prefix}_${field}`); + } + } + fields.push('active_pollers', 'workflow_slot_capacity', 'activity_slot_capacity', 'available_workflow_slots', + 'available_activity_slots'); + return {task_queue: queue, ...zeros(fields), ...values}; +} + +function checkSummary(recorded, namespace, expected, label) { + assert.equal(recorded.status, 200, `${label} status`); + const body = recorded.response; + assert.deepStrictEqual(Object.keys(body).sort(), ['cardinality', 'generated_at', 'latency_buckets_seconds', 'namespace', + 'observed_series_count', 'observed_series_count_precision', 'schema', 'series', 'series_count', + 'suppressed_series_count', 'suppressed_series_count_precision'], `${label} fields`); + assert.match(body.generated_at, MICROS, `${label} generated_at`); + const counts = {workflows: expected.workflows.length, activities: expected.activities.length, task_queues: expected.task_queues.length}; + const total = counts.workflows + counts.activities + counts.task_queues; + assert.deepStrictEqual({...body, generated_at: null, series: null}, + {schema: 'durable-workflow.prometheus-metrics.v1', generated_at: null, namespace, + latency_buckets_seconds: BOUNDS.map(Number), series: null, series_count: total, observed_series_count: total, + observed_series_count_precision: 'exact', suppressed_series_count: 0, suppressed_series_count_precision: 'exact', + cardinality: cardinality(counts)}, `${label} summary`); + assert.deepStrictEqual(Object.keys(body.series).sort(), ['activities', 'task_queues', 'workflows'], `${label} families`); + for (const family of ['workflows', 'activities']) { + assert.equal(body.series[family].length, expected[family].length, `${label} ${family} series`); + body.series[family].forEach((series, index) => { + const {latency_seconds: latency, ...counters} = series; + const {latency_count: count, ...wanted} = expected[family][index]; + assert.deepStrictEqual(counters, wanted, `${label} ${family}[${index}]`); + checkLatency(latency, count, expected.minimum, `${label} ${family}[${index}]`); + }); + } + assert.deepStrictEqual(body.series.task_queues, expected.task_queues, `${label} task queues`); +} + +// Reviewed Prometheus summary (SystemController::prometheusMetrics). Counters, +// task-queue state, worker capacity and the bounded-series disclosure are +// exact; latency sums and buckets depend on elapsed time, so the contract +// checks their bounds instead. +export function checkPrometheusMetrics(fixture, observation, workflowId) { + const definition = fixture.prometheus_metrics; + const state = observation.prometheus_metrics; + const projection = {scope: 'prometheus_metrics', series: 6}; + assert.ok(state && typeof state === 'object', 'actual Prometheus metrics observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_metrics'}, 'embedded metrics explicitly inapplicable'); + return projection; + } + const namespace = `${workflowId}-prometheus`.toLowerCase(); + const [first, second] = [`${namespace}-q1`, `${namespace}-q2`]; + assert.deepStrictEqual([state.applicable, state.namespace, state.queues], [true, namespace, [first, second]], 'scenario identity'); + assert.deepStrictEqual(state.statuses, definition.statuses, 'scenario requests'); + const minimum = definition.minimum_duration_ms / 1000; + checkSummary(state.empty, namespace, {workflows: [], activities: [], task_queues: [], minimum}, 'empty namespace'); + const [typeA, typeB] = definition.workflow_types; + const run = (queue, type, values) => ({task_queue: queue, workflow_type: type, started_total: 1, completed_total: 0, + failed_total: 0, cancelled_total: 0, terminated_total: 0, running: 0, latency_count: 0, ...values}); + checkSummary(state.read, namespace, { + minimum, + workflows: [ + run(first, typeA, {completed_total: 1, latency_count: 1}), + run(first, typeB, {terminated_total: 1, latency_count: 1}), + run(second, typeA, {running: 1}), + ], + activities: [{task_queue: first, workflow_type: typeA, activity_type: definition.activity_type, started_total: 1, + completed_total: 1, failed_total: 0, cancelled_total: 0, running: 0, latency_count: 1}], + task_queues: [ + queueEntry(first, {...definition.first_queue, active_pollers: 1, workflow_slot_capacity: definition.workflow_slots, + activity_slot_capacity: definition.activity_slots, available_workflow_slots: definition.workflow_slots, + available_activity_slots: definition.activity_slots}), + queueEntry(second, definition.second_queue), + ], + }, 'after the scenario'); + return projection; +} diff --git a/scripts/conformance/server-parity/prometheus-metrics-probe.php b/scripts/conformance/server-parity/prometheus-metrics-probe.php new file mode 100644 index 000000000..3fbbe285c --- /dev/null +++ b/scripts/conformance/server-parity/prometheus-metrics-probe.php @@ -0,0 +1,81 @@ +createNamespace($namespace, 'prometheus metrics λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $http = new PrometheusMetricsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30]); + $raw = static function (string $method, string $path, ?array $body = null, bool $worker = false) use ($http, $token, $namespace): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + $worker ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ? '1.20' : '2', + ]), ...($body === null ? [] : ['json' => $body])]); + + return ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + [$typeA, $typeB] = $definition['workflow_types']; + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $poll = ['worker_id' => $worker, 'task_queue' => $first, 'timeout_seconds' => 0]; + $state = ['applicable' => true, 'namespace' => $namespace, 'queues' => [$first, $second], 'statuses' => []]; + $step = static function (string $label, array $recorded) use (&$state): array { + $state['statuses'][$label] = $recorded['status']; + + return $recorded['response']; + }; + $state['empty'] = $raw('GET', 'system/prometheus-metrics'); + $step('register', $raw('POST', 'worker/register', ['worker_id' => $worker, 'task_queue' => $first, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$typeA, $typeB], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => [], + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http']), + 'max_concurrent_workflow_tasks' => $definition['workflow_slots'], + 'max_concurrent_activity_tasks' => $definition['activity_slots']], true)); + $step('start completed', $raw('POST', 'workflows', ['workflow_id' => $namespace.'-completed', + 'workflow_type' => $typeA, 'task_queue' => $first, 'input' => $empty])); + $task = $step('claim', $raw('POST', 'worker/workflow-tasks/poll', $poll, true))['task'] ?? []; + $step('schedule activity', $raw('POST', 'worker/workflow-tasks/'.($task['task_id'] ?? 'none').'/complete', [ + 'lease_owner' => $task['lease_owner'] ?? null, 'workflow_task_attempt' => $task['workflow_task_attempt'] ?? null, + 'commands' => [['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], 'arguments' => $empty, + 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], 'start_to_close_timeout' => 300]]], true)); + $activity = $step('claim activity', $raw('POST', 'worker/activity-tasks/poll', $poll, true))['task'] ?? []; + // Each run and activity lasts at least this long, so its duration is above + // the lowest latency bounds. + usleep($definition['minimum_duration_ms'] * 1000); + $step('complete activity', $raw('POST', 'worker/activity-tasks/'.($activity['task_id'] ?? 'none').'/complete', [ + 'activity_attempt_id' => $activity['activity_attempt_id'] ?? null, 'lease_owner' => $activity['lease_owner'] ?? null, + 'result' => $empty], true)); + $task = $step('claim resume', $raw('POST', 'worker/workflow-tasks/poll', $poll, true))['task'] ?? []; + $step('complete run', $raw('POST', 'worker/workflow-tasks/'.($task['task_id'] ?? 'none').'/complete', [ + 'lease_owner' => $task['lease_owner'] ?? null, 'workflow_task_attempt' => $task['workflow_task_attempt'] ?? null, + 'commands' => [['type' => 'complete_workflow', 'result' => $empty]]], true)); + $step('start terminated', $raw('POST', 'workflows', ['workflow_id' => $namespace.'-terminated', + 'workflow_type' => $typeB, 'task_queue' => $first, 'input' => $empty])); + usleep($definition['minimum_duration_ms'] * 1000); + $step('terminate', $raw('POST', 'workflows/'.$namespace.'-terminated/terminate', ['reason' => 'prometheus fixture'])); + $step('start waiting', $raw('POST', 'workflows', ['workflow_id' => $namespace.'-waiting', + 'workflow_type' => $typeA, 'task_queue' => $second, 'input' => $empty])); + // Past the second of its creation, the waiting run's task is due on every + // backend, including PHP's second-precision comparison on SQLite. + usleep($definition['minimum_duration_ms'] * 1000); + $state['read'] = $raw('GET', 'system/prometheus-metrics'); + + return $state; +} diff --git a/scripts/conformance/server-parity/repair-pass-contract.mjs b/scripts/conformance/server-parity/repair-pass-contract.mjs new file mode 100644 index 000000000..cbf6348d0 --- /dev/null +++ b/scripts/conformance/server-parity/repair-pass-contract.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; + +// The fixture's requests and answers with its namespace in place of "@own". +export function repairPassCases(fixture, namespace) { + const own = value => JSON.parse(JSON.stringify(value).split('@own').join(namespace)); + const definition = fixture.repair_pass; + return definition.requests.map(([label, body]) => [label, own(body), own(definition.expected[label])]); +} + +// Reviewed repair pass answers (SystemController::repairPass), exact. +export function checkRepairPass(fixture, observation, workflowId) { + const state = observation.repair_pass; + const projection = {scope: 'repair_pass', cases: fixture.repair_pass.requests.length}; + assert.ok(state && typeof state === 'object', 'actual repair pass observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_system_routes'}, 'embedded repair pass explicitly inapplicable'); + return projection; + } + const namespace = `${workflowId}-repair`.toLowerCase(); + const cases = repairPassCases(fixture, namespace); + assert.deepStrictEqual([state.applicable, state.namespace], [true, namespace], 'repair namespace'); + assert.deepStrictEqual(Object.keys(state.responses), cases.map(([label]) => label), 'every request recorded in order'); + for (const [label, , [status, body]] of cases) { + const recorded = state.responses[label]; + assert.deepStrictEqual([recorded.status, recorded.response], [status, body], label); + } + return projection; +} diff --git a/scripts/conformance/server-parity/repair-pass-probe.php b/scripts/conformance/server-parity/repair-pass-probe.php new file mode 100644 index 000000000..6d894369e --- /dev/null +++ b/scripts/conformance/server-parity/repair-pass-probe.php @@ -0,0 +1,36 @@ +createNamespace($namespace, 'repair pass λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new RepairPassHttp(['base_uri' => rtrim($url, '/').'/api/system/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $own = static fn (mixed $value): mixed => json_decode( + str_replace('@own', $namespace, json_encode($value, JSON_THROW_ON_ERROR)), true, flags: JSON_THROW_ON_ERROR); + $state = ['applicable' => true, 'namespace' => $namespace, 'responses' => []]; + foreach ($fixture['repair_pass']['requests'] as [$label, $body]) { + $response = $http->request('POST', 'repair/pass', ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), 'json' => $own($body)]); + $state['responses'][$label] = ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + } + + return $state; +} diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index 5ded7dc5a..bf1845f4f 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -116,6 +116,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $client->completeWorkflowTask($task['task_id'], $task['lease_owner'], $task['workflow_task_attempt'], [[ 'type' => 'schedule_activity', 'activity_type' => 'restart.activity', 'arguments' => $client->payloadCodec()->envelope([$value]), 'queue' => 'restart-v1', + 'retry_policy' => $restartContract['activity_retry_policy'], ]]); $activity = $client->pollActivityTask('restart-old', 'restart-v1', 0); assertRestart(is_array($activity) && $activity['attempt_number'] === 1, 'Prepare did not lease attempt one.'); @@ -456,7 +457,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array }; $worker = new Worker($client, 'restart-v1', workerId: 'restart-new', clock: $clock); $worker->registerWorkflow('restart.workflow', fn (WorkflowContext $context, array $input): mixed => - $context->activity('restart.activity', [$input])); + $context->activity('restart.activity', [$input], ['retry_policy' => $restartContract['activity_retry_policy']])); $worker->registerWorkflow('restart.timer', function (WorkflowContext $context, array $input): array { $context->sleep(10); @@ -481,12 +482,18 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array assertRestart($execution->status === 'completed' && $execution->output === $value, 'Recovered durable outcome differs.'); $history = $client->workflowHistory($receipt['workflow_id'], $receipt['run_id'])['events']; $types = array_column($history, 'event_type'); +// The expired original lease consumes attempt one, as published PHP's repair +// loop settles it; the declared retry budget gives the recovered worker two. assertRestart($types === ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted', - 'ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted'], 'Recovered event inventory differs.'); + 'ActivityRetryScheduled', 'ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted'], 'Recovered event inventory differs.'); assertRestart(array_column($history, 'sequence') === range(1, count($history)), 'History is not contiguous.'); assertRestart($history[0]['payload']['workflow_run_id'] === $receipt['run_id'], 'Original start/run identity was lost.'); assertRestart($history[3]['payload']['activity_attempt_id'] === $old['activity_attempt_id'], 'Original attempt history was lost.'); -assertRestart($history[4]['payload']['activity_attempt_id'] !== $old['activity_attempt_id'], 'A stale attempt identity was reused.'); +assertRestart($history[4]['payload']['activity_attempt_id'] === $old['activity_attempt_id'] + && $history[4]['payload']['retry_reason'] === 'lease_expired' && $history[4]['payload']['failure_reason'] === 'lease_expired' + && $history[4]['payload']['lease_owner'] === $old['lease_owner'] && ! array_key_exists('timeout_kind', $history[4]['payload']), + 'The expired original lease was not settled by its retry policy.'); +assertRestart($history[5]['payload']['activity_attempt_id'] !== $old['activity_attempt_id'], 'A stale attempt identity was reused.'); $timer = $receipt['timer']; $timerExecution = $client->describeWorkflow($timer['workflow_id'], $timer['run_id']); assertRestart($timerExecution->status === 'completed' && $timerExecution->output === $value, 'Timer did not recover its original durable outcome.'); diff --git a/scripts/conformance/server-parity/storage-test-probe.php b/scripts/conformance/server-parity/storage-test-probe.php index bb61bfcbf..de276b772 100644 --- a/scripts/conformance/server-parity/storage-test-probe.php +++ b/scripts/conformance/server-parity/storage-test-probe.php @@ -42,9 +42,20 @@ function finishHttpStorageTest(Client $admin, array $fixture, string $workflowId foreach ($definition['invalid'] as $body) { $state['invalid'][] = $raw('POST', 'storage/test', $body === [] ? (object) [] : $body); } - $state['passed'] = $raw('POST', 'storage/test', $sizes); + // The published SQLite reference can refuse a round trip with a retryable + // 503 while its maintenance loop holds the file; a later attempt succeeds. + $roundTrip = static function () use ($raw, $sizes): array { + for ($attempt = 1; ; $attempt++) { + $recorded = $raw('POST', 'storage/test', $sizes); + if ($attempt === 10 || $recorded['status'] !== 503 || ($recorded['response']['retryable'] ?? null) !== true) { + return $recorded; + } + usleep(500_000); + } + }; + $state['passed'] = $roundTrip(); // A repeated diagnostic writes and removes the same content-addressed objects. - $state['repeated'] = $raw('POST', 'storage/test', $sizes); + $state['repeated'] = $roundTrip(); return $state; } diff --git a/scripts/conformance/server-parity/system-metrics-contract.mjs b/scripts/conformance/server-parity/system-metrics-contract.mjs new file mode 100644 index 000000000..4f4bf2efb --- /dev/null +++ b/scripts/conformance/server-parity/system-metrics-contract.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; + +const MICROS_OR_MILLIS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}(\d{3})?Z$/; + +function nulls(keys) { + return Object.fromEntries(keys.map(key => [key, null])); +} + +function zeros(keys) { + return Object.fromEntries(keys.map(key => [key, 0])); +} + +// Reviewed namespace metrics (SystemController::metrics). Namespace-scoped +// values are exact. Projection drift is server-wide and counts rows other +// fixtures leave stale or orphaned, so only its shape and totals are checked. +export function checkSystemMetrics(fixture, observation, workflowId) { + const definition = fixture.system_metrics; + const state = observation.system_metrics; + const projection = {scope: 'system_metrics', resources: Object.keys(definition.usage).length}; + assert.ok(state && typeof state === 'object', 'actual system metrics observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_metrics'}, 'embedded metrics explicitly inapplicable'); + return projection; + } + const namespace = `${workflowId}-metrics`.toLowerCase(); + assert.deepStrictEqual([state.applicable, state.namespace, state.registered, state.started, state.claimed_type, state.failed, state.scheduled], + [true, namespace, 201, [201, 201], definition.workflow_types[0], 200, 201], 'metrics scenario'); + const resources = Object.keys(definition.usage); + const durableReasons = [...resources.map(resource => `namespace_${resource}_exhausted`), 'namespace_durable_state_quota_unavailable']; + const expected = (failures, usage) => ({ + dw_workflow_task_consecutive_failures: failures, + dw_namespace_request_admission_rejections: { + max_requests_per_minute: null, max_concurrent_requests: null, requests_this_minute: 0, rejections_this_minute: 0, + rejections_by_reason: zeros(['namespace_request_rate_exhausted', 'namespace_request_concurrency_exhausted', + 'namespace_request_admission_unavailable']), + configuration_status: 'valid', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_three_reason_inventory'}, + }, + dw_namespace_durable_state_usage: { + limits: nulls(resources), usage, remaining: nulls(resources), rejections_this_minute: 0, + rejections_by_reason: zeros(durableReasons), configuration_status: 'valid', measurement_status: 'available', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_reason_inventory'}, + }, + dw_runtime_external_payload_namespace_usage: { + max_bytes: null, max_objects: null, used_bytes: 0, used_objects: 0, remaining_bytes: null, remaining_objects: null, + rejections_this_minute: 0, rejections_by_reason: zeros(['external_payload_namespace_bytes_exhausted', + 'external_payload_namespace_objects_exhausted', 'external_payload_namespace_quota_unavailable']), + configuration_status: 'valid', measurement_status: 'available', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_three_reason_inventory'}, + }, + }); + const empty = {...definition.failures, max_consecutive_failures: 0, failed_task_count: 0, workflow_type_count: 0, by_workflow_type: []}; + const check = (recorded, failures, usage, label) => { + assert.equal(recorded.status, 200, `${label} status`); + const {generated_at: generatedAt, namespace: observed, metrics, cardinality} = recorded.response; + assert.deepStrictEqual(Object.keys(recorded.response).sort(), ['cardinality', 'generated_at', 'metrics', 'namespace'], `${label} fields`); + assert.match(generatedAt, MICROS_OR_MILLIS, `${label} generated_at in UTC`); + assert.equal(observed, namespace, `${label} namespace`); + const {dw_projection_drift_total: drift, ...scoped} = metrics; + assert.deepStrictEqual(scoped, expected(failures, usage), `${label} namespace metrics`); + assert.ok(drift && Array.isArray(drift.by_table), `${label} projection drift family`); + assert.deepStrictEqual(Object.keys(drift).sort(), ['by_table', 'label_cardinality_policy', 'scope', 'table_count', 'tables_with_drift', 'total'], + `${label} drift fields`); + const tables = definition.projection_tables; + assert.deepStrictEqual(drift.by_table.map(row => row.table), tables, `${label} drift tables`); + for (const row of drift.by_table) { + assert.deepStrictEqual(Object.keys(row).sort(), ['needs_rebuild', 'table'], `${label} ${row.table} drift fields`); + assert.ok(Number.isInteger(row.needs_rebuild) && row.needs_rebuild >= 0, `${label} ${row.table} drift`); + } + const counts = drift.by_table.map(row => row.needs_rebuild); + assert.deepStrictEqual( + {total: drift.total, table_count: drift.table_count, tables_with_drift: drift.tables_with_drift, scope: drift.scope, + label_cardinality_policy: drift.label_cardinality_policy}, + {total: counts.reduce((sum, count) => sum + count, 0), table_count: tables.length, + tables_with_drift: counts.filter(count => count > 0).length, scope: 'server', + label_cardinality_policy: {namespace: 'server_scope_no_label', table: {cardinality_class: 'finite_projection_table_inventory', + values: tables, selection: 'fixed_projection_table_inventory'}}}, + `${label} drift totals`); + assert.deepStrictEqual(cardinality, {metric_label_sets: Object.fromEntries(Object.entries(metrics) + .map(([name, family]) => [name, family.label_cardinality_policy]))}, `${label} cardinality`); + }; + check(state.empty, empty, zeros(resources), 'empty namespace'); + check(state.read, definition.failures, definition.usage, 'after the scenario'); + check(state.posted, definition.failures, definition.usage, 'read by POST'); + return projection; +} diff --git a/scripts/conformance/server-parity/system-metrics-probe.php b/scripts/conformance/server-parity/system-metrics-probe.php new file mode 100644 index 000000000..2b25adc69 --- /dev/null +++ b/scripts/conformance/server-parity/system-metrics-probe.php @@ -0,0 +1,61 @@ +createNamespace($namespace, 'system metrics λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new SystemMetricsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, bool $worker = false) use ($http, $token, $namespace): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + $worker ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ? '1.20' : '2', + ]), ...($body === null ? [] : ['json' => $body])]); + + return ['status' => $response->getStatusCode(), + 'response' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + $input = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $state = ['applicable' => true, 'namespace' => $namespace]; + $state['empty'] = $raw('GET', 'system/metrics'); + $state['registered'] = $raw('POST', 'worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => $definition['workflow_types'], 'supported_activity_types' => [], + 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])], true)['status']; + $state['started'] = []; + foreach ($definition['workflow_types'] as $index => $type) { + $state['started'][] = $raw('POST', 'workflows', ['workflow_id' => $namespace.'-'.$index, + 'workflow_type' => $type, 'task_queue' => $queue, 'input' => $input])['status']; + } + // A short long-poll waits out a claim the maintenance loop contends. + $task = $raw('POST', 'worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5], true)['response']['task'] ?? []; + $state['claimed_type'] = $task['workflow_type'] ?? null; + $state['failed'] = $raw('POST', 'worker/workflow-tasks/'.($task['task_id'] ?? 'none').'/fail', [ + 'lease_owner' => $task['lease_owner'] ?? null, 'workflow_task_attempt' => $task['workflow_task_attempt'] ?? null, + 'failure' => ['message' => 'metrics fixture failure', 'type' => 'RuntimeException'], + ], true)['status']; + $state['scheduled'] = $raw('POST', 'schedules', ['schedule_id' => $namespace.'-schedule', 'spec' => ['intervals' => [['every' => 'P1D']]], + 'action' => ['workflow_type' => $definition['workflow_types'][0], 'task_queue' => $queue, 'input' => $input]])['status']; + $state['read'] = $raw('GET', 'system/metrics'); + $state['posted'] = $raw('POST', 'system/metrics'); + + return $state; +} diff --git a/tests/Fixtures/ServerParity/payload-cleanup.json b/tests/Fixtures/ServerParity/payload-cleanup.json new file mode 100644 index 000000000..eb6612bd0 --- /dev/null +++ b/tests/Fixtures/ServerParity/payload-cleanup.json @@ -0,0 +1,391 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "payload-cleanup", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "payload cleanup λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "payload cleanup λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "payload_cleanup": { + "requests": [ + [ + "status fresh", + "GET", + "external-payload-cleanup", + null + ], + [ + "pass", + "POST", + "external-payload-cleanup/pass", + [] + ], + [ + "status after pass", + "GET", + "external-payload-cleanup", + null + ], + [ + "pass limit 5", + "POST", + "external-payload-cleanup/pass", + { + "limit": 5 + } + ], + [ + "pass limit 0", + "POST", + "external-payload-cleanup/pass", + { + "limit": 0 + } + ], + [ + "pass limit 1001", + "POST", + "external-payload-cleanup/pass", + { + "limit": 1001 + } + ], + [ + "pass limit text", + "POST", + "external-payload-cleanup/pass", + { + "limit": "x" + } + ], + [ + "pass limit fraction", + "POST", + "external-payload-cleanup/pass", + { + "limit": 2.5 + } + ], + [ + "pass limit null", + "POST", + "external-payload-cleanup/pass", + { + "limit": null + } + ], + [ + "pass limit numeric text", + "POST", + "external-payload-cleanup/pass", + { + "limit": "7" + } + ], + [ + "pass without body", + "POST", + "external-payload-cleanup/pass", + null + ], + [ + "status final", + "GET", + "external-payload-cleanup", + null + ] + ], + "expected": { + "status fresh": [ + 200, + { + "namespace": "@own", + "cleanup": { + "schema": "durable-workflow.v2.runtime-external-payload-cleanup-metrics.v1", + "backlog": { + "expired_unclaimed": 0, + "failed_registration_objects": 0 + }, + "bounds": { + "abandoned_upload_expiry_seconds": 3600, + "default_batch_size": 100, + "maximum_batch_size": 1000 + }, + "totals": { + "passes": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0 + }, + "last_pass": { + "status": null, + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0, + "completed_at": null, + "last_storage_failure_at": null + }, + "identity_policy": { + "provider_credentials_exposed": false, + "provider_locations_exposed": false, + "reusable_reference_identities_exposed": false + } + } + } + ], + "pass": [ + 200, + { + "namespace": "@own", + "cleanup": { + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "skipped": 0, + "blocked": 0, + "storage_driver_failures": 0, + "backlog": 0, + "scan_limit": 100, + "scan_pressure": false + } + } + ], + "status after pass": [ + 200, + { + "namespace": "@own", + "cleanup": { + "schema": "durable-workflow.v2.runtime-external-payload-cleanup-metrics.v1", + "backlog": { + "expired_unclaimed": 0, + "failed_registration_objects": 0 + }, + "bounds": { + "abandoned_upload_expiry_seconds": 3600, + "default_batch_size": 100, + "maximum_batch_size": 1000 + }, + "totals": { + "passes": 1, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0 + }, + "last_pass": { + "status": "completed", + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0, + "completed_at": "@second", + "last_storage_failure_at": null + }, + "identity_policy": { + "provider_credentials_exposed": false, + "provider_locations_exposed": false, + "reusable_reference_identities_exposed": false + } + } + } + ], + "pass limit 5": [ + 200, + { + "namespace": "@own", + "cleanup": { + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "skipped": 0, + "blocked": 0, + "storage_driver_failures": 0, + "backlog": 0, + "scan_limit": 5, + "scan_pressure": false + } + } + ], + "pass limit 0": [ + 422, + { + "message": "The limit field must be at least 1.", + "reason": "validation_failed", + "errors": { + "limit": [ + "The limit field must be at least 1." + ] + }, + "validation_errors": { + "limit": [ + "The limit field must be at least 1." + ] + } + } + ], + "pass limit 1001": [ + 422, + { + "message": "The limit field must not be greater than 1000.", + "reason": "validation_failed", + "errors": { + "limit": [ + "The limit field must not be greater than 1000." + ] + }, + "validation_errors": { + "limit": [ + "The limit field must not be greater than 1000." + ] + } + } + ], + "pass limit text": [ + 422, + { + "message": "The limit field must be an integer.", + "reason": "validation_failed", + "errors": { + "limit": [ + "The limit field must be an integer." + ] + }, + "validation_errors": { + "limit": [ + "The limit field must be an integer." + ] + } + } + ], + "pass limit fraction": [ + 422, + { + "message": "The limit field must be an integer.", + "reason": "validation_failed", + "errors": { + "limit": [ + "The limit field must be an integer." + ] + }, + "validation_errors": { + "limit": [ + "The limit field must be an integer." + ] + } + } + ], + "pass limit null": [ + 200, + { + "namespace": "@own", + "cleanup": { + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "skipped": 0, + "blocked": 0, + "storage_driver_failures": 0, + "backlog": 0, + "scan_limit": 100, + "scan_pressure": false + } + } + ], + "pass limit numeric text": [ + 200, + { + "namespace": "@own", + "cleanup": { + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "skipped": 0, + "blocked": 0, + "storage_driver_failures": 0, + "backlog": 0, + "scan_limit": 7, + "scan_pressure": false + } + } + ], + "pass without body": [ + 200, + { + "namespace": "@own", + "cleanup": { + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "skipped": 0, + "blocked": 0, + "storage_driver_failures": 0, + "backlog": 0, + "scan_limit": 100, + "scan_pressure": false + } + } + ], + "status final": [ + 200, + { + "namespace": "@own", + "cleanup": { + "schema": "durable-workflow.v2.runtime-external-payload-cleanup-metrics.v1", + "backlog": { + "expired_unclaimed": 0, + "failed_registration_objects": 0 + }, + "bounds": { + "abandoned_upload_expiry_seconds": 3600, + "default_batch_size": 100, + "maximum_batch_size": 1000 + }, + "totals": { + "passes": 5, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0 + }, + "last_pass": { + "status": "completed", + "processed": 0, + "deleted_references": 0, + "deleted_backing_objects": 0, + "shared_objects_preserved": 0, + "blocked_outcomes": 0, + "storage_driver_failures": 0, + "completed_at": "@second", + "last_storage_failure_at": null + }, + "identity_policy": { + "provider_credentials_exposed": false, + "provider_locations_exposed": false, + "reusable_reference_identities_exposed": false + } + } + } + ] + } + }, + "contract": "GET /api/system/external-payload-cleanup and POST /api/system/external-payload-cleanup/pass (published Server 2.11) report and run the namespace's external payload cleanup. A fresh namespace has no expired unclaimed payloads and no recorded pass. Each valid pass reports what it processed, deleted, preserved, skipped and blocked, with its scan limit (default 100, at most 1000), and records a completed pass in the namespace's totals with its whole-second completion time. A limit that is not an integer from 1 to 1000 answers Laravel's 422 without recording a pass; a null or numeric-text limit is accepted. Embedded applications have no HTTP system routes." +} diff --git a/tests/Fixtures/ServerParity/prometheus-metrics.json b/tests/Fixtures/ServerParity/prometheus-metrics.json new file mode 100644 index 000000000..512910178 --- /dev/null +++ b/tests/Fixtures/ServerParity/prometheus-metrics.json @@ -0,0 +1,30 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "prometheus-metrics", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "prometheus metrics λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "prometheus metrics λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "prometheus_metrics": { + "workflow_types": ["parity.v1.prom_a", "parity.v1.prom_b"], + "activity_type": "parity.v1.prom_act", + "empty_blob": "wwHioz3/VYAiNwwA", + "workflow_slots": 4, + "activity_slots": 3, + "minimum_duration_ms": 1100, + "statuses": { + "register": 201, "start completed": 201, "claim": 200, "schedule activity": 200, "claim activity": 200, + "complete activity": 200, "claim resume": 200, "complete run": 200, "start terminated": 201, "terminate": 200, + "start waiting": 201 + }, + "first_queue": {"workflow_tasks_added_last_minute": 3, "workflow_tasks_dispatched_last_minute": 3, "activity_tasks_added_last_minute": 1}, + "second_queue": {"workflow_ready_tasks": 1, "workflow_ready_due_tasks": 1, "workflow_tasks_added_last_minute": 1, "workflow_tasks_dispatched_last_minute": 1} + }, + "contract": "GET /api/system/prometheus-metrics (published Server 2.11) summarizes a namespace for Prometheus scrapers: workflow series by task queue and workflow type, activity series by task queue, workflow type and activity type, and task-queue runtime state, each family capped at 100 series with its suppression disclosed and its label policy declared. A fresh namespace reports no series. After a worker with four workflow and three activity slots registers on the first queue, a run there completes after one activity, a second run there is terminated, and a run waits on a second queue without a worker, the summary holds three workflow series, one activity series and two task queues with exact counters, task counts and worker capacity. Each counted run and activity lasts at least 1.1 seconds. Latency sums cover those durations, buckets are cumulative, and every duration falls within the 900-second bound. The buckets below the minimum duration are empty, except on published PHP with SQLite: it binds each bound as text, which SQLite orders after every number, so every bucket holds every duration. That reference defect is recorded, not adopted; native reports the cumulative histogram on every backend. Embedded applications have no HTTP metrics." +} diff --git a/tests/Fixtures/ServerParity/repair-pass.json b/tests/Fixtures/ServerParity/repair-pass.json new file mode 100644 index 000000000..8bc98874c --- /dev/null +++ b/tests/Fixtures/ServerParity/repair-pass.json @@ -0,0 +1,417 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "repair-pass", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "repair pass λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "repair pass λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "repair_pass": { + "requests": [ + [ + "filtered pass", + { + "run_ids": [ + "@own-missing" + ], + "queue": "@own-queue" + } + ], + [ + "trimmed filters", + { + "run_ids": [ + " @own-missing ", + "@own-other" + ], + "instance_id": " @own-instance ", + "connection": " @own-connection ", + "queue": " @own-queue " + } + ], + [ + "empty instance", + { + "run_ids": [ + "@own-missing" + ], + "queue": "@own-queue", + "instance_id": "", + "connection": "" + } + ], + [ + "run ids not a list", + { + "run_ids": "x", + "queue": "@own-queue" + } + ], + [ + "empty run id", + { + "run_ids": [ + "" + ], + "queue": "@own-queue" + } + ], + [ + "too many run ids", + { + "run_ids": [ + "@own-r0", + "@own-r1", + "@own-r2", + "@own-r3", + "@own-r4", + "@own-r5", + "@own-r6", + "@own-r7", + "@own-r8", + "@own-r9", + "@own-r10", + "@own-r11", + "@own-r12", + "@own-r13", + "@own-r14", + "@own-r15", + "@own-r16", + "@own-r17", + "@own-r18", + "@own-r19", + "@own-r20", + "@own-r21", + "@own-r22", + "@own-r23", + "@own-r24", + "@own-r25", + "@own-r26", + "@own-r27", + "@own-r28", + "@own-r29", + "@own-r30", + "@own-r31", + "@own-r32", + "@own-r33", + "@own-r34", + "@own-r35", + "@own-r36", + "@own-r37", + "@own-r38", + "@own-r39", + "@own-r40", + "@own-r41", + "@own-r42", + "@own-r43", + "@own-r44", + "@own-r45", + "@own-r46", + "@own-r47", + "@own-r48", + "@own-r49", + "@own-r50", + "@own-r51", + "@own-r52", + "@own-r53", + "@own-r54", + "@own-r55", + "@own-r56", + "@own-r57", + "@own-r58", + "@own-r59", + "@own-r60", + "@own-r61", + "@own-r62", + "@own-r63", + "@own-r64", + "@own-r65", + "@own-r66", + "@own-r67", + "@own-r68", + "@own-r69", + "@own-r70", + "@own-r71", + "@own-r72", + "@own-r73", + "@own-r74", + "@own-r75", + "@own-r76", + "@own-r77", + "@own-r78", + "@own-r79", + "@own-r80", + "@own-r81", + "@own-r82", + "@own-r83", + "@own-r84", + "@own-r85", + "@own-r86", + "@own-r87", + "@own-r88", + "@own-r89", + "@own-r90", + "@own-r91", + "@own-r92", + "@own-r93", + "@own-r94", + "@own-r95", + "@own-r96", + "@own-r97", + "@own-r98", + "@own-r99", + "@own-r100" + ], + "queue": "@own-queue" + } + ], + [ + "throttle not boolean", + { + "run_ids": [ + "@own-missing" + ], + "queue": "@own-queue", + "respect_throttle": "yes" + } + ], + [ + "queue too long", + { + "run_ids": [ + "@own-missing" + ], + "queue": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + } + ], + [ + "connection too long", + { + "run_ids": [ + "@own-missing" + ], + "queue": "@own-queue", + "connection": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + } + ] + ], + "expected": { + "filtered pass": [ + 200, + { + "connection": null, + "queue": "@own-queue", + "run_ids": [ + "@own-missing" + ], + "instance_id": null, + "respect_throttle": false, + "selected_command_contract_candidates": 0, + "backfilled_command_contracts": 0, + "command_contract_backfill_unavailable": 0, + "command_contract_failures": [], + "existing_task_failures": [], + "missing_run_failures": [], + "throttled": false, + "selected_existing_task_candidates": 0, + "selected_missing_task_candidates": 0, + "selected_total_candidates": 0, + "repaired_existing_tasks": 0, + "repaired_missing_tasks": 0, + "dispatched_tasks": 0, + "deadline_expired_candidates": 0, + "deadline_expired_tasks_created": 0, + "cancellation_deadlines_enforced": 0, + "deadline_expired_failures": [], + "activity_timeout_candidates": 0, + "activity_timeouts_enforced": 0, + "activity_timeout_failures": [], + "service_call_timeout_candidates": 0, + "service_call_timeouts_enforced": 0, + "service_call_timeout_failures": [] + } + ], + "trimmed filters": [ + 200, + { + "connection": "@own-connection", + "queue": "@own-queue", + "run_ids": [ + "@own-missing", + "@own-other" + ], + "instance_id": "@own-instance", + "respect_throttle": false, + "selected_command_contract_candidates": 0, + "backfilled_command_contracts": 0, + "command_contract_backfill_unavailable": 0, + "command_contract_failures": [], + "existing_task_failures": [], + "missing_run_failures": [], + "throttled": false, + "selected_existing_task_candidates": 0, + "selected_missing_task_candidates": 0, + "selected_total_candidates": 0, + "repaired_existing_tasks": 0, + "repaired_missing_tasks": 0, + "dispatched_tasks": 0, + "deadline_expired_candidates": 0, + "deadline_expired_tasks_created": 0, + "cancellation_deadlines_enforced": 0, + "deadline_expired_failures": [], + "activity_timeout_candidates": 0, + "activity_timeouts_enforced": 0, + "activity_timeout_failures": [], + "service_call_timeout_candidates": 0, + "service_call_timeouts_enforced": 0, + "service_call_timeout_failures": [] + } + ], + "empty instance": [ + 200, + { + "connection": null, + "queue": "@own-queue", + "run_ids": [ + "@own-missing" + ], + "instance_id": null, + "respect_throttle": false, + "selected_command_contract_candidates": 0, + "backfilled_command_contracts": 0, + "command_contract_backfill_unavailable": 0, + "command_contract_failures": [], + "existing_task_failures": [], + "missing_run_failures": [], + "throttled": false, + "selected_existing_task_candidates": 0, + "selected_missing_task_candidates": 0, + "selected_total_candidates": 0, + "repaired_existing_tasks": 0, + "repaired_missing_tasks": 0, + "dispatched_tasks": 0, + "deadline_expired_candidates": 0, + "deadline_expired_tasks_created": 0, + "cancellation_deadlines_enforced": 0, + "deadline_expired_failures": [], + "activity_timeout_candidates": 0, + "activity_timeouts_enforced": 0, + "activity_timeout_failures": [], + "service_call_timeout_candidates": 0, + "service_call_timeouts_enforced": 0, + "service_call_timeout_failures": [] + } + ], + "run ids not a list": [ + 422, + { + "message": "The run ids field must be an array.", + "reason": "validation_failed", + "errors": { + "run_ids": [ + "The run ids field must be an array." + ] + }, + "validation_errors": { + "run_ids": [ + "The run ids field must be an array." + ] + } + } + ], + "empty run id": [ + 422, + { + "message": "The run_ids.0 field must be a string. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "run_ids.0": [ + "The run_ids.0 field must be a string.", + "The run_ids.0 field must be at least 1 characters." + ] + }, + "validation_errors": { + "run_ids.0": [ + "The run_ids.0 field must be a string.", + "The run_ids.0 field must be at least 1 characters." + ] + } + } + ], + "too many run ids": [ + 422, + { + "message": "The run ids field must not have more than 100 items.", + "reason": "validation_failed", + "errors": { + "run_ids": [ + "The run ids field must not have more than 100 items." + ] + }, + "validation_errors": { + "run_ids": [ + "The run ids field must not have more than 100 items." + ] + } + } + ], + "throttle not boolean": [ + 422, + { + "message": "The respect throttle field must be true or false.", + "reason": "validation_failed", + "errors": { + "respect_throttle": [ + "The respect throttle field must be true or false." + ] + }, + "validation_errors": { + "respect_throttle": [ + "The respect throttle field must be true or false." + ] + } + } + ], + "queue too long": [ + 422, + { + "message": "The queue field must not be greater than 128 characters.", + "reason": "validation_failed", + "errors": { + "queue": [ + "The queue field must not be greater than 128 characters." + ] + }, + "validation_errors": { + "queue": [ + "The queue field must not be greater than 128 characters." + ] + } + } + ], + "connection too long": [ + 422, + { + "message": "The connection field must not be greater than 128 characters.", + "reason": "validation_failed", + "errors": { + "connection": [ + "The connection field must not be greater than 128 characters." + ] + }, + "validation_errors": { + "connection": [ + "The connection field must not be greater than 128 characters." + ] + } + } + ] + } + }, + "contract": "POST /api/system/repair/pass (published Server 2.11) runs one task repair pass: it recovers ready tasks that need redispatch and expired leases, creates missing workflow tasks, enforces expired run, activity and service-call deadlines, and reports each count with its failures, answering 207 when any failed. The pass is server-wide, so the fixture scopes every pass to run ids and a task queue that do not exist; each then reports zero candidates and echoes its trimmed filters, with empty strings read as null. Run ids are at most 100 non-empty strings; respect_throttle is a boolean; connection and queue are at most 128 characters. Invalid requests answer Laravel's 422. Embedded applications have no HTTP system routes." +} diff --git a/tests/Fixtures/ServerParity/system-metrics.json b/tests/Fixtures/ServerParity/system-metrics.json new file mode 100644 index 000000000..72c15458d --- /dev/null +++ b/tests/Fixtures/ServerParity/system-metrics.json @@ -0,0 +1,32 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "system-metrics", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "system metrics λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "system metrics λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "system_metrics": { + "workflow_types": ["parity.v1.metrics_a", "parity.v1.metrics_b"], + "empty_blob": "wwHioz3/VYAiNwwA", + "failures": { + "max_consecutive_failures": 1, "failed_task_count": 1, "workflow_type_count": 1, "workflow_type_limit": 20, + "workflow_types_truncated": false, "suppressed_workflow_type_count": 0, "suppressed_failed_task_count": 0, + "label_cardinality_policy": {"namespace": "request_scope_not_label", "workflow_type": {"cardinality_class": "bounded_series", "limit": 20, "selection": "top_by_max_consecutive_failures_then_name"}}, + "by_workflow_type": [{"workflow_type": "parity.v1.metrics_a", "max_consecutive_failures": 1, "failed_task_count": 1}] + }, + "usage": { + "workflow_instances": 2, "workflow_runs": 2, "open_workflow_runs": 2, "schedules": 1, "schedule_history_events": 1, + "worker_registrations": 1, "workflow_history_events": 4, "workflow_tasks": 3, "pending_workflow_tasks": 2, + "workflow_timers": 0, "pending_workflow_timers": 0, "workflow_run_waits": 0, "open_workflow_run_waits": 0, + "workflow_commands": 2, "workflow_streams": 0, "workflow_stream_items": 0 + }, + "projection_tables": ["run_summaries", "run_waits", "run_timeline_entries", "run_timer_entries", "run_lineage_entries"] + }, + "contract": "GET and POST /api/system/metrics (published Server 2.11) report the namespace's workflow task failures, the server's projection drift, and the namespace's request admission, durable-state and external payload usage, each with its bounded label policy. In a fresh namespace that holds two started runs of two workflow types, one failed workflow task of the first type and one schedule, the failure series names that type with one consecutive failure, and the sixteen durable-state counts are exact. No admission, durable-state or payload limit is configured, so limits and remaining values are null and rejection counters zero, and no external payload is stored. Projection drift is server-wide and counts stale or orphaned rows that other work on the server leaves, so it is checked for its fixed table inventory, non-negative counts and consistent totals, not exact values. Embedded applications have no HTTP metrics." +} diff --git a/tests/Fixtures/ServerParityProfiles/current-activity-restart.json b/tests/Fixtures/ServerParityProfiles/current-activity-restart.json index acad803e3..a0be66f44 100644 --- a/tests/Fixtures/ServerParityProfiles/current-activity-restart.json +++ b/tests/Fixtures/ServerParityProfiles/current-activity-restart.json @@ -6,5 +6,7 @@ "clock_or_row_edit": false, "worker_revocation": false, "receipt_rebudget": false, + "activity_retry_policy": {"max_attempts": 2, "backoff_seconds": [0]}, + "lease_loss": "Published PHP claims only ready activity tasks. Its repair loop settles an expired activity lease through the activity's retry policy: the original attempt closes as expired and ActivityRetryScheduled records retry_reason lease_expired. The original activity therefore declares two attempts, so the recovered worker runs attempt two after that retry.", "contract": "The new native revision follows the published PHP ordinary five-minute activity lease. Recovery waits for the already committed original lease, with this bound declared before preparing a new cohort. The previous sub-forty-second native recovery observations and failures remain unchanged. No current run receives a replacement lease or deadline to fit the test. This wait allocation does not claim performance or modify workflow execution budgets." } diff --git a/tests/Unit/ServerParityPayloadCleanupContractTest.mjs b/tests/Unit/ServerParityPayloadCleanupContractTest.mjs new file mode 100644 index 000000000..a5b4ce945 --- /dev/null +++ b/tests/Unit/ServerParityPayloadCleanupContractTest.mjs @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkPayloadCleanup} from '../../scripts/conformance/server-parity/payload-cleanup-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/payload-cleanup.json', import.meta.url))); +const definition = fixture.payload_cleanup; +const id = 'parity-v1-payload-cleanup'; +const own = `${id}-cleanup`; + +const observation = () => { + const requests = {}; + for (const [label] of definition.requests) { + const [status, body] = definition.expected[label]; + requests[label] = {status, response: JSON.parse(JSON.stringify(body).split('"@own"').join(JSON.stringify(own)) + .split('"@second"').join('"2026-10-22T23:08:38.000000Z"'))}; + } + return {mode: 'http', payload_cleanup: {applicable: true, namespace: own, requests}}; +}; + +test('the payload cleanup accepts the published answers', () => { + assert.deepStrictEqual(checkPayloadCleanup(fixture, observation(), id), {scope: 'payload_cleanup', cases: 12}); +}); + +test('the fixture covers status, passes and limit validation', () => { + const statuses = Object.values(definition.expected).map(([status]) => status); + assert.deepStrictEqual([...new Set(statuses)].sort(), [200, 422]); + assert.equal(definition.expected['status final'][1].cleanup.totals.passes, 5); + assert.equal(definition.expected['status fresh'][1].cleanup.last_pass.status, null); +}); + +test('the embedded payload cleanup is explicitly inapplicable', () => { + checkPayloadCleanup(fixture, {mode: 'embedded', payload_cleanup: {applicable: false, reason: 'embedded_has_no_http_system_routes'}}, id); + assert.throws(() => checkPayloadCleanup(fixture, {mode: 'embedded', payload_cleanup: {applicable: true}}, id)); +}); + +const response = (state, label) => state.requests[label].response; +for (const [label, mutate] of [ + ['an unimplemented route', state => { state.requests.pass = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; }], + ['a missing request', state => { delete state.requests['pass limit 5']; }], + ['another namespace', state => { response(state, 'pass').namespace = 'default'; }], + ['a pass that is not recorded', state => { response(state, 'status after pass').cleanup.totals.passes = 0; }], + ['a refused pass counted', state => { response(state, 'status final').cleanup.totals.passes = 9; }], + ['a millisecond completion time', state => { response(state, 'status final').cleanup.last_pass.completed_at = '2026-10-22T23:08:38.123000Z'; }], + ['a missing completion time', state => { response(state, 'status after pass').cleanup.last_pass.completed_at = null; }], + ['an ignored limit', state => { response(state, 'pass limit 5').cleanup.scan_limit = 100; }], + ['an accepted fractional limit', state => { state.requests['pass limit fraction'] = {status: 200, response: {}}; }], + ['a refused null limit', state => { state.requests['pass limit null'].status = 422; }], + ['a min message for the maximum', state => { response(state, 'pass limit 1001').message = 'The limit field must be at least 1.'; }], + ['a pass that reports pressure', state => { response(state, 'pass').cleanup.scan_pressure = true; }], + ['another expiry bound', state => { response(state, 'status fresh').cleanup.bounds.abandoned_upload_expiry_seconds = 60; }], + ['an exposed storage location', state => { response(state, 'status fresh').cleanup.identity_policy.provider_locations_exposed = true; }], +]) { + test(`the payload cleanup rejects ${label}`, () => { + const value = observation(); + mutate(value.payload_cleanup); + assert.throws(() => checkPayloadCleanup(fixture, value, id)); + }); +} diff --git a/tests/Unit/ServerParityPrometheusMetricsContractTest.mjs b/tests/Unit/ServerParityPrometheusMetricsContractTest.mjs new file mode 100644 index 000000000..42cc12799 --- /dev/null +++ b/tests/Unit/ServerParityPrometheusMetricsContractTest.mjs @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkPrometheusMetrics} from '../../scripts/conformance/server-parity/prometheus-metrics-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/prometheus-metrics.json', import.meta.url))); +const definition = fixture.prometheus_metrics; +const id = 'parity-v1-prometheus-metrics'; +const namespace = `${id}-prometheus`; +const [first, second] = [`${namespace}-q1`, `${namespace}-q2`]; +const [typeA, typeB] = definition.workflow_types; +const bounds = [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900]; +const selection = { + workflows: 'bounded_task_queue_and_workflow_type_ascending', + activities: 'bounded_task_queue_workflow_type_and_activity_type_ascending', + task_queues: 'task_queue_name_ascending', +}; +const dimensions = {workflows: ['task_queue', 'workflow_type'], activities: ['task_queue', 'workflow_type', 'activity_type'], + task_queues: ['task_queue']}; + +// A histogram of durations, cumulative or with PHP-on-SQLite's text bounds. +function latency(durations, textBounds = false) { + return {count: durations.length, sum: durations.reduce((sum, value) => sum + value, 0), + buckets: Object.fromEntries(bounds.map(bound => [String(bound), + textBounds ? durations.length : durations.filter(value => value <= bound).length]))}; +} + +function queue(name, values = {}) { + const entry = {task_queue: name}; + for (const prefix of ['workflow', 'activity']) { + for (const field of ['ready_tasks', 'ready_due_tasks', 'delayed_tasks', 'leased_tasks', 'expired_leases', + 'tasks_added_last_minute', 'tasks_dispatched_last_minute']) { + entry[`${prefix}_${field}`] = 0; + } + } + for (const field of ['active_pollers', 'workflow_slot_capacity', 'activity_slot_capacity', 'available_workflow_slots', + 'available_activity_slots']) { + entry[field] = 0; + } + return {...entry, ...values}; +} + +function summary(series) { + const counts = Object.fromEntries(Object.entries(series).map(([family, list]) => [family, list.length])); + const total = counts.workflows + counts.activities + counts.task_queues; + const labels = family => ({namespace: 'request_scope_not_label', ...Object.fromEntries(dimensions[family] + .map(dimension => [dimension, {cardinality_class: 'bounded_series', limit: 100, selection: selection[family]}]))}); + const disclosure = family => ({label_dimensions: dimensions[family], limit: 100, observed_series_count: counts[family], + observed_series_count_precision: 'exact', reported_series_count: counts[family], suppressed_series_count: 0, + suppressed_started_total: 0, suppressed_counts_precision: 'exact', truncated: false, selection: selection[family]}); + return {status: 200, response: {schema: 'durable-workflow.prometheus-metrics.v1', generated_at: '2026-10-22T22:29:44.981017Z', + namespace, latency_buckets_seconds: bounds, series, series_count: total, observed_series_count: total, + observed_series_count_precision: 'exact', suppressed_series_count: 0, suppressed_series_count_precision: 'exact', + cardinality: {series_limits: {workflows: disclosure('workflows'), activities: disclosure('activities'), + task_queues: disclosure('task_queues')}, metric_label_sets: {dw_workflow_runs_total: labels('workflows'), + dw_workflow_run_latency_seconds: labels('workflows'), dw_activity_executions_total: labels('activities'), + dw_activity_execution_latency_seconds: labels('activities'), dw_task_queue_runtime_state: labels('task_queues')}}}}; +} + +function observation(textBounds = false) { + const run = (queueName, type, values, durations) => ({task_queue: queueName, workflow_type: type, started_total: 1, + completed_total: 0, failed_total: 0, cancelled_total: 0, terminated_total: 0, running: 0, ...values, + latency_seconds: latency(durations, textBounds)}); + const read = summary({ + workflows: [run(first, typeA, {completed_total: 1}, [2.673]), run(first, typeB, {terminated_total: 1}, [1.161]), + run(second, typeA, {running: 1}, [])], + activities: [{task_queue: first, workflow_type: typeA, activity_type: definition.activity_type, started_total: 1, + completed_total: 1, failed_total: 0, cancelled_total: 0, running: 0, latency_seconds: latency([1.102992], textBounds)}], + task_queues: [queue(first, {...definition.first_queue, active_pollers: 1, workflow_slot_capacity: 4, activity_slot_capacity: 3, + available_workflow_slots: 4, available_activity_slots: 3}), queue(second, definition.second_queue)], + }); + return structuredClone({mode: 'http', prometheus_metrics: {applicable: true, namespace, queues: [first, second], + statuses: definition.statuses, empty: summary({workflows: [], activities: [], task_queues: []}), read}}); +} + +test('the Prometheus summary accepts the cumulative histogram', () => { + assert.deepStrictEqual(checkPrometheusMetrics(fixture, observation(), id), {scope: 'prometheus_metrics', series: 6}); +}); + +test("the Prometheus summary records PHP on SQLite's text-bound buckets", () => { + checkPrometheusMetrics(fixture, observation(true), id); +}); + +test('the embedded Prometheus summary is explicitly inapplicable', () => { + checkPrometheusMetrics(fixture, {mode: 'embedded', prometheus_metrics: {applicable: false, reason: 'embedded_has_no_http_metrics'}}, id); + assert.throws(() => checkPrometheusMetrics(fixture, {mode: 'embedded', prometheus_metrics: {applicable: true}}, id)); +}); + +const read = state => state.read.response; +for (const [label, mutate] of [ + ['an unimplemented route', state => { state.read = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; }], + ['a refused scenario request', state => { state.statuses.terminate = 422; }], + ['another namespace', state => { read(state).namespace = 'default'; }], + ['a missing series', state => { read(state).series.workflows.pop(); }], + ['series out of order', state => { read(state).series.workflows.reverse(); }], + ['a terminated run counted as cancelled', state => { Object.assign(read(state).series.workflows[1], {terminated_total: 0, cancelled_total: 1}); }], + ['a waiting run not running', state => { read(state).series.workflows[2].running = 0; }], + ['a duration for a waiting run', state => { read(state).series.workflows[2].latency_seconds.count = 1; }], + ['a duration in a bucket below its length', state => { read(state).series.workflows[0].latency_seconds.buckets['0.5'] = 1; }], + ['non-cumulative buckets', state => { read(state).series.workflows[0].latency_seconds.buckets['900'] = 0; }], + ['a sum below the minimum duration', state => { read(state).series.activities[0].latency_seconds.sum = 0.2; }], + ['a missing bucket', state => { delete read(state).series.activities[0].latency_seconds.buckets['300']; }], + ['a delayed due task', state => { Object.assign(read(state).series.task_queues[1], {workflow_ready_due_tasks: 0, workflow_delayed_tasks: 1}); }], + ['missing worker capacity', state => { read(state).series.task_queues[0].workflow_slot_capacity = 0; }], + ['a worker counted on the second queue', state => { read(state).series.task_queues[1].active_pollers = 1; }], + ['an undisclosed truncation', state => { read(state).cardinality.series_limits.workflows.truncated = true; }], + ['another series limit', state => { read(state).cardinality.metric_label_sets.dw_workflow_runs_total.task_queue.limit = 50; }], + ['a millisecond time', state => { read(state).generated_at = '2026-10-22T22:29:44.981Z'; }], + ['an extra field', state => { read(state).control_plane = {}; }], +]) { + test(`the Prometheus summary rejects ${label}`, () => { + const value = observation(); + mutate(value.prometheus_metrics); + assert.throws(() => checkPrometheusMetrics(fixture, value, id)); + }); +} diff --git a/tests/Unit/ServerParityRepairPassContractTest.mjs b/tests/Unit/ServerParityRepairPassContractTest.mjs new file mode 100644 index 000000000..5fbb73f75 --- /dev/null +++ b/tests/Unit/ServerParityRepairPassContractTest.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkRepairPass, repairPassCases} from '../../scripts/conformance/server-parity/repair-pass-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/repair-pass.json', import.meta.url))); +const definition = fixture.repair_pass; +const id = 'parity-v1-repair-pass'; +const own = `${id}-repair`; + +const observation = () => { + const responses = {}; + for (const [label, , [status, response]] of repairPassCases(fixture, own)) { + responses[label] = {status, response: structuredClone(response)}; + } + return {mode: 'http', repair_pass: {applicable: true, namespace: own, responses}}; +}; + +test('the repair pass accepts the published answers', () => { + assert.deepStrictEqual(checkRepairPass(fixture, observation(), id), {scope: 'repair_pass', cases: 9}); +}); + +test('the fixture scopes every pass away from other work', () => { + for (const [label, body] of definition.requests) { + assert.ok(Array.isArray(body.run_ids) || typeof body.run_ids === 'string', `${label} names run ids`); + assert.ok(typeof body.queue !== 'string' || body.queue.includes('@own') || body.queue.length > 128, `${label} names its own queue`); + } + const passes = Object.values(definition.expected).filter(([status]) => status === 200); + assert.equal(passes.length, 3); + for (const [, report] of passes) { + assert.equal(report.selected_total_candidates, 0); + assert.equal(report.throttled, false); + } +}); + +test('the placeholders become the probe namespace', () => { + const [, body, [, report]] = repairPassCases(fixture, own).find(([label]) => label === 'trimmed filters'); + assert.equal(body.instance_id, ` ${own}-instance `); + assert.equal(report.instance_id, `${own}-instance`); + assert.ok(!JSON.stringify(repairPassCases(fixture, own)).includes('@own')); +}); + +test('the embedded repair pass is explicitly inapplicable', () => { + checkRepairPass(fixture, {mode: 'embedded', repair_pass: {applicable: false, reason: 'embedded_has_no_http_system_routes'}}, id); + assert.throws(() => checkRepairPass(fixture, {mode: 'embedded', repair_pass: {applicable: true}}, id)); +}); + +const report = (state, label) => state.responses[label].response; +for (const [label, mutate] of [ + ['an unimplemented route', state => { state.responses['filtered pass'] = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; }], + ['a missing request', state => { delete state.responses['queue too long']; }], + ['another namespace', state => { state.namespace = 'default'; }], + ['untrimmed filters', state => { report(state, 'trimmed filters').queue = ` ${own}-queue `; }], + ['an empty instance kept as text', state => { report(state, 'empty instance').instance_id = ''; }], + ['a throttled pass', state => { report(state, 'filtered pass').throttled = true; }], + ['a selected candidate', state => { report(state, 'filtered pass').selected_total_candidates = 1; }], + ['a missing service call count', state => { delete report(state, 'filtered pass').service_call_timeout_candidates; }], + ['a partial pass', state => { state.responses['filtered pass'].status = 207; }], + ['an accepted text throttle', state => { state.responses['throttle not boolean'] = {status: 200, response: {}}; }], + ['another run id message', state => { report(state, 'empty run id').errors['run_ids.0'] = ['The run_ids.0 field must be a string.']; }], + ['an accepted run id list over 100', state => { state.responses['too many run ids'].status = 200; }], +]) { + test(`the repair pass rejects ${label}`, () => { + const value = observation(); + mutate(value.repair_pass); + assert.throws(() => checkRepairPass(fixture, value, id)); + }); +} diff --git a/tests/Unit/ServerParitySystemMetricsContractTest.mjs b/tests/Unit/ServerParitySystemMetricsContractTest.mjs new file mode 100644 index 000000000..4f1886de8 --- /dev/null +++ b/tests/Unit/ServerParitySystemMetricsContractTest.mjs @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkSystemMetrics} from '../../scripts/conformance/server-parity/system-metrics-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/system-metrics.json', import.meta.url))); +const definition = fixture.system_metrics; +const id = 'parity-v1-system-metrics'; +const namespace = `${id}-metrics`; +const resources = Object.keys(definition.usage); +const zeros = keys => Object.fromEntries(keys.map(key => [key, 0])); +const nulls = keys => Object.fromEntries(keys.map(key => [key, null])); + +// A response as published Server 2.11.1 answered the scenario, with the +// timeline drift PHP's asynchronous projection showed. +function response(scenario, timelineDrift = 0) { + const failures = scenario ? definition.failures : {...definition.failures, max_consecutive_failures: 0, + failed_task_count: 0, workflow_type_count: 0, by_workflow_type: []}; + const counts = definition.projection_tables.map(table => table === 'run_timeline_entries' ? timelineDrift : 0); + const metrics = { + dw_workflow_task_consecutive_failures: failures, + dw_projection_drift_total: { + total: timelineDrift, table_count: 5, tables_with_drift: timelineDrift > 0 ? 1 : 0, scope: 'server', + label_cardinality_policy: {namespace: 'server_scope_no_label', table: {cardinality_class: 'finite_projection_table_inventory', + values: definition.projection_tables, selection: 'fixed_projection_table_inventory'}}, + by_table: definition.projection_tables.map((table, index) => ({table, needs_rebuild: counts[index]})), + }, + dw_namespace_request_admission_rejections: { + max_requests_per_minute: null, max_concurrent_requests: null, requests_this_minute: 0, rejections_this_minute: 0, + rejections_by_reason: zeros(['namespace_request_rate_exhausted', 'namespace_request_concurrency_exhausted', + 'namespace_request_admission_unavailable']), + configuration_status: 'valid', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_three_reason_inventory'}, + }, + dw_namespace_durable_state_usage: { + limits: nulls(resources), usage: scenario ? definition.usage : zeros(resources), remaining: nulls(resources), + rejections_this_minute: 0, + rejections_by_reason: zeros([...resources.map(resource => `namespace_${resource}_exhausted`), 'namespace_durable_state_quota_unavailable']), + configuration_status: 'valid', measurement_status: 'available', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_reason_inventory'}, + }, + dw_runtime_external_payload_namespace_usage: { + max_bytes: null, max_objects: null, used_bytes: 0, used_objects: 0, remaining_bytes: null, remaining_objects: null, + rejections_this_minute: 0, rejections_by_reason: zeros(['external_payload_namespace_bytes_exhausted', + 'external_payload_namespace_objects_exhausted', 'external_payload_namespace_quota_unavailable']), + configuration_status: 'valid', measurement_status: 'available', + label_cardinality_policy: {namespace: 'request_scope_not_label', reason: 'finite_three_reason_inventory'}, + }, + }; + const cardinality = {metric_label_sets: Object.fromEntries(Object.entries(metrics) + .map(([name, family]) => [name, family.label_cardinality_policy]))}; + return structuredClone({status: 200, response: {generated_at: '2026-10-22T22:15:23.466270Z', namespace, metrics, cardinality}}); +} + +const observation = (timelineDrift = 1) => ({mode: 'http', system_metrics: { + applicable: true, namespace, empty: response(false), registered: 201, started: [201, 201], + claimed_type: definition.workflow_types[0], failed: 200, scheduled: 201, + read: response(true, timelineDrift), posted: response(true, timelineDrift), +}}); + +test('the system metrics accept the published answers', () => { + assert.deepStrictEqual(checkSystemMetrics(fixture, observation(), id), {scope: 'system_metrics', resources: 16}); +}); + +test('the system metrics accept native answers with no drift and millisecond times', () => { + const value = observation(0); + value.system_metrics.read.response.generated_at = '2026-10-22T22:15:23.466Z'; + checkSystemMetrics(fixture, value, id); +}); + +test('the fixture pins the scenario counts', () => { + assert.deepStrictEqual(resources.length, 16); + assert.deepStrictEqual([definition.usage.workflow_tasks, definition.usage.pending_workflow_tasks, definition.usage.workflow_commands], [3, 2, 2]); + assert.deepStrictEqual(definition.failures.by_workflow_type, [{workflow_type: 'parity.v1.metrics_a', max_consecutive_failures: 1, failed_task_count: 1}]); +}); + +test('the embedded system metrics are explicitly inapplicable', () => { + checkSystemMetrics(fixture, {mode: 'embedded', system_metrics: {applicable: false, reason: 'embedded_has_no_http_metrics'}}, id); + assert.throws(() => checkSystemMetrics(fixture, {mode: 'embedded', system_metrics: {applicable: true}}, id)); +}); + +for (const [label, mutate] of [ + ['an unimplemented route', state => { state.read = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; }], + ['a refused POST', state => { state.posted.status = 405; }], + ['another namespace', state => { state.read.response.namespace = 'default'; }], + ['a wrong durable-state count', state => { state.read.response.metrics.dw_namespace_durable_state_usage.usage.workflow_tasks = 2; }], + ['a configured limit', state => { state.read.response.metrics.dw_namespace_durable_state_usage.limits.schedules = 10; }], + ['a missing quota reason', state => { delete state.read.response.metrics.dw_namespace_durable_state_usage.rejections_by_reason.namespace_durable_state_quota_unavailable; }], + ['the failure on the other type', state => { state.read.response.metrics.dw_workflow_task_consecutive_failures.by_workflow_type[0] = {workflow_type: 'parity.v1.metrics_b', max_consecutive_failures: 1, failed_task_count: 1}; }], + ['a failure counted before any attempt', state => { state.empty.response.metrics.dw_workflow_task_consecutive_failures.failed_task_count = 1; }], + ['a stored external payload', state => { state.read.response.metrics.dw_runtime_external_payload_namespace_usage.used_objects = 1; }], + ['a drift total unlike its tables', state => { state.read.response.metrics.dw_projection_drift_total.total = 0; }], + ['a negative drift', state => { state.read.response.metrics.dw_projection_drift_total.by_table[0].needs_rebuild = -1; }], + ['a missing drift table', state => { state.read.response.metrics.dw_projection_drift_total.by_table.pop(); }], + ['a namespace-scoped drift', state => { state.read.response.metrics.dw_projection_drift_total.scope = 'namespace'; }], + ['an extra metric family', state => { state.read.response.metrics.dw_extra = {label_cardinality_policy: {}}; }], + ['a cardinality summary unlike its families', state => { state.read.response.cardinality.metric_label_sets.dw_projection_drift_total = {}; }], + ['an extra top-level field', state => { state.read.response.control_plane = {}; }], + ['a local time', state => { state.read.response.generated_at = '2026-10-22 22:15:23'; }], + ['the second type claimed first', state => { state.claimed_type = definition.workflow_types[1]; }], +]) { + test(`the system metrics reject ${label}`, () => { + const value = observation(); + mutate(value.system_metrics); + assert.throws(() => checkSystemMetrics(fixture, value, id)); + }); +}