diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 9cecaf1bf..c4e233777 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -71,6 +71,9 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore the patch cohort's maintenance loop on every path + run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs + - name: Validate embedded Workflow package authority run: php scripts/ci/test-workflow-package-authority.php diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 9960470e7..8ece6ee8b 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -43,6 +43,7 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -99,6 +100,7 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1529,7 +1531,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md new file mode 100644 index 000000000..b96d9b562 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-condition-wait-fields.md @@ -0,0 +1,21 @@ +# October 23, optional condition wait fields + +- **The gap.** Native refused an `open_condition_wait` command without a + `condition_key`, or without a `condition_definition_fingerprint` of the form + `sha256:` followed by 64 hex digits. Published PHP accepts both. +- **How PHP 2.11.1 does it.** `normalizeOpenConditionWaitCommand` treats the key + and the fingerprint as optional strings of any form. It trims them and leaves + them out when empty. `applyOpenConditionWait` records only the fields that + remain, on `ConditionWaitOpened` and on a condition timeout's + `TimerScheduled`. Run summaries name a keyless wait "Waiting for condition", + without a key. Observed live against the published image: a bare + `open_condition_wait`, a key of `" ready "` with the fingerprint + `"any form"`, and blank values all behave this way. +- **What native does now.** The same. The open wait projection stores no + target name for a keyless wait, and the run summary reads such a wait as + PHP does. A key or fingerprint that is not a string is still refused by the + completion transport rules. +- **How it was checked.** A new integration test covers a bare wait, a padded + key with an arbitrary fingerprint, blank values with a timeout, and a key + that is not a string. The 63 existing condition, timed-condition and + observation tests still pass. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md new file mode 100644 index 000000000..c369f493f --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-message-streams.md @@ -0,0 +1,64 @@ +# October 23, message streams + +- **The gap.** All three published SDKs declare `message-streams` from + worker protocol 1.15. Native refused a worker that asked for them, did not + serve the append and diagnostics routes, and ignored + `message_stream_cursors` and `message_stream_waits` on a completion. +- **How PHP 2.11.1 does it.** Learned from its source, then observed live + against the published image: + - Each accepted message is delivered to the current run as the + runtime-reserved signal `__durable_workflow_message_stream`, in the + append's transaction. The signal skips the declared-signal and argument + checks but not the pending-signal and payload ceilings. + - SDK receives open a condition wait keyed by the stream cursor. A stream + message wakes that wait. + - When a signal wakes a condition wait, PHP consumes it without + `SignalApplied`, unless the run's durable contract declares that signal. + This holds for any undeclared signal, not only stream messages. + - The stream cursor's `MessageCursorAdvanced` is recorded after the + completion, with the completed task's snapshot. + - On continue-as-new, the next run receives a payload-free cursor + checkpoint for each acknowledged stream, then every unconsumed message + again, by append order. The old run rejects the signals it had not + consumed. +- **What native does now.** + - `POST /api/workflows/{id}/message-streams/{stream}/messages`, with PHP's + validation order, refusals and diagnostics of malformed input. + - `GET /api/workflows/{id}/message-streams` and + `GET /api/workflows/{id}/message-streams/{stream}`. + - Completion cursors and waits: validated before anything changes, + recorded in the completion's transaction, handed over on continue-as-new. + - The worker capability is PHP's contract with `supported: true`, and + cluster info publishes `message_streams_contract`. + - Continue-as-new no longer refuses a run with message streams. +- **Native changes found on the way.** + - A signal consumed by a condition wait no longer records `SignalApplied` + unless it is declared, and keeps the wait's sequence. + - The instance-level `MessageCursorAdvanced` carries the task snapshot, as + PHP's does. + - A completion reports the run's next message task in + `created_task_ids`, as PHP does. +- **How it was checked.** + - The `message-streams` fixture: 15 appends, 8 refused completions, the + deliveries each worker read, both runs' full event sequences, and the + diagnostics at each step. + - Published PHP 2.11.1 and native pass it on SQLite. The probe's output + is the same size on both, and both runs' event sequences are identical. + - The contract test refuses 50 changed observations, among them a + `SignalApplied` for a consumed message, a missing next message task, a + redelivered consumed message and a missing cursor checkpoint. +- **Not done yet** (Server #325 follow-ups): + - **External payload references.** Native refuses a reference as stream + input with 501 `message_stream_external_payload_unavailable`. PHP keeps + the reference and resolves it when history reaches a worker. + - **Cleanup and upgrade.** Retention cleanup of streams + (`MessageStreamRetentionCleanup`) is not ported. Takeover of PHP data + with streams is still refused. + - **Cluster `capabilities` map.** Native has none yet, so + `capabilities.message_streams` is absent; that map is the cluster info + work. + - **History presentation, outside streams.** Task snapshots lack PHP's + `last_dispatch_attempt_at` and `last_dispatched_at`. A consumed signal's + `SignalReceived` lacks `workflow_sequence` in poll history. + - **Condition fingerprint.** Native requires `condition_definition_fingerprint` + on `open_condition_wait`; PHP accepts a wait without one. diff --git a/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md new file mode 100644 index 000000000..f139ee749 --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-23-task-snapshots.md @@ -0,0 +1,79 @@ +# October 23, task snapshots on history events and described commands + +- **The gap.** Published PHP's `WorkflowHistoryEvent::record` adds a `task` + snapshot to every event it records with the task model in hand. Native + added snapshots event by event and missed several. Driving one workflow + through a side effect, an activity, a timer, a child, a signal wait and a + condition wait against both servers showed that native had no snapshot on: + - `TimerFired`, `ChildWorkflowScheduled`, `ChildRunStarted`; + - `SignalWaitOpened`, `ConditionWaitOpened`, `ConditionWaitSatisfied`; + - `SignalApplied` and the instance `MessageCursorAdvanced`. + + Native's own snapshots on `TimerScheduled`, `ActivityStarted`, + `ActivityCompleted` and `ActivityFailed` also left out fields that PHP + keeps, such as `queue`, `connection`, `available_at` and `repair_count`. +- **What native does now.** + - When an event of a type PHP records with the task model is appended with + a task and no snapshot of its own, native adds the task's snapshot. The + list of those event types comes from the Workflow package's source. + - `TimerScheduled` uses the full snapshot. The activity events' snapshots + keep PHP's fields. + - As `RunTimerTask` does, a timer task is leased to itself while its timer + fires: status leased, `leased_at`, `lease_owner` set to the task id, a + workflow-task lease, and one more attempt. Its lease is cleared once the + timer has fired. `TimerFired` therefore carries the leased timer task. +- **How it was checked.** The same probe now finds the same events with + snapshots on both servers, with the same fields. A new integration test + checks the timer, child, signal-wait, signal and condition-wait events. +- **Described commands.** A field-by-field diff of the same workflow's + description on both servers found two more gaps. Each command PHP + describes names the run's `workflow_class` (`WorkflowCommand::record` keeps + it), and says whether its payload preview is omitted + (`payload_preview_omitted`, true only for a stored external payload over the + size limit). Native now sends both. With fresh workers on both sides, the + descriptions then differ only in `request_route_name` and the + `request_fingerprint` hashed from it. +- **Security review corrections.** + - The size check read only PHP's `external_storage` envelope. Native's + stored envelope names its reference `external_payload`, so the flag was + always false. It now reads both, which also corrects the run's input and + output previews. As PHP does, an omitted preview also omits the + command's `target_name`. + - The query read any command payload starting with `dw-`, and SQLite's + `LIKE` and MySQL's case-insensitive collation ignore case. It also read the + wrong column for signals. Native keeps a signal's stored input reference + in its signal record (`workflow_signal_records.arguments`) and a start's + in the command row (`workflow_commands.payload`). Other commands keep + none. The query now reads only that column for each of the two command + types, and only when its first 30 characters equal native's stored prefix + exactly: `COLLATE BINARY` on SQLite, a binary cast on MySQL and MariaDB, + and PostgreSQL's case-sensitive text comparison. On MySQL 8.0 and MariaDB + 10.11 the binary cast refuses the upper-case prefix that the column's + collation would match. + - A task snapshot is read only from a task of the event's own run. + - Tests: + - a real uploaded reference on a signal, found in the signal record and + not in the command row; + - that signal record rewritten with the same reference at the 2 MiB limit + (not omitted) and one byte over (omitted, with no target name). The + described command never includes the reference; + - counterfactuals that are not native references: an upper-case or v2 + prefix, the prefix with no valid reference after it, and a large inline + payload starting with `dw-`. An oversized reference in the signal's + command row is not counted either; + - the same sizes and counterfactuals in a start command's row; + - on SQLite, the query itself returns only the exact prefix in the + command type's own column; + - an event recorded on one run with another run's task carries no + snapshot. + + Each test fails without its correction. +- **Left as they are.** + - PHP's command snapshots carry `request_route_name`, a name Laravel + generates at random for the deployment's cached routes. Native has no + such name. + - Native stamps a dispatch on every new task, so the snapshot of a task + created inside a completion keeps `last_dispatch_attempt_at` and + `last_dispatched_at`, which PHP's does not (Server #325 follow-up). + - A message stream's `SignalApplied` and the condition wait it satisfies + follow the message streams change (#709). diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 582784510..eec7b7b9b 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -144,7 +144,10 @@ any other unknown path or method answers PHP's own 404 or 405 - **Run operations.** - Imports from embedded v2 and Waterline v1. - **Other families.** - - Message streams. + - Message streams are served + ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), + except external payload references as input, stream retention cleanup + and takeover of PHP data with streams. - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every @@ -271,9 +274,12 @@ contract fixture before its correction. Add a fixture for each before changing them. - **Retention of runs with external payloads or message streams.** Published PHP deletes a pruned run's external payload objects and releases - its message stream items. Native does not clean either up yet. It skips - such a run with `external_payload_cleanup_unavailable` or - `message_stream_cleanup_unavailable` and keeps its details. + its message stream items. It also compacts its instance's consumed inbound + stream items, and removes the instance's streams with its last run. Native + does not clean any of these up yet. It skips such a run, including any run + whose instance has inbound streams, with + `external_payload_cleanup_unavailable` or + `message_stream_cleanup_unavailable`, and keeps its details. - **Deeply nested JSON bodies.** Published PHP decodes JSON up to 512 levels deep. Native checks every body as PHP does before any route, but its handlers parse with serde_json's 128-level limit. A body nested 129 to 512 diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 062f7720a..e24c73bb7 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -540,7 +540,9 @@ where } payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":count}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":count,"repair_count":DB::number(&task,"repair_count")?,"queue":DB::string(&task,"queue")?, + "connection":DB::PHP_CONNECTION}); let failed_event = payload.clone(); Self::append( &mut tx, diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 52c26eb99..d4764e814 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -35,6 +35,12 @@ pub(super) trait Backend: Database { const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND payload->>'timer_id'=$2 ORDER BY sequence DESC LIMIT 1"; /// The history sequence of a signal's `SignalReceived` event. const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND payload->>'signal_id'=$2 ORDER BY sequence LIMIT 1"; + // PostgreSQL compares text case-sensitively. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "substr(s.arguments,1,30)='dw-native-external-payload-v1:'", + "substr(c.payload,1,30)='dw-native-external-payload-v1:'" + ); const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND payload->>'selection_group_id'=$2 ORDER BY sequence LIMIT 1"; // Frozen run command/task repair counters use each backend's actual width. const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; @@ -185,6 +191,11 @@ ORDER BY t.available_order,t.id LIMIT 100 const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='timer' AND status='ready' AND json_extract(payload,'$.timer_id')=$2 AND namespace=$3"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='TimerScheduled' AND json_extract(payload,'$.timer_id')=$2 ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SignalReceived' AND json_extract(payload,'$.signal_id')=$2 ORDER BY sequence LIMIT 1"; + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "substr(s.arguments,1,30) COLLATE BINARY='dw-native-external-payload-v1:'", + "substr(c.payload,1,30) COLLATE BINARY='dw-native-external-payload-v1:'" + ); const HISTORY_PAGE: &str = "SELECT * FROM (SELECT *,SUM(length(CAST(payload AS BLOB))) OVER (ORDER BY sequence) AS page_bytes FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT $3) AS page WHERE page_bytes<=8388608 ORDER BY sequence"; // Normalize only the two supported UTC storage shapes. Removing trailing // fractional zeros preserves exact decimal ordering, including earlier @@ -457,6 +468,12 @@ impl Backend for MySql { const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; const SIGNAL_RECEIVED_SEQUENCE_SQL: &str = "SELECT sequence FROM workflow_history_events WHERE workflow_run_id=? AND event_type='SignalReceived' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.signal_id'))=? ORDER BY sequence LIMIT 1"; + // The payload column's collation ignores case; compare bytes. + const RUN_COMMANDS_SQL: &str = + super::store::run_commands_sql!( + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)='dw-native-external-payload-v1:'", + "CAST(SUBSTRING(c.payload,1,30) AS BINARY)='dw-native-external-payload-v1:'" + ); // First use the preserved run/sequence index to materialize only IDs and // byte counts for this bounded page. Sorting JSON payloads in the window // exhausts MySQL's default sort buffer before the size rejection can run. diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 7a5d4d62c..e8590924a 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -105,17 +105,15 @@ where } // Published PHP continues a run whatever it has open. Its activities, // timers, children and waits stay with the closed predecessor, and no - // parent-close policy applies. Streams, service calls and live - // queries are not transferred by native yet. + // parent-close policy applies. Message streams pass to the next run + // when the completion records its stream acknowledgements. Durable + // output streams, service calls and live queries are not transferred + // by native yet. for (sql, selected) in [ ( "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", workflow.as_str(), ), - ( - "SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index a75eadc46..4c1c00360 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -585,6 +585,89 @@ impl Runtime { Ok(view) } + delegate!(message_stream_workflow_bound(workflow_id: &str) -> Result); + delegate!(record_malformed_message(workflow_id: &str, stream: &str, message_id: &Value) -> Result<()>); + delegate!(message_stream_diagnostics(workflow_id: &str, stream: Option<&str>) -> Result>); + + /// `MessageStreamController::append`: the stream name, the workflow in + /// this namespace, the request rules and the payload envelope, in PHP's + /// order. A refused request or payload counts as malformed input. + pub(crate) async fn append_message(&self, workflow_id: &str, stream: &str, body: Value, context: Value) -> Result<(StatusCode, Value)> { + super::message_streams::check_path_name(stream)?; + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"accepted":false,"workflow_id":workflow_id,"stream_name":stream,"outcome":"rejected","reason":"instance_not_found"}), + }); + } + let body = super::laravel_rules::normalize(&body); + let blob = match self.message_payload(&body).await { + Ok(blob) => blob, + Err(error) => { + if matches!(&error, super::RuntimeError::Protocol { status, .. } | super::RuntimeError::Refused { status, .. } if *status == StatusCode::UNPROCESSABLE_ENTITY) { + self.record_malformed_message(workflow_id, stream, &body["message_id"]).await?; + } + return Err(error); + } + }; + let message_id = body["message_id"].as_str().unwrap_or_default().to_owned(); + let (codec, structural) = (self.signal_codec.clone(), self.structural); + match &*self.storage { + Storage::Sqlite(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::Postgres(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + Storage::MySql(store) => store.append_message(workflow_id, stream, &message_id, "avro", &blob, &context, codec, structural).await, + } + } + + /// The request rules, then `AvroPayloadEnvelopeResolver::resolve`: no + /// input is an empty argument list, plain values are encoded, and an + /// inline envelope keeps its exact bytes. + async fn message_payload(&self, body: &Value) -> Result { + super::message_streams::check_append(body)?; + if body.get("input").is_none_or(Value::is_null) { + return super::queries::empty_arguments(self.signal_codec.clone()).await; + } + super::input_envelope::transport(body)?; + if let Some(blob) = self.plain_arguments(body).await? { + return Ok(blob); + } + if self.prepare_external(&body["input"]).await?.is_some() { + // PHP keeps the reference and resolves it for workers; native + // does not resolve stream payload references yet. + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_IMPLEMENTED, + response: json!({"accepted":false,"workflow_id":null,"message_id":body["message_id"],"outcome":"rejected", + "reason":"message_stream_external_payload_unavailable"}), + }); + } + self.decodable(&body["input"]["blob"]).await?; + super::envelope(body, "input") + } + + /// `MessageStreamController::index` and `show`. + pub(crate) async fn message_streams(&self, workflow_id: &str, stream: Option<&str>) -> Result { + if let Some(stream) = stream { + super::message_streams::check_path_name(stream)?; + } + if !self.message_stream_workflow_bound(workflow_id).await? { + return Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"message":"Workflow not found.","reason":"instance_not_found"}), + }); + } + let streams = self.message_stream_diagnostics(workflow_id, stream).await?; + match stream { + None => Ok(json!({"workflow_id":workflow_id,"count":streams.len(),"streams":streams})), + Some(stream) => match streams.into_iter().next() { + Some(found) => Ok(json!({"workflow_id":workflow_id,"stream":found})), + None => Err(super::RuntimeError::Protocol { + status: StatusCode::NOT_FOUND, + response: json!({"workflow_id":workflow_id,"stream_name":stream,"reason":"message_stream_not_found"}), + }), + }, + } + } + /// `ActivityController::index`. pub(crate) async fn standalone_activities(&self, query: Value) -> Result { let request = super::standalone_activities::list_request(&query)?; @@ -1256,17 +1339,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol) + .complete_workflow(task_id, body, self.transition_codec.clone(), external, worker, protocol, self.structural) .await } } diff --git a/rust/src/runtime/external_wait_summary.rs b/rust/src/runtime/external_wait_summary.rs index 0f9a959a1..06abf03fa 100644 --- a/rust/src/runtime/external_wait_summary.rs +++ b/rust/src/runtime/external_wait_summary.rs @@ -19,6 +19,8 @@ pub(super) struct ExternalWaitHistory { struct ExpectedWait { kind: &'static str, + /// The signal name, or the condition key; PHP allows a condition + /// without one, which reads as empty. name: String, sequence: i64, timeout: Option, @@ -27,6 +29,26 @@ struct ExpectedWait { identity: Value, } +/// A wait's name: a signal always has one; a condition's key is optional. +fn wait_name<'a>(kind: &str, value: &'a Value) -> Option<&'a str> { + match value.as_str().filter(|name| !name.is_empty()) { + Some(name) => Some(name), + None if kind == "condition" && value.is_null() => Some(""), + None => None, + } +} + +impl ExpectedWait { + /// `conditionLabel`: the kind, then the name when there is one. + fn label(&self) -> String { + if self.name.is_empty() { + self.kind.to_owned() + } else { + format!("{} {}", self.kind, self.name) + } + } +} + impl ExternalWaitHistory { pub(super) fn apply(&mut self, kind: &str, payload: &Value) -> bool { if matches!(kind, "TimerScheduled" | "TimerFired" | "TimerCancelled") { @@ -70,7 +92,7 @@ impl ExternalWaitHistory { } let (Some(id), Some(name), Some(sequence)) = ( payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64().filter(|v| *v > 0), ) else { return false; @@ -102,7 +124,7 @@ impl ExternalWaitHistory { let (Some(timer_id), Some(wait_id), Some(name), Some(sequence), Some(delay)) = ( payload["timer_id"].as_str().filter(|v| !v.is_empty()), payload[id_key].as_str().filter(|v| !v.is_empty()), - payload[name_key].as_str().filter(|v| !v.is_empty()), + wait_name(wait_kind, &payload[name_key]), payload["sequence"].as_i64(), payload["delay_seconds"].as_i64(), ) else { @@ -230,14 +252,11 @@ where kind: expected.kind, id: DB::string(row, "wait_id")?, reason: if expected.fired { - format!( - "Waiting to apply {} {} timeout", - expected.kind, expected.name - ) + format!("Waiting to apply {} timeout", expected.label()) } else if expected.timeout.is_some() { - format!("Waiting for {} {} or timeout", expected.kind, expected.name) + format!("Waiting for {} or timeout", expected.label()) } else { - format!("Waiting for {} {}", expected.kind, expected.name) + format!("Waiting for {}", expected.label()) }, started_at: DB::optional_instant(row, "opened_at")?, deadline_at: expected.timer.as_ref().map(|(_, at)| *at), @@ -272,7 +291,7 @@ where } fn external_wait_matches(row: &DB::Row, expected: &ExpectedWait) -> Result { if DB::string(row, "kind")? != expected.kind - || DB::optional_string(row, "target_name")?.as_deref() != Some(expected.name.as_str()) + || DB::optional_string(row, "target_name")?.unwrap_or_default() != expected.name || DB::optional_number(row, "sequence")? != Some(expected.sequence) || DB::optional_string(row, "history_authority")?.as_deref() != Some("typed_history") || DB::optional_string(row, "source_status")?.as_deref() diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 3cc195f41..16ee463f2 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -120,6 +120,12 @@ pub fn router(runtime: Runtime) -> Router { post(repair_run), ) .route("/api/workflows/{workflow_id}/debug", get(debug_current)) + .route("/api/workflows/{workflow_id}/message-streams", get(list_message_streams)) + .route("/api/workflows/{workflow_id}/message-streams/{stream_name}", get(show_message_stream)) + .route( + "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + post(append_message), + ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/debug", get(debug_run), @@ -942,6 +948,7 @@ async fn cluster( "auth_composition_contract": auth_composition, "principal_attribution_contract": principal_attribution, "service_catalog_read_contract": service_catalog, + "message_streams_contract": super::message_streams::manifest(), "payload_codec": "avro", "qualified_for_php_takeover": false}), )) } @@ -983,6 +990,39 @@ fn present(body: &Value, field: &str) -> Option { body.get(field).filter(|value| !value.is_null()).cloned() } +/// `POST /api/workflows/{workflow_id}/message-streams/{stream_name}/messages`. +async fn append_message( + Extension(runtime): Extension, + request: CommandRequest, + Path((workflow_id, stream)): Path<(String, String)>, + Json(body): Json, +) -> Result<(StatusCode, Json)> { + let normalized = super::laravel_rules::normalize(&body); + let context = request.context(&body, &workflow_id, "message_stream_append", metadata(&[ + ("stream_name", Some(json!(stream))), + ("message_id", present(&normalized, "message_id")), + ("request_id", present(&normalized, "request_id")), + ])); + let (status, response) = runtime.append_message(&workflow_id, &stream, body, context).await?; + Ok((status, Json(response))) +} + +/// `GET /api/workflows/{workflow_id}/message-streams`. +async fn list_message_streams( + Extension(runtime): Extension, + Path(workflow_id): Path, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, None).await?)) +} + +/// `GET /api/workflows/{workflow_id}/message-streams/{stream_name}`. +async fn show_message_stream( + Extension(runtime): Extension, + Path((workflow_id, stream)): Path<(String, String)>, +) -> Result> { + Ok(Json(runtime.message_streams(&workflow_id, Some(&stream)).await?)) +} + fn signal_context(request: &CommandRequest, body: &Value, workflow_id: &str, name: &str) -> Value { request.context(body, workflow_id, "signal", metadata(&[ ("request_id", present(body, "request_id")), @@ -1872,7 +1912,9 @@ async fn complete_workflow( headers: HeaderMap, Json(body): Json, ) -> Result> { - // PHP's transport rules come before every guard, ownership included. + // PHP validates the message-stream fields first, then its transport + // rules, before every guard, ownership included. + super::message_streams::check_completion_fields(&body).map_err(worker_refusal)?; let body = super::completion_rules::checked(body).map_err(worker_refusal)?; if body["commands"].as_array().is_some_and(|commands| { commands @@ -1897,6 +1939,16 @@ async fn complete_workflow( "remediation":format!("Complete typed search-attribute updates only through server nodes advertising worker protocol {minimum} or newer.")}), }); } + if !super::message_streams::Completion::of(&body).is_empty() && !protocol_at_least(&headers, 15)? { + let minimum = super::message_streams::MINIMUM_PROTOCOL_VERSION; + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":body["workflow_task_attempt"], + "outcome":"rejected","reason":"message_streams_unavailable", + "requested_version":request_protocol(&headers)?,"minimum_protocol_version":minimum, + "remediation":format!("Send the X-Durable-Workflow-Protocol-Version header with worker protocol {minimum} or newer.")}), + }); + } let attempt = body["workflow_task_attempt"].clone(); runtime .complete_workflow(&task_id, body, Some(request.principal().clone()), headers.get("x-durable-workflow-protocol-version").and_then(|value| value.to_str().ok())) diff --git a/rust/src/runtime/message_streams.rs b/rust/src/runtime/message_streams.rs new file mode 100644 index 000000000..42e87ab1a --- /dev/null +++ b/rust/src/runtime/message_streams.rs @@ -0,0 +1,719 @@ +//! Message streams, as published Server 2.11.1 serves them +//! (`MessageStreamController`, `MessageStreamService`, +//! `MessageStreamsContract`). +//! +//! A client appends ordered messages to a named stream of a workflow +//! instance. The server assigns each its position and delivers it to the +//! current run as the runtime-reserved `__durable_workflow_message_stream` +//! signal, in the append's transaction. Workers acknowledge what they consumed +//! with `message_stream_cursors` on a workflow task completion, which records +//! `MessageCursorAdvanced`, and report open receives with +//! `message_stream_waits`. When a run continues as new, the next run receives +//! a payload-free cursor checkpoint and every unconsumed message again. +//! +//! An external payload reference as message input is refused for now: +//! published PHP keeps it by reference and resolves it when history crosses +//! to a worker, which native does not do yet. +use super::laravel_rules::{self, Rule}; +use super::store::{PreparedControlArguments, SignalWrite, Store, id, now}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, backend::Backend}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::Semaphore; + +pub(super) const INTERNAL_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_SCHEMA: &str = "durable-workflow.v2.message-stream.message"; +const CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor"; +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.15"; +const MAX_BATCH: i64 = 100; +const NAME_MESSAGE: &str = "must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens."; + +/// `MessageStreamsContract::manifest`. +pub(super) fn manifest() -> Value { + json!({ + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": MINIMUM_PROTOCOL_VERSION, + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": INTERNAL_SIGNAL, + "message_schema": MESSAGE_SCHEMA, + "cursor_schema": CURSOR_SCHEMA, + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics", + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict", + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run", + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true, + }, + "batch": { + "minimum": 1, + "maximum": MAX_BATCH, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages", + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention", + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await", + }, + "conformance": { + "runtimes": ["php", "python", "rust"], + "targets": ["server", "cloud"], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order", + ], + "artifacts": "published", + }, + }) +} + +/// The worker capability: the manifest, supported. +pub(super) fn worker_capability() -> Value { + let mut capability = manifest(); + capability["supported"] = json!(true); + capability +} + +fn identifier(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) +} + +/// `/^[A-Za-z0-9._:-]{1,128}$/`. +pub(super) fn valid_name(name: &str) -> bool { + name.len() <= 128 && identifier(name) +} + +fn invalid(field: &str, message: String) -> RuntimeError { + laravel_rules::failure(vec![(field.to_owned(), vec![message])]).expect("one message") +} + +/// `validateStreamName`: a path stream name outside the pattern. +pub(super) fn check_path_name(name: &str) -> Result<()> { + if valid_name(name) { + Ok(()) + } else { + Err(invalid("stream_name", format!("The stream name {NAME_MESSAGE}"))) + } +} + +/// The append body's rules, after Laravel's normalization. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("message_id", &[Rule::Required, Rule::String, Rule::Max(191), Rule::Pattern(identifier)]), + ("input", &[Rule::Nullable, Rule::Array]), + ("request_id", &[Rule::Nullable, Rule::String, Rule::Max(255)]), +]; + +pub(super) fn check_append(body: &Value) -> Result<()> { + laravel_rules::validate(body, APPEND_RULES) +} + +/// `diagnosticMessageId`: a refused message id as diagnostics keep it. +pub(super) fn diagnostic_message_id(value: &Value) -> String { + match value.as_str() { + None => "[omitted]".into(), + Some(id) if id.len() <= 191 && identifier(id) => id.into(), + Some(id) => { + use sha2::Digest; + format!("sha256:{}", hex(sha2::Sha256::digest(id.as_bytes()))) + } + } +} + +/// The payload hash PHP stores: sha256 of `codec \0 blob`. +pub(super) fn payload_hash(codec: &str, blob: &str) -> String { + use sha2::Digest; + let mut digest = sha2::Sha256::new(); + digest.update(codec.as_bytes()); + digest.update([0]); + digest.update(blob.as_bytes()); + hex(digest.finalize()) +} + +fn hex(digest: impl AsRef<[u8]>) -> String { + digest.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The worker protocol's message-stream fields on a completion, checked as +/// PHP's first `validate` call does, before the completion's own rules. +const COMPLETION_RULES: &laravel_rules::Rules = &[ + ("message_stream_cursors", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_cursors.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_cursors.*.through_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), + ("message_stream_waits", &[Rule::Nullable, Rule::Array, Rule::Max(100)]), + ("message_stream_waits.*.stream_name", &[Rule::Required, Rule::String, Rule::Max(128)]), + ("message_stream_waits.*.after_position", &[Rule::Required, Rule::Integer, Rule::Min(0)]), +]; + +pub(super) fn check_completion_fields(body: &Value) -> Result<()> { + laravel_rules::validate(&laravel_rules::normalize(body), COMPLETION_RULES) +} + +/// A completion's cursor advances and pending waits, in request order. +pub(super) struct Completion { + pub(super) cursors: Vec, + pub(super) waits: Vec, +} + +impl Completion { + pub(super) fn of(body: &Value) -> Self { + let list = |field: &str| match &body[field] { + Value::Array(items) => items.clone(), + Value::Object(items) => items.values().cloned().collect(), + _ => Vec::new(), + }; + Self { cursors: list("message_stream_cursors"), waits: list("message_stream_waits") } + } + + pub(super) fn is_empty(&self) -> bool { + self.cursors.is_empty() && self.waits.is_empty() + } +} + +/// `metadataStreamName`. +fn metadata_name(item: &Value, field: &str) -> Result { + match item["stream_name"].as_str() { + Some(name) if valid_name(name) => Ok(name.to_owned()), + _ => Err(invalid(field, format!("The stream_name {NAME_MESSAGE}"))), + } +} + +fn non_negative(value: &Value) -> Option { + value.as_i64().filter(|value| *value >= 0) +} + +/// The rejection PHP's service answers for an append it cannot accept. +fn rejection(status: StatusCode, reason: &str, workflow_id: &str, stream: &str, message_id: &str, extra: Value) -> RuntimeError { + let mut response = json!({"accepted":false,"duplicate":false,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":null,"outcome":"rejected","reason":reason}); + if let Value::Object(extra) = extra { + response.as_object_mut().expect("object").extend(extra); + } + RuntimeError::Protocol { status, response } +} + +/// The value a stream delivery signal carries: one map in a list. +fn map(entries: Vec<(&str, crate::codec::Value)>) -> crate::codec::Value { + crate::codec::Value::Map(entries.into_iter().map(|(key, value)| (key.to_owned(), value)).collect::>()) +} + +/// A runtime signal's encoded arguments and command payload. +async fn runtime_arguments(codec: Arc, delivery: crate::codec::Value) -> Result { + let values = vec![delivery]; + let arguments = super::queries::encode_arguments(codec.clone(), crate::codec::Value::Array(values.clone())).await?; + let command = super::queries::encode_arguments( + codec.clone(), + map(vec![ + ("name", crate::codec::Value::String(INTERNAL_SIGNAL.into())), + ("arguments", crate::codec::Value::Array(values.clone())), + ("validation_errors", crate::codec::Value::Array(vec![])), + ]), + ) + .await?; + let value = super::queries::encode_arguments(codec, values[0].clone()).await?; + Ok(PreparedControlArguments { arguments, values, value, command }) +} + +/// `CommandContext::controlPlane`: the context of a delivery with no request. +fn control_plane_context() -> Value { + json!({"caller":{"type":"control_plane","label":"Control Plane"},"auth":{"status":"not_applicable","method":"none"}}) +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// `NamespaceWorkflowScope::workflowBound`. + pub(crate) async fn message_stream_workflow_bound(&self, workflow_id: &str) -> Result { + Ok(Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// The stream row, created at position zero when absent. + async fn locked_stream(&self, tx: &mut Transaction<'_, DB>, workflow_id: &str, stream: &str) -> Result { + const SELECT: &str = "SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3"; + if let Some(row) = Self::query(SELECT) + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(stream) + .fetch_optional(&mut **tx) + .await? + { + return Ok(row); + } + let at = now(); + Self::query("INSERT INTO workflow_inbound_streams(namespace,workflow_instance_id,stream_name,last_position,cursor_position,duplicate_count,malformed_count,created_at,updated_at) VALUES ($1,$2,$3,0,0,0,0,$4,$5)") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + Ok(Self::query(SELECT).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_one(&mut **tx).await?) + } + + /// The instance's current run, if it is still open. + async fn active_current_run(tx: &mut Transaction<'_, DB>, workflow_id: &str) -> Result> { + let Some(run) = Self::query("SELECT r.* FROM workflow_instances i JOIN workflow_runs r ON r.id=i.current_run_id WHERE i.id=$1") + .bind(workflow_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let open = matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting"); + Ok(open.then_some(run)) + } + + /// `MessageStreamService::append`, in one transaction. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn append_message( + &self, + workflow_id: &str, + stream: &str, + message_id: &str, + payload_codec: &str, + payload_blob: &str, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<(StatusCode, Value)> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() { + return Err(rejection(StatusCode::NOT_FOUND, "instance_not_found", workflow_id, stream, message_id, json!({}))); + } + let Some(run) = Self::active_current_run(&mut tx, workflow_id).await? else { + return Err(rejection(StatusCode::CONFLICT, "run_not_active", workflow_id, stream, message_id, json!({}))); + }; + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let stream_id = DB::number(&row, "id")?; + let cursor = DB::number(&row, "cursor_position")?; + let hash = payload_hash(payload_codec, payload_blob); + let at = now(); + if let Some(existing) = Self::query("SELECT position,payload_hash FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).fetch_optional(&mut *tx).await? + { + let same = DB::string(&existing, "payload_hash")? == hash; + let outcome = if same { "duplicate" } else { "message_identity_conflict" }; + Self::query("UPDATE workflow_inbound_streams SET duplicate_count=duplicate_count+1,last_input_outcome=$1,last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(outcome).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + tx.commit().await?; + let response = json!({"accepted":same,"duplicate":same,"workflow_id":workflow_id,"stream_name":stream, + "message_id":message_id,"position":DB::number(&existing,"position")?,"cursor_position":cursor, + "outcome":if same { "duplicate" } else { "rejected" },"reason":if same { Value::Null } else { json!("message_identity_conflict") }}); + return Ok((if same { StatusCode::OK } else { StatusCode::CONFLICT }, response)); + } + let position = DB::number(&row, "last_position")? + 1; + Self::query("INSERT INTO workflow_inbound_stream_items(stream_id,namespace,workflow_instance_id,stream_name,message_id,position,payload_codec,payload_blob,payload_hash,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)") + .bind(stream_id).bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).bind(message_id).bind(position) + .bind(payload_codec).bind(payload_blob).bind(&hash).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,last_input_outcome='accepted',last_input_message_id=$2,last_input_at=$3,updated_at=$4 WHERE id=$5") + .bind(position).bind(message_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id) + .execute(&mut *tx).await?; + let delivery = Self::message_delivery(stream, message_id, position, payload_codec, payload_blob); + let run_id = DB::string(&run, "id")?; + match Self::deliver_runtime_signal(&mut tx, workflow_id, delivery, context, codec, limits).await? { + Ok(command_id) => { + Self::mark_delivered(&mut tx, stream_id, message_id, &run_id, &command_id).await?; + } + Err(reason) => { + Self::query("DELETE FROM workflow_inbound_stream_items WHERE stream_id=$1 AND message_id=$2") + .bind(stream_id).bind(message_id).execute(&mut *tx).await?; + Self::query("UPDATE workflow_inbound_streams SET last_position=$1,last_input_outcome='message_stream_not_supported' WHERE id=$2") + .bind(position - 1).bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + self.wake.notify_waiters(); + return Err(rejection(StatusCode::CONFLICT, "message_stream_not_supported", workflow_id, stream, message_id, + json!({"detail":reason,"remediation":"Use a PHP, Python, or Rust worker that advertises the message_streams capability."}))); + } + } + tx.summary_changed(&run_id); + tx.commit().await?; + self.wake.notify_waiters(); + Ok((StatusCode::ACCEPTED, json!({"accepted":true,"duplicate":false,"workflow_id":workflow_id,"run_id":run_id, + "stream_name":stream,"message_id":message_id,"position":position,"cursor_position":cursor, + "outcome":"accepted","reason":null}))) + } + + /// `MessageStreamService::recordMalformed`: counts a refused append + /// against the stream of an open run, keeping no payload. + pub(crate) async fn record_malformed_message(&self, workflow_id: &str, stream: &str, message_id: &Value) -> Result<()> { + let mut tx = self.begin().await?; + let bound = Self::query("SELECT id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await?; + if bound.is_none() || Self::active_current_run(&mut tx, workflow_id).await?.is_none() { + return Ok(()); + } + let row = self.locked_stream(&mut tx, workflow_id, stream).await?; + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET malformed_count=malformed_count+1,last_input_outcome='malformed',last_input_message_id=$1,last_input_at=$2,updated_at=$3 WHERE id=$4") + .bind(diagnostic_message_id(message_id)).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut *tx).await?; + tx.commit().await?; + Ok(()) + } + + /// `MessageStreamService::diagnostics`: every stream of the instance, or + /// the named one, by name, without payloads. + pub(crate) async fn message_stream_diagnostics(&self, workflow_id: &str, stream: Option<&str>) -> Result> { + let rows = match stream { + Some(stream) => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).bind(stream).fetch_all(&self.pool).await?, + None => Self::query("SELECT * FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(workflow_id).fetch_all(&self.pool).await?, + }; + let time = |row: &DB::Row, field: &str| -> Result { + Ok(super::task_queues::json_time(DB::optional_instant(row, field)?)) + }; + rows.iter() + .map(|row| { + let last = DB::number(row, "last_position")?; + let cursor = DB::number(row, "cursor_position")?; + let wait = match DB::optional_string(row, "waiting_run_id")? { + Some(run_id) => json!({"run_id":run_id,"after_position":DB::optional_number(row,"waiting_after_position")?, + "since":time(row,"waiting_since")?}), + None => Value::Null, + }; + Ok(json!({"stream_name":DB::string(row,"stream_name")?,"last_position":last,"cursor_position":cursor, + "pending_count":(last - cursor).max(0),"pending_wait":wait, + "duplicate_count":DB::number(row,"duplicate_count")?,"malformed_count":DB::number(row,"malformed_count")?, + "last_input_outcome":DB::optional_string(row,"last_input_outcome")?, + "last_input_message_id":DB::optional_string(row,"last_input_message_id")?, + "last_input_at":time(row,"last_input_at")?})) + }) + .collect() + } + + /// `MessageStreamService::validateCompletion`: the worker's cursor + /// advances and waits, checked before the completion changes anything. + pub(super) async fn validate_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + completion: &Completion, + ) -> Result<()> { + let mut proposed = BTreeMap::new(); + for (index, cursor) in completion.cursors.iter().enumerate() { + let field = |name: &str| format!("message_stream_cursors.{index}.{name}"); + let stream = metadata_name(cursor, &field("stream_name"))?; + if proposed.contains_key(&stream) { + return Err(invalid(&field("stream_name"), "A stream may be acknowledged only once per workflow task completion.".into())); + } + let Some(through) = non_negative(&cursor["through_position"]) else { + return Err(invalid(&field("through_position"), "The through_position must be a non-negative integer.".into())); + }; + let row = Self::query("SELECT id,last_position,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await?; + let Some(row) = row.filter(|row| DB::number(row, "last_position").is_ok_and(|last| through <= last)) else { + return Err(invalid(&field("through_position"), "The cursor cannot advance beyond the last accepted stream position.".into())); + }; + let current = DB::number(&row, "cursor_position")?; + if through > current { + let delivered = Self::scalar("SELECT COUNT(*) FROM workflow_inbound_stream_items WHERE stream_id=$1 AND position>$2 AND position<=$3 AND delivered_run_id=$4") + .bind(DB::number(&row, "id")?).bind(current).bind(through).bind(run_id).fetch_one(&mut **tx).await?; + if delivered != through - current { + return Err(invalid(&field("through_position"), "The cursor must advance contiguously through messages delivered to this workflow run.".into())); + } + } + proposed.insert(stream, current.max(through)); + } + let mut waiting = std::collections::BTreeSet::new(); + for (index, wait) in completion.waits.iter().enumerate() { + let field = |name: &str| format!("message_stream_waits.{index}.{name}"); + let stream = metadata_name(wait, &field("stream_name"))?; + if !waiting.insert(stream.clone()) { + return Err(invalid(&field("stream_name"), "A stream may have only one pending wait per workflow task completion.".into())); + } + let Some(after) = non_negative(&wait["after_position"]) else { + return Err(invalid(&field("after_position"), "The after_position must be a non-negative integer.".into())); + }; + let expected = match proposed.get(&stream) { + Some(cursor) => *cursor, + None => Self::query("SELECT cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND stream_name=$3") + .bind(self.namespace.as_ref()).bind(instance_id).bind(&stream).fetch_optional(&mut **tx).await? + .map(|row| DB::number(&row, "cursor_position")).transpose()?.unwrap_or(0), + }; + if after != expected { + return Err(invalid(&field("after_position"), "The pending wait must start after the durable stream cursor.".into())); + } + } + Ok(()) + } + + /// `MessageStreamService::recordCompletion`, after the completion has + /// been applied in the same transaction: consumed messages, each moved + /// cursor with `MessageCursorAdvanced`, pending waits, then the hand-over + /// to a run that continued this one. + pub(super) async fn record_stream_completion( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + run_id: &str, + task_id: &str, + completion: &Completion, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + for cursor in &completion.cursors { + let stream = cursor["stream_name"].as_str().unwrap_or_default(); + let through = cursor["through_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + let previous = DB::number(&row, "cursor_position")?; + if through <= previous { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET consumed_run_id=$1,consumed_task_id=$2,consumed_at=$3,updated_at=$4 WHERE stream_id=$5 AND position>$6 AND position<=$7") + .bind(run_id).bind(task_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .bind(previous).bind(through).execute(&mut **tx).await?; + Self::query("UPDATE workflow_inbound_streams SET cursor_position=$1,waiting_run_id=NULL,waiting_after_position=NULL,waiting_since=NULL,updated_at=$2 WHERE id=$3") + .bind(through).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?).execute(&mut **tx).await?; + let task = Self::query("SELECT * FROM workflow_tasks WHERE id=$1").bind(task_id).fetch_one(&mut **tx).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":stream,"previous_position":previous, + "new_position":through,"task":Self::task_snapshot(&task)?}), Some(task_id), None).await?; + } + for wait in &completion.waits { + let stream = wait["stream_name"].as_str().unwrap_or_default(); + let after = wait["after_position"].as_i64().unwrap_or(0); + let row = self.locked_stream(tx, instance_id, stream).await?; + if DB::number(&row, "last_position")? > after { + continue; + } + let at = now(); + Self::query("UPDATE workflow_inbound_streams SET waiting_run_id=$1,waiting_after_position=$2,waiting_since=$3,updated_at=$4 WHERE id=$5") + .bind(run_id).bind(after).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::number(&row, "id")?) + .execute(&mut **tx).await?; + } + let current = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1") + .bind(instance_id).fetch_one(&mut **tx).await?; + let current = DB::optional_string(¤t, "current_run_id")?; + if let Some(current) = current.filter(|current| current != run_id) { + self.carry_streams_forward(tx, instance_id, ¤t, codec, limits).await?; + } + Ok(()) + } + + /// `carryForward`: a cursor checkpoint for each acknowledged stream the + /// current run has not received, then every unconsumed message the + /// current run has not received, by append order. + async fn carry_streams_forward( + &self, + tx: &mut Transaction<'_, DB>, + instance_id: &str, + current: &str, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result<()> { + let context = control_plane_context(); + let streams = Self::query("SELECT id,stream_name,cursor_position FROM workflow_inbound_streams WHERE namespace=$1 AND workflow_instance_id=$2 AND cursor_position>0 AND (cursor_checkpoint_run_id IS NULL OR cursor_checkpoint_run_id<>$3) ORDER BY stream_name") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + for stream in streams { + let checkpoint = map(vec![ + ("schema", crate::codec::Value::String(CURSOR_SCHEMA.into())), + ("stream_name", crate::codec::Value::String(DB::string(&stream, "stream_name")?)), + ("through_position", crate::codec::Value::Long(DB::number(&stream, "cursor_position")?)), + ]); + if Self::deliver_runtime_signal(tx, instance_id, checkpoint, &context, codec.clone(), limits).await?.is_err() { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_cursor_carry_failed" }); + } + Self::query("UPDATE workflow_inbound_streams SET cursor_checkpoint_run_id=$1 WHERE id=$2") + .bind(current).bind(DB::number(&stream, "id")?).execute(&mut **tx).await?; + } + loop { + let items = Self::query("SELECT stream_id,stream_name,message_id,position,payload_codec,payload_blob FROM workflow_inbound_stream_items WHERE namespace=$1 AND workflow_instance_id=$2 AND consumed_at IS NULL AND (delivered_run_id IS NULL OR delivered_run_id<>$3) ORDER BY id LIMIT 100") + .bind(self.namespace.as_ref()).bind(instance_id).bind(current).fetch_all(&mut **tx).await?; + if items.is_empty() { + return Ok(()); + } + for item in items { + let (stream, message_id) = (DB::string(&item, "stream_name")?, DB::string(&item, "message_id")?); + let delivery = Self::message_delivery(&stream, &message_id, DB::number(&item, "position")?, + &DB::string(&item, "payload_codec")?, &DB::string(&item, "payload_blob")?); + match Self::deliver_runtime_signal(tx, instance_id, delivery, &context, codec.clone(), limits).await? { + Ok(command_id) => { + Self::mark_delivered(tx, DB::number(&item, "stream_id")?, &message_id, current, &command_id).await?; + } + Err(_) => { + return Err(RuntimeError::Refused { status: StatusCode::INTERNAL_SERVER_ERROR, reason: "message_stream_carry_failed" }); + } + } + } + } + } + + /// The message a stream delivers: its schema, identity, position and the + /// inline payload envelope. + fn message_delivery(stream: &str, message_id: &str, position: i64, payload_codec: &str, payload_blob: &str) -> crate::codec::Value { + use crate::codec::Value as Payload; + map(vec![ + ("schema", Payload::String(MESSAGE_SCHEMA.into())), + ("stream_name", Payload::String(stream.into())), + ("message_id", Payload::String(message_id.into())), + ("position", Payload::Long(position)), + ("payload_envelope", map(vec![ + ("codec", Payload::String(payload_codec.into())), + ("blob", Payload::String(payload_blob.into())), + ])), + ]) + } + + async fn mark_delivered(tx: &mut Transaction<'_, DB>, stream_id: i64, message_id: &str, run_id: &str, command_id: &str) -> Result<()> { + let at = now(); + Self::query("UPDATE workflow_inbound_stream_items SET delivered_run_id=$1,workflow_command_id=$2,delivered_at=$3,updated_at=$4 WHERE stream_id=$5 AND message_id=$6") + .bind(run_id).bind(command_id).bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(stream_id).bind(message_id) + .execute(&mut **tx).await?; + Ok(()) + } + + /// `WorkflowStub::attemptRuntimeSignalWithArguments` for the instance's + /// current run: no declared-signal admission or argument contract, but + /// the pending-signal and payload ceilings still apply. The accepted + /// command's id, or the refusal's reason. + async fn deliver_runtime_signal( + tx: &mut Transaction<'_, DB>, + workflow_id: &str, + delivery: crate::codec::Value, + context: &Value, + codec: Arc, + limits: super::memo_limits::StructuralLimits, + ) -> Result> { + let instance = Self::query("SELECT current_run_id,last_message_sequence FROM workflow_instances WHERE id=$1") + .bind(workflow_id).fetch_one(&mut **tx).await?; + let run_id = DB::string(&instance, "current_run_id")?; + let run = Self::query("SELECT * FROM workflow_runs WHERE id=$1").bind(&run_id).fetch_one(&mut **tx).await?; + if !matches!(DB::string(&run, "status")?.as_str(), "pending" | "running" | "waiting") { + return Ok(Err("run_not_active")); + } + let prepared = runtime_arguments(codec.clone(), delivery).await?; + let command_id = id(); + let signal_id = id(); + let command_sequence = DB::number(&run, "last_command_sequence")? + 1; + let message_sequence = DB::number(&instance, "last_message_sequence")? + 1; + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: &run_id, workflow_type: &workflow_type, name: INTERNAL_SIGNAL, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context }; + let pending = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND status='received'") + .bind(&run_id).fetch_one(&mut **tx).await?; + let exceeded = super::memo_limits::Exceeded::pending_signals(limits.pending_signals, pending) + .or_else(|| super::memo_limits::Exceeded::payload(limits.payload_bytes, &prepared.arguments)); + if let Some(exceeded) = exceeded { + let oversized = exceeded.kind == "payload_size"; + let (values, errors, diagnostics) = if oversized { + (vec![], json!({"arguments":[exceeded.message]}), exceeded.fields()) + } else { + (prepared.values.clone(), json!([]), json!({})) + }; + let command = super::store::prepare_rejected_command(codec.clone(), INTERNAL_SIGNAL.into(), values, + errors.clone(), diagnostics, "signal_codec_unavailable").await?; + let arguments = if oversized { super::queries::empty_arguments(codec).await? } else { prepared.arguments.clone() }; + let row_errors = if oversized { errors } else { json!([]) }; + Self::record_rejected_signal(tx, &write, None, "structural_limit_exceeded", &command, &arguments, &row_errors).await?; + return Ok(Err("structural_limit_exceeded")); + } + Self::record_accepted_signal(tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.summary_changed(&run_id); + Ok(Ok(command_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_follow_phps_pattern() { + assert!(valid_name("orders")); + assert!(valid_name("a.b_c:d-e")); + assert!(valid_name(&"x".repeat(128))); + assert!(!valid_name(&"x".repeat(129))); + assert!(!valid_name("")); + assert!(!valid_name("bad name")); + assert!(!valid_name("bad/name")); + } + + #[test] + fn refused_message_ids_are_bounded_for_diagnostics() { + assert_eq!(diagnostic_message_id(&Value::Null), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!(7)), "[omitted]"); + assert_eq!(diagnostic_message_id(&json!("m3")), "m3"); + let long = "a".repeat(192); + assert_eq!(diagnostic_message_id(&json!(long)).len(), "sha256:".len() + 64); + assert!(diagnostic_message_id(&json!("bad id!")).starts_with("sha256:")); + } + + #[test] + fn payload_hash_covers_codec_and_blob() { + assert_ne!(payload_hash("avro", "a"), payload_hash("avro", "b")); + assert_ne!(payload_hash("avro", "ab"), payload_hash("avroa", "b")); + assert_eq!(payload_hash("avro", "x").len(), 64); + } + + #[test] + fn the_worker_capability_is_the_supported_manifest() { + let capability = worker_capability(); + assert_eq!(capability["supported"], true); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); + assert_eq!(capability["transport"]["internal_signal"], INTERNAL_SIGNAL); + assert_eq!(manifest()["batch"]["maximum"], 100); + } + + #[test] + fn completion_fields_follow_laravels_rules() { + assert!(check_completion_fields(&json!({})).is_ok()); + assert!(check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":0}]})).is_ok()); + let Err(RuntimeError::Protocol { response, .. }) = + check_completion_fields(&json!({"message_stream_cursors":[{"stream_name":"s","through_position":-1}]})) + else { + panic!("expected a validation failure"); + }; + assert_eq!(response["message"], "The message_stream_cursors.0.through_position field must be at least 0."); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index a38f42362..677a65aeb 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -118,6 +118,7 @@ mod service_catalog; mod sqlite; mod start_definition; mod start_request; +mod message_streams; mod storage_test; mod store; mod system_metrics; diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index 9e888fde5..b1a8e1047 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -78,9 +78,12 @@ pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the -/// skip reason. An abandoned activity's hold is read from history. +/// skip reason. An abandoned activity's hold is read from history. A run +/// whose instance has inbound message streams is held too: native does not +/// compact or remove stream state as PHP's MessageStreamRetentionCleanup +/// does yet, so it keeps the run rather than orphan the streams. pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)", 2, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; diff --git a/rust/src/runtime/run_describe.rs b/rust/src/runtime/run_describe.rs index 070e4e364..c18bf74d1 100644 --- a/rust/src/runtime/run_describe.rs +++ b/rust/src/runtime/run_describe.rs @@ -31,8 +31,12 @@ fn max_payload_bytes() -> i64 { } /// `oversizedPayloadPreview`: an external payload larger than the limit. -fn oversized(envelope: &Value) -> bool { - envelope["external_storage"]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes()) +/// PHP's stored envelope names its reference `external_storage`; native's +/// names it `external_payload`. +pub(super) fn oversized(envelope: &Value) -> bool { + ["external_storage", "external_payload"] + .iter() + .any(|key| envelope[key]["size_bytes"].as_i64().is_some_and(|size| size > max_payload_bytes())) } /// Carbon's `toIso8601String()`, which PHP's run description uses. @@ -315,6 +319,8 @@ mod tests { assert!(oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024 + 1}}))); assert!(!oversized(&json!({"external_storage":{"size_bytes":2 * 1024 * 1024}}))); assert!(!oversized(&json!({"codec":"avro","blob":"AA=="}))); + assert!(oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024 + 1}}))); + assert!(!oversized(&json!({"codec":"avro","external_payload":{"size_bytes":2 * 1024 * 1024}}))); assert_eq!(without_empty(json!({"a":null,"b":[],"c":{},"d":0})), json!({"c":{},"d":0})); let time = DateTime::parse_from_rfc3339("2026-10-21T07:38:51.123Z").unwrap().with_timezone(&Utc); assert_eq!(iso8601(Some(time)), json!("2026-10-21T07:38:51+00:00")); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index edeba7358..9c36fe912 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -15,8 +15,70 @@ pub(super) struct PreparedControlArguments { pub(super) value: String, pub(super) command: String, } +/// The events PHP's Workflow package records with the task model in hand, +/// which therefore carry the task's snapshot. +const TASK_SNAPSHOT_EVENTS: &[&str] = &[ + "ActivityCompleted", + "ActivityFailed", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ChildRunStarted", + "ChildWorkflowScheduled", + "ConditionWaitOpened", + "ConditionWaitSatisfied", + "ConditionWaitTimedOut", + "FailureHandled", + "MemoUpserted", + "MessageCursorAdvanced", + "SearchAttributesUpserted", + "SelectionOperationCancelled", + "SideEffectRecorded", + "SignalApplied", + "SignalWaitOpened", + "TimerFired", + "TimerScheduled", + "VersionMarkerRecorded", + "WorkflowCompleted", + "WorkflowContinuedAsNew", + "WorkflowFailed", + "WorkflowTimedOut", +]; +/// Describe's command journal. Each backend passes its exact, case-sensitive +/// test for native's stored-reference prefix +/// (`external_resolution::STORED_PREFIX`) over the column that holds a +/// command's input: a signal's in its signal record (`s.arguments`), a start's +/// in the command row (`c.payload`). Other commands keep no stored reference. +/// Only a stored reference is read; inline payloads are not. +macro_rules! run_commands_sql { + ($signal:literal, $start:literal) => { + concat!( + "SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name,CASE WHEN c.command_type='signal' AND ", + $signal, + " THEN s.arguments WHEN c.command_type='start' AND ", + $start, + " THEN c.payload END AS stored_payload FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id" + ) + }; +} +pub(super) use run_commands_sql; +/// One signal command's identity and sequences in its run, as both the +/// public signal route and the runtime's message-stream deliveries write it. +pub(super) struct SignalWrite<'a> { + pub(super) workflow_id: &'a str, + pub(super) run_id: &'a str, + pub(super) workflow_type: &'a str, + pub(super) name: &'a str, + pub(super) command_id: &'a str, + pub(super) signal_id: &'a str, + pub(super) command_sequence: i64, + pub(super) message_sequence: i64, + pub(super) context: &'a Value, +} /// The runtime-owned message-stream delivery signal (`WorkflowStub::MESSAGE_STREAM_RUNTIME_SIGNAL`). -const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; +const MESSAGE_STREAM_SIGNAL: &str = super::message_streams::INTERNAL_SIGNAL; pub(super) const TASK_CANDIDATES_SQL: &str = "SELECT t.*,r.workflow_type FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.namespace=$1 AND t.queue=$2 AND t.task_type=$3 AND r.status IN ('pending','running','waiting') AND (t.status='ready' OR (t.status='leased' AND t.task_type='workflow' AND t.lease_expires_at<=$4)) AND (t.available_at IS NULL OR t.available_at<=$5) AND (COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,'')) IS NULL OR COALESCE(NULLIF(t.compatibility,''),NULLIF(r.compatibility,''))=$6) ORDER BY t.priority,t.available_at,t.id LIMIT 100"; /// `nextEligibleSignal`: a run's received signals in PHP's order: command /// sequence (unsequenced last), receipt time, then id. @@ -577,18 +639,30 @@ where // refused signals which intentionally have no workflow history event. // Select metadata only: large customer payloads are never decoded or // copied merely to display the command's target and admission result. - let command_rows = Self::query("SELECT c.id,c.command_sequence,c.command_type,c.target_scope,c.requested_workflow_run_id,c.resolved_workflow_run_id,c.source,c.status,c.outcome,c.rejection_reason,c.workflow_type,c.workflow_class,c.accepted_at,c.applied_at,c.rejected_at,c.context,s.signal_name FROM workflow_commands c LEFT JOIN workflow_signal_records s ON s.workflow_command_id=c.id WHERE c.workflow_run_id=$1 AND c.workflow_instance_id=$2 ORDER BY c.command_sequence,c.created_at,c.id") + // Only a stored external reference is read, to tell whether its preview + // is omitted; inline payloads stay unread. + let command_rows = Self::query(DB::RUN_COMMANDS_SQL) .bind(DB::string(&row, "id")?).bind(workflow_id).fetch_all(&self.pool).await?; let mut commands = Vec::with_capacity(command_rows.len()); for command in command_rows { + // `oversizedPayloadPreview` over the stored reference's envelope; + // an omitted preview also omits the target name, as PHP does. + let omitted = DB::optional_string(&command, "stored_payload")? + .as_deref() + .and_then(super::external_resolution::stored_envelope) + .is_some_and(|envelope| super::run_describe::oversized(&envelope)); + let target_name = DB::optional_string(&command, "signal_name")?.filter(|_| !omitted); let mut public = json!({"id":DB::string(&command,"id")?,"sequence":DB::optional_number(&command,"command_sequence")?, "type":DB::string(&command,"command_type")?,"target_scope":DB::string(&command,"target_scope")?, "requested_run_id":DB::optional_string(&command,"requested_workflow_run_id")?, "resolved_run_id":DB::optional_string(&command,"resolved_workflow_run_id")?, - "target_name":DB::optional_string(&command,"signal_name")?,"source":DB::string(&command,"source")?, + "target_name":target_name,"source":DB::string(&command,"source")?, "status":DB::string(&command,"status")?,"outcome":DB::optional_string(&command,"outcome")?, "reason":DB::optional_string(&command,"rejection_reason")?,"rejection_reason":DB::optional_string(&command,"rejection_reason")?, - "workflow_type":DB::optional_string(&command,"workflow_type")?,"workflow_class":DB::optional_string(&command,"workflow_class")?, + // `WorkflowCommand::record` keeps the run's class on each command. + "workflow_type":DB::optional_string(&command,"workflow_type")?, + "workflow_class":DB::optional_string(&command,"workflow_class")?.or(DB::optional_string(&row,"workflow_class")?), + "payload_preview_omitted":omitted, "accepted_at":DB::optional_instant(&command,"accepted_at")?,"applied_at":DB::optional_instant(&command,"applied_at")?, "rejected_at":DB::optional_instant(&command,"rejected_at")?}); let fields = public.as_object_mut().expect("command object"); @@ -834,25 +908,9 @@ where "signal_codec_unavailable", ) .await?; - let rejected_at = now(); - Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(outcome).bind(&workflow_type) - .bind(&prepared.command).bind(reason).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(rejected_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; - Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name) - .bind(outcome).bind(command_sequence).bind(&row_arguments).bind(DB::document(&row_errors)).bind(reason) - .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut *tx).await?; - Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) - .await?; - Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) - .await?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_rejected_signal(&mut tx, &write, outcome, reason, &prepared.command, &row_arguments, &row_errors).await?; tx.commit().await?; let mut response = json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run, "requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence, @@ -867,40 +925,88 @@ where // signal wait reservation or notification is added here. return Err(super::RuntimeError::Protocol { status, response }); } - let wait = Self::open_wait(&mut tx, ¤t_run).await?; - let mut wait_id = format!("signal-command:{command_id}"); - if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == name) { + let workflow_type = DB::string(&run, "workflow_type")?; + let write = SignalWrite { workflow_id, run_id: ¤t_run, workflow_type: &workflow_type, name, + command_id: &command_id, signal_id: &signal_id, command_sequence, message_sequence, context: &context }; + Self::record_accepted_signal(&mut tx, &write, &prepared.command, &prepared.arguments, &DB::string(&run, "queue")?).await?; + tx.commit().await?; + self.wake.notify_waiters(); + Ok( + json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), + ) + } + + /// A refused signal's durable admission record: its rejected command, + /// its rejected signal row and the advanced sequences. No history, task + /// or wait reservation is added. + pub(super) async fn record_rejected_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + outcome: Option<&str>, + reason: &str, + command: &str, + arguments: &str, + errors: &Value, + ) -> Result<()> { + let rejected_at = now(); + Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,rejection_reason,command_sequence,message_sequence,rejected_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','rejected',$5,$6,'avro',$7,$8,$9,$10,$11,$12)") + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(outcome).bind(write.workflow_type) + .bind(command).bind(reason).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(rejected_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; + Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,status,outcome,command_sequence,payload_codec,arguments,validation_errors,rejection_reason,rejected_at,closed_at) VALUES ($1,$2,$3,$4,$5,$6,'rejected',$7,$8,'avro',$9,$10,$11,$12,$13)") + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name) + .bind(outcome).bind(write.command_sequence).bind(arguments).bind(DB::document(errors)).bind(reason) + .bind(DB::bind_time(rejected_at)).bind(DB::bind_time(rejected_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await + } + + /// `recordAcceptedSignalCommand`, `recordAcceptedSignal` and + /// `recordSignalReceivedHistory`: the accepted command, the received + /// signal reserving an open wait for its name, the advanced sequences, + /// `SignalReceived`, then the run's next message task if none is open. + pub(super) async fn record_accepted_signal( + tx: &mut Transaction<'_, DB>, + write: &SignalWrite<'_>, + command: &str, + arguments: &str, + queue: &str, + ) -> Result<()> { + let wait = Self::open_wait(tx, write.run_id).await?; + let mut wait_id = format!("signal-command:{}", write.command_id); + if let Some(wait) = wait.as_ref().filter(|wait| wait["signal_name"] == write.name) { let reserved = Self::scalar("SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_wait_id=$2 AND status='received'") - .bind(¤t_run).bind(text(wait, "signal_wait_id")?).fetch_one(&mut *tx).await?; + .bind(write.run_id).bind(text(wait, "signal_wait_id")?).fetch_one(&mut **tx).await?; if reserved == 0 { wait_id = text(wait, "signal_wait_id")?.to_owned(); } } let accepted_at = now(); Self::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,command_type,source,status,outcome,workflow_type,payload_codec,payload,command_sequence,message_sequence,accepted_at,context) VALUES ($1,$2,$3,$4,'signal','control_plane','accepted','signal_received',$5,'avro',$6,$7,$8,$9,$10)") - .bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(DB::string(&run, "workflow_type")?) - .bind(&prepared.command).bind(command_sequence).bind(message_sequence).bind(DB::bind_time(accepted_at)) - .bind(DB::document(&context)).execute(&mut *tx).await?; + .bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.workflow_type) + .bind(command).bind(write.command_sequence).bind(write.message_sequence).bind(DB::bind_time(accepted_at)) + .bind(DB::document(write.context)).execute(&mut **tx).await?; Self::query("INSERT INTO workflow_signal_records(id,workflow_command_id,workflow_instance_id,workflow_run_id,resolved_workflow_run_id,signal_name,signal_wait_id,status,outcome,command_sequence,payload_codec,arguments,received_at) VALUES ($1,$2,$3,$4,$5,$6,$7,'received','signal_received',$8,'avro',$9,$10)") - .bind(&signal_id).bind(&command_id).bind(workflow_id).bind(¤t_run).bind(¤t_run).bind(name).bind(&wait_id) - .bind(command_sequence).bind(&prepared.arguments).bind(DB::bind_time(accepted_at)).execute(&mut *tx).await?; + .bind(write.signal_id).bind(write.command_id).bind(write.workflow_id).bind(write.run_id).bind(write.run_id).bind(write.name).bind(&wait_id) + .bind(write.command_sequence).bind(arguments).bind(DB::bind_time(accepted_at)).execute(&mut **tx).await?; + Self::advance_signal_sequences(tx, write).await?; + Self::append(tx, write.run_id, "SignalReceived", json!({"workflow_command_id":write.command_id,"signal_id":write.signal_id, + "workflow_instance_id":write.workflow_id,"workflow_run_id":write.run_id,"signal_name":write.name,"signal_wait_id":wait_id, + "arguments":wire(arguments),"payload_codec":"avro"}), None, Some(write.command_id)).await?; + Self::enqueue_next_message(tx, write.run_id, queue).await + } + + async fn advance_signal_sequences(tx: &mut Transaction<'_, DB>, write: &SignalWrite<'_>) -> Result<()> { Self::query("UPDATE workflow_runs SET last_command_sequence=$1 WHERE id=$2") - .bind(command_sequence) - .bind(¤t_run) - .execute(&mut *tx) + .bind(write.command_sequence) + .bind(write.run_id) + .execute(&mut **tx) .await?; Self::query("UPDATE workflow_instances SET last_message_sequence=$1 WHERE id=$2") - .bind(message_sequence) - .bind(workflow_id) - .execute(&mut *tx) + .bind(write.message_sequence) + .bind(write.workflow_id) + .execute(&mut **tx) .await?; - Self::append(&mut tx, ¤t_run, "SignalReceived", json!({"workflow_command_id":command_id,"signal_id":signal_id,"workflow_instance_id":workflow_id,"workflow_run_id":current_run,"signal_name":name,"signal_wait_id":wait_id,"arguments":wire(&prepared.arguments),"payload_codec":"avro"}), None, Some(&command_id)).await?; - Self::enqueue_next_message(&mut tx, ¤t_run, &DB::string(&run, "queue")?).await?; - tx.commit().await?; - self.wake.notify_waiters(); - Ok( - json!({"workflow_id":workflow_id,"run_id":current_run,"resolved_run_id":current_run,"requested_run_id":null,"signal_name":name,"command_id":command_id,"command_sequence":command_sequence,"workflow_type":DB::string(&run, "workflow_type")?,"command_status":"accepted","command_source":"control_plane","target_scope":"instance","outcome":"signal_received","reason":null,"rejection_reason":null,"validation_errors":[]}), - ) + Ok(()) } pub(super) async fn authored_sequence( @@ -1098,6 +1204,17 @@ where run_id: &str, queue: &str, ) -> Result<()> { + Self::next_message_task(tx, run_id, queue).await.map(drop) + } + + /// `PendingMessageTask::createForRun`: the run's next update or signal + /// task, when it is open, has no open workflow task and holds a message + /// that can advance it. The created task's id. + pub(super) async fn next_message_task( + tx: &mut Transaction<'_, DB>, + run_id: &str, + queue: &str, + ) -> Result> { let status = Self::query("SELECT status FROM workflow_runs WHERE id=$1") .bind(run_id) .fetch_one(&mut **tx) @@ -1106,12 +1223,12 @@ where DB::string(&status, "status")?.as_str(), "pending" | "running" | "waiting" ) { - return Ok(()); + return Ok(None); } let open = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(run_id).fetch_one(&mut **tx).await?; if open != 0 { - return Ok(()); + return Ok(None); } // `nextReadyUpdate`: the earliest accepted update, unless an earlier // signal is still unapplied. That signal then goes first. @@ -1129,7 +1246,7 @@ where .await? .is_none() { - Self::create_update_task( + let task_id = Self::create_update_task( tx, run_id, queue, @@ -1137,7 +1254,7 @@ where &DB::string(&update, "workflow_command_id")?, ) .await?; - return Ok(()); + return Ok(Some(task_id)); } let (wait, condition_open, declared) = Self::signal_delivery(tx, run_id).await?; let signals = Self::query(RECEIVED_SIGNALS).bind(run_id).fetch_all(&mut **tx).await?; @@ -1147,10 +1264,10 @@ where if !Self::signal_can_advance(wait.as_ref(), condition_open, declared.as_deref(), &name, &signal_wait_id) { continue; } - Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; - break; + let task_id = Self::create_task(tx, run_id, queue, "workflow", json!({"resume_source_kind":"workflow_signal","workflow_signal_id":DB::string(&signal,"id")?,"signal_name":name,"signal_wait_id":signal_wait_id})).await?; + return Ok(Some(task_id)); } - Ok(()) + Ok(None) } async fn apply_task_signal( @@ -1187,6 +1304,12 @@ where return Err(refuse(StatusCode::CONFLICT, reason)); } let direct = wait.as_ref().is_some_and(|wait| wait.get("signal_name").is_some()); + // `recordAppliedSignalForSignalResume` applies a signal its open + // signal wait matches, or one the durable contract declares, with + // `SignalApplied`. Otherwise `markConditionWaitSignalConsumed`: the + // open condition wait consumes it at the wait's sequence, without + // that event (an undeclared or runtime-reserved signal). + let applies = direct || declared; let condition_timer = match &wait { Some(wait) if !direct => Self::condition_timer(tx, run_id, wait).await?, _ => None, @@ -1236,7 +1359,7 @@ where .bind(run_id) .execute(&mut **tx) .await?; - Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position}), Some(task_id), None).await?; + Self::append(tx, run_id, "MessageCursorAdvanced", json!({"stream_key":format!("instance:{}",DB::string(&run,"workflow_instance_id")?),"previous_position":previous,"new_position":position,"task":Self::task_snapshot(task)?}), Some(task_id), None).await?; if let Some(wait) = wait.as_ref().filter(|_| direct) { Self::cancel_wait_timeout(tx, run_id, wait, task_id, applied_at).await?; } @@ -1244,15 +1367,17 @@ where if direct { applied["sequence"] = json!(sequence); } - Self::append( - tx, - run_id, - "SignalApplied", - applied, - Some(task_id), - Some(&command_id), - ) - .await?; + if applies { + Self::append( + tx, + run_id, + "SignalApplied", + applied, + Some(task_id), + Some(&command_id), + ) + .await?; + } if !direct && let Some(mut satisfied) = wait && !condition_timer.as_ref().is_some_and(|(_, fired)| *fired) @@ -1768,7 +1893,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":leased_at, - "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue}); + "lease_owner":worker_id,"lease_expires_at":expires,"attempt_count":attempt,"queue":queue, + "connection":DB::PHP_CONNECTION}); Self::append_at( &mut tx, &run_id, @@ -2167,7 +2293,9 @@ where external: super::external_resolution::PreparedCommands, worker: Option, protocol: Option<&str>, + limits: super::memo_limits::StructuralLimits, ) -> Result { + let transition_codec_for_streams = transition_codec.clone(); let commands = body["commands"] .as_array() .filter(|c| !c.is_empty() && c.len() <= 100) @@ -2260,17 +2388,9 @@ where { return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_request")); } - text(command, "condition_key")?; + // `normalizeOpenConditionWaitCommand`: the key and the + // definition fingerprint are optional strings, of any form. super::wait_timeouts::timeout_for(command, "invalid_condition_timeout")?; - let fingerprint = text(command, "condition_definition_fingerprint")?; - if !fingerprint.strip_prefix("sha256:").is_some_and(|hash| { - hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) - }) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_condition_fingerprint", - )); - } } "open_signal_wait" => { reject_fields(command, &["type", "signal_name", "timeout_seconds"])?; @@ -2323,6 +2443,17 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + // `MessageStreamService::validateCompletion`, before anything changes. + let streams = super::message_streams::Completion::of(&body); + let stream_instance = Self::query("SELECT workflow_instance_id FROM workflow_runs WHERE id=$1") + .bind(DB::string(&task, "workflow_run_id")?) + .fetch_one(&mut *tx) + .await?; + let stream_instance = DB::string(&stream_instance, "workflow_instance_id")?; + if !streams.is_empty() { + self.validate_stream_completion(&mut tx, &stream_instance, &DB::string(&task, "workflow_run_id")?, &streams) + .await?; + } // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. @@ -2549,8 +2680,7 @@ where Self::query("INSERT INTO workflow_run_timers(id,workflow_run_id,sequence,status,delay_seconds,fire_at) VALUES ($1,$2,$3,'pending',$4,$5)") .bind(&timer_id).bind(&run_id).bind(sequence).bind(delay).bind(DB::bind_time(fire_at)).execute(&mut *tx).await?; let mut timer_payload = json!({"timer_id":timer_id,"sequence":sequence,"delay_seconds":delay,"fire_at":fire_at, - "task":{"id":task_id,"type":"workflow","status":"leased","lease_owner":DB::optional_string(&task,"lease_owner")?, - "lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?,"attempt_count":DB::number(&task,"attempt_count")?}}); + "task":Self::task_snapshot(&task)?}); super::selections::merge(&mut timer_payload, &metadata); Self::append_at( &mut tx, @@ -2616,6 +2746,18 @@ where .unwrap() .remove("condition_wait_occurrence_id"); } + // `normalizeOptionalString`: PHP trims the condition's key and + // definition fingerprint and drops them when empty. + if !direct { + for field in ["condition_key", "condition_definition_fingerprint"] { + match payload[field].as_str().map(str::trim).filter(|value| !value.is_empty()) { + Some(value) => payload[field] = json!(value), + None => { + payload.as_object_mut().unwrap().remove(field); + } + } + } + } let pending_wait_id = if direct { let pending = Self::query("SELECT signal_wait_id FROM workflow_signal_records WHERE workflow_run_id=$1 AND signal_name=$2 AND status='received' ORDER BY command_sequence,id LIMIT 1") .bind(&run_id).bind(text(command, "signal_name")?).fetch_optional(&mut *tx).await?; @@ -2715,10 +2857,14 @@ where .bind(task_id) .execute(&mut *tx) .await?; - Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; Self::resume_undelivered_cancellation(&mut tx, &run_id, &DB::string(&run, "queue")?) .await?; - Self::enqueue_next_message(&mut tx, &run_id, &DB::string(&run, "queue")?).await?; + // PHP reports the run's next message task with the tasks the + // commands created. + if let Some(next) = Self::next_message_task(&mut tx, &run_id, &DB::string(&run, "queue")?).await? { + created.push(next); + } + Self::record_completion(&mut tx, task_id, &json!({COMPLETION_RECEIPT: {"request": body, "created_task_ids": created}})).await?; // PHP's executor leaves every run a completed task did not close // waiting, even when the task recorded only markers or side effects. Self::query("UPDATE workflow_runs SET status='waiting' WHERE id=$1 AND status IN ('pending','running')") @@ -2733,6 +2879,12 @@ where if let Some(worker) = &worker { Self::attribute_worker_terminal(&mut tx, &run_id, task_id, worker).await?; } + // `MessageStreamService::recordCompletion`: only a continued run + // changes the instance's current run, which then receives the streams. + if !streams.is_empty() || commands.iter().any(|command| command["type"] == "continue_as_new") { + self.record_stream_completion(&mut tx, &stream_instance, &run_id, task_id, &streams, transition_codec_for_streams, limits) + .await?; + } tx.summary_changed(&run_id); tx.commit().await?; self.wake.notify_waiters(); @@ -2948,7 +3100,8 @@ where payload["task"] = json!({"id":task_id,"type":"activity","status":"leased", "repair_count":DB::number(&task,"repair_count")?, "available_at":DB::optional_instant(&task,"available_at")?,"leased_at":DB::optional_instant(&task,"leased_at")?, - "lease_owner":DB::optional_string(&task,"lease_owner")?,"attempt_count":DB::number(&task,"attempt_count")?}); + "lease_owner":DB::optional_string(&task,"lease_owner")?,"lease_expires_at":DB::optional_instant(&task,"lease_expires_at")?, + "attempt_count":DB::number(&task,"attempt_count")?,"queue":DB::string(&task,"queue")?,"connection":DB::PHP_CONNECTION}); Self::append( &mut tx, &run_id, @@ -3074,6 +3227,15 @@ where json!({"timer_id":timer_id}) }; super::selections::merge(&mut payload, &metadata); + // `RunTimerTask`: the timer task is leased to itself while its + // timer fires, so `TimerFired` carries the leased task. + Self::query("UPDATE workflow_tasks SET status='leased',leased_at=$1,lease_owner=$2,lease_expires_at=$3,attempt_count=attempt_count+1 WHERE id=$4") + .bind(DB::bind_time(fired_at)) + .bind(&task_id) + .bind(DB::bind_time(fired_at + chrono::Duration::seconds(self.workflow_task_lease_seconds))) + .bind(&task_id) + .execute(&mut *tx) + .await?; Self::append( &mut tx, &run_id, @@ -3097,7 +3259,7 @@ where .bind(&timer_id) .execute(&mut *tx) .await?; - Self::query("UPDATE workflow_tasks SET status='completed' WHERE id=$1") + Self::query("UPDATE workflow_tasks SET status='completed',lease_expires_at=NULL WHERE id=$1") .bind(&task_id) .execute(&mut *tx) .await?; @@ -3426,9 +3588,22 @@ where .execute(&mut **tx) .await?; let recorded_at = recorded_at.unwrap_or_else(now); - // `WorkflowHistoryEvent::record`: an event recorded with a command - // carries that command's snapshot. let mut payload = payload; + // `WorkflowHistoryEvent::record`: an event PHP records with its task + // model carries that task's snapshot. + if let Some(task_id) = task_id + && TASK_SNAPSHOT_EVENTS.contains(&event_type) + && payload.as_object().is_some_and(|fields| !fields.contains_key("task")) + && let Some(task) = Self::query("SELECT * FROM workflow_tasks WHERE id=$1 AND workflow_run_id=$2") + .bind(task_id) + .bind(run_id) + .fetch_optional(&mut **tx) + .await? + { + // Only the run's own task: another run's is never copied. + payload["task"] = Self::task_snapshot(&task)?; + } + // An event recorded with a command carries that command's snapshot. if let Some(command_id) = command_id.filter(|_| command_snapshot) && payload.as_object().is_some_and(|fields| !fields.contains_key("command")) && let Some(snapshot) = Self::command_snapshot(tx, command_id).await? @@ -4055,7 +4230,7 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); value["runtime_external_payload_transport"] = super::payload_discovery::transport_manifest(); @@ -4086,3 +4261,7 @@ pub(crate) fn capabilities() -> Value { } value } + +#[cfg(test)] +#[path = "store_record_tests.rs"] +mod record_tests; diff --git a/rust/src/runtime/store_record_tests.rs b/rust/src/runtime/store_record_tests.rs new file mode 100644 index 000000000..9529c9473 --- /dev/null +++ b/rust/src/runtime/store_record_tests.rs @@ -0,0 +1,152 @@ +//! History recording and described-command boundaries that a public route +//! cannot reach with mismatched identifiers. +use super::Store; +use crate::runtime::backend::Backend; +use crate::runtime::external_resolution::STORED_PREFIX; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[test] +fn described_commands_read_only_the_exact_native_stored_prefix() { + // Each backend compares the first 30 characters, byte for byte, with the + // whole prefix: a signal's in its signal record, a start's in its command. + assert_eq!(STORED_PREFIX.len(), 30); + let quoted = format!("'{STORED_PREFIX}'"); + for (backend, sql, signal, start) in [ + ("sqlite", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30) COLLATE BINARY=", "substr(c.payload,1,30) COLLATE BINARY="), + ("postgres", ::RUN_COMMANDS_SQL, + "substr(s.arguments,1,30)=", "substr(c.payload,1,30)="), + ("mysql", ::RUN_COMMANDS_SQL, + "CAST(SUBSTRING(s.arguments,1,30) AS BINARY)=", "CAST(SUBSTRING(c.payload,1,30) AS BINARY)="), + ] { + let expected = format!("CASE WHEN c.command_type='signal' AND {signal}{quoted} THEN s.arguments \ + WHEN c.command_type='start' AND {start}{quoted} THEN c.payload END"); + assert!(sql.contains(&expected), "{backend}: {sql}"); + assert!(!sql.contains("LIKE"), "{backend}: {sql}"); + } +} + +async fn recorded_task(store: &Store, run_id: &str) -> Value { + let row = Store::::query( + "SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SideEffectRecorded'", + ) + .bind(run_id) + .fetch_one(&store.pool) + .await + .unwrap(); + ::document_row(&row, "payload").unwrap()["task"].clone() +} + +#[tokio::test] +async fn a_task_snapshot_comes_only_from_the_events_own_run() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("record.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + store + .register( + json!({"worker_id":"record-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]}), + &crate::runtime::QueryTaskConfig::default(), + None, + ) + .await + .unwrap(); + let mut runs = Vec::new(); + for workflow_id in ["record-own", "record-other"] { + let started = store + .start( + json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":{"codec":"avro","blob":"AA=="}}), + BTreeMap::new(), + None, + None, + ) + .await + .unwrap(); + runs.push(started["run_id"].as_str().unwrap().to_owned()); + } + let task = Store::::query("SELECT id FROM workflow_tasks WHERE workflow_run_id=$1") + .bind(&runs[0]) + .fetch_one(&store.pool) + .await + .unwrap(); + let task_id = ::string(&task, "id").unwrap(); + + let mut tx = store.begin().await.unwrap(); + for run_id in &runs { + Store::::record(&mut tx, run_id, "SideEffectRecorded", json!({"sequence":1}), + Some(&task_id), None, None, false) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + assert_eq!(recorded_task(&store, &runs[0]).await["id"], json!(task_id), "the run's own task"); + // Recorded on the other run with the first run's task, the event carries + // no snapshot at all. + assert!(recorded_task(&store, &runs[1]).await.is_null()); +} + +// The query itself, run on rows whose inputs begin like a stored reference: +// only native's exact prefix in the command type's own input column is read. +#[tokio::test] +async fn described_commands_select_no_inline_payload() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::::new( + crate::runtime::sqlite::open(directory.path().join("commands.sqlite").to_str().unwrap()) + .await + .unwrap(), + ); + let exact = format!("{STORED_PREFIX}{{}}"); + let large = format!("dw-{}", "A".repeat(4096)); + let near = format!("dw-native-external-payload-v1{}", "A".repeat(4096)); + let upper = exact.to_uppercase(); + // (id, type, command payload, signal arguments, read) + let rows = [ + ("c1", "signal", "AA==", Some(exact.as_str()), Some(exact.as_str())), + ("c2", "signal", "AA==", Some(upper.as_str()), None), + ("c3", "signal", "AA==", Some(near.as_str()), None), + ("c4", "signal", "AA==", Some(large.as_str()), None), + // A signal's command row is not its input. + ("c5", "signal", exact.as_str(), Some("AA=="), None), + ("c6", "start", exact.as_str(), None, Some(exact.as_str())), + ("c7", "start", large.as_str(), None, None), + // Other commands keep no stored reference. + ("c8", "update", exact.as_str(), None, None), + ]; + for (sequence, (id, kind, payload, arguments, _)) in rows.iter().enumerate() { + Store::::query("INSERT INTO workflow_commands(id,workflow_instance_id,workflow_run_id,command_type,status,payload,command_sequence) VALUES ($1,'instance','run',$2,'accepted',$3,$4)") + .bind(*id) + .bind(*kind) + .bind(*payload) + .bind(sequence as i64 + 1) + .execute(&store.pool) + .await + .unwrap(); + if let Some(arguments) = arguments { + Store::::query("INSERT INTO workflow_signal_records(id,workflow_command_id,signal_name,status,arguments) VALUES ($1,$2,'s','received',$3)") + .bind(format!("s-{id}")) + .bind(*id) + .bind(*arguments) + .execute(&store.pool) + .await + .unwrap(); + } + } + let selected = Store::::query(::RUN_COMMANDS_SQL) + .bind("run") + .bind("instance") + .fetch_all(&store.pool) + .await + .unwrap(); + assert_eq!(selected.len(), rows.len()); + for (row, (id, _, _, _, read)) in selected.iter().zip(&rows) { + assert_eq!(::string(row, "id").unwrap(), *id); + let stored = ::optional_string(row, "stored_payload").unwrap(); + assert_eq!(stored.as_deref(), *read, "{id}"); + } +} diff --git a/rust/src/runtime/wait_projection.rs b/rust/src/runtime/wait_projection.rs index 6467aa436..6dfdfc2ba 100644 --- a/rust/src/runtime/wait_projection.rs +++ b/rust/src/runtime/wait_projection.rs @@ -267,7 +267,8 @@ where ) -> Result<()> { if event_type == "ConditionWaitOpened" { let wait_id = super::text(payload, "condition_wait_id")?; - let key = super::text(payload, "condition_key")?; + // PHP records a condition wait without a key when none was given. + let key = payload["condition_key"].as_str(); let sequence = wait_sequence(payload)?; let projection_id = projection_id(run_id, wait_id); let summary = payload["timeout_seconds"].as_i64().map_or_else( diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 2915050d2..8fd65192f 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -87,6 +87,8 @@ mod rejected_updates; mod continue_messages; #[path = "execution/update_wait.rs"] mod update_wait; +#[path = "execution/keyless_condition_waits.rs"] +mod keyless_condition_waits; #[path = "execution/payload_limits.rs"] mod payload_limits; #[path = "execution/start_definition.rs"] @@ -113,6 +115,8 @@ mod history_pages; mod declared_signals; #[path = "execution/worker_history_presentation.rs"] mod worker_history_presentation; +#[path = "execution/message_streams.rs"] +mod message_streams; #[path = "execution/poll_waits.rs"] mod poll_waits; #[path = "execution/child_cascade.rs"] diff --git a/rust/tests/execution/external_payloads.rs b/rust/tests/execution/external_payloads.rs index c3771c42e..5acb9e25a 100644 --- a/rust/tests/execution/external_payloads.rs +++ b/rust/tests/execution/external_payloads.rs @@ -133,7 +133,7 @@ async fn unclaimed(database: &TestDatabase) -> i64 { } /// One integer the database answers. -async fn count(database: &TestDatabase, query: &'static str) -> i64 { +async fn count(database: &TestDatabase, query: impl sqlx::SqlSafeStr) -> i64 { match database { TestDatabase::Sqlite(dir) => { let pool = sqlx::SqlitePool::connect(&format!( diff --git a/rust/tests/execution/external_payloads/commands.rs b/rust/tests/execution/external_payloads/commands.rs index 82237259c..e93b63183 100644 --- a/rust/tests/execution/external_payloads/commands.rs +++ b/rust/tests/execution/external_payloads/commands.rs @@ -151,3 +151,86 @@ async fn signal_update_and_query_input_resolve_runtime_references_like_php() { runtime.close().await; database.remove().await; } + +/// One described command of `kind`, which must show only metadata. +async fn described(app: &Router, workflow_id: &str, kind: &str) -> Value { + let (status, body) = json_request(app, "GET", &format!("/api/workflows/{workflow_id}"), "operator-token", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let command = body["commands"].as_array().unwrap().iter().find(|command| command["type"] == kind).unwrap().clone(); + for field in ["payload", "external_payload", "external_storage", "reference_id", "sha256", "size_bytes", "arguments"] { + assert!(command.get(field).is_none(), "{field}: {command}"); + } + assert!(!command.to_string().contains("ep_"), "no reference identity: {command}"); + command +} + +// `WorkflowController::oversizedPayloadPreview` over each command's stored +// envelope: only a stored external reference larger than the payload limit +// (2 MiB by default) omits the preview, and then the target name too. Native +// keeps a signal's stored reference in its signal record and a start's in the +// command row. +#[tokio::test] +async fn described_commands_omit_only_an_oversized_stored_reference() { + let temp = tempfile::tempdir().unwrap(); + let (database, runtime, app) = setup(ExternalPayloadConfig::default(), &temp.path().join("objects")).await; + waiting_run(&app, "described-reference", json!({"worker_id":"describe-worker","task_queue":"described-reference", + "runtime":"php","supported_workflow_types":["echo"],"supported_activity_types":[]})).await; + let reference = upload(&app, &argument("signal")).await; + let (status, body) = json_request(&app, "POST", "/api/workflows/described-reference/signal/s", "operator-token", + json!({"input":{"codec":"avro","external_payload":reference}})).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + let signal = command["id"].as_str().unwrap().to_owned(); + // The real stored reference is in the signal record, not the command row. + let identity = reference["reference_id"].as_str().unwrap(); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_signal_records WHERE workflow_command_id='{signal}' AND arguments LIKE 'dw-native-external-payload-v1:%{identity}%'"))).await, 1); + assert_eq!(count(&database, sqlx::AssertSqlSafe(format!( + "SELECT COUNT(*) FROM workflow_commands WHERE id='{signal}' AND payload LIKE 'dw-native%'"))).await, 0); + + // The same stored reference, as native writes it, at sizes around the + // limit, and counterfactuals that are not native stored references. + let limit = 2 * 1024 * 1024_i64; + let stored = |size: i64| { + let mut reference = reference.clone(); + reference["size_bytes"] = json!(size); + format!("dw-native-external-payload-v1:{reference}") + }; + let cases = [ + (stored(limit), false), + (stored(limit + 1), true), + // Not native's exact prefix, so never read. + (stored(limit + 1).replace("dw-native-external-payload-v1:", "DW-NATIVE-EXTERNAL-PAYLOAD-V1:"), false), + (stored(limit + 1).replace("dw-native-external-payload-v1:", "dw-native-external-payload-v2:"), false), + // Native's prefix with no valid reference after it. + (format!("dw-native-external-payload-v1:{{\"size_bytes\":{}}}", limit + 1), false), + // A large inline payload. + (format!("dw-{}", "A".repeat(limit as usize + 1)), false), + ]; + for (arguments, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{arguments}' WHERE workflow_command_id='{signal}'"))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "{}: {command}", &arguments[..arguments.len().min(48)]); + assert_eq!(command.get("target_name").is_none(), *omitted, "{command}"); + } + // An oversized reference in the signal's command row is not its input. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_signal_records SET arguments='{}' WHERE workflow_command_id='{signal}'", stored(limit)))).await; + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{}' WHERE id='{signal}'", stored(limit + 1)))).await; + let command = described(&app, "described-reference", "signal").await; + assert_eq!((command["payload_preview_omitted"].clone(), command["target_name"].clone()), (json!(false), json!("s")), "{command}"); + + // A start keeps its stored input reference in the command row. + let start = described(&app, "described-reference", "start").await["id"].as_str().unwrap().to_owned(); + for (payload, omitted) in &cases { + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_commands SET payload='{payload}' WHERE id='{start}'"))).await; + let command = described(&app, "described-reference", "start").await; + assert_eq!(command["payload_preview_omitted"], json!(omitted), "start {}: {command}", &payload[..payload.len().min(48)]); + } + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/keyless_condition_waits.rs b/rust/tests/execution/keyless_condition_waits.rs new file mode 100644 index 000000000..2d66bfa6a --- /dev/null +++ b/rust/tests/execution/keyless_condition_waits.rs @@ -0,0 +1,63 @@ +use super::*; + +// Workflow 2.9's `normalizeOpenConditionWaitCommand`: a condition wait's key +// and definition fingerprint are optional strings of any form. PHP trims them +// and leaves them out when empty. + +fn opened(history: &Value) -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "ConditionWaitOpened") + .map(|event| event["payload"].clone()).unwrap() +} + +#[tokio::test] +async fn condition_waits_need_neither_a_key_nor_a_fingerprint() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "keyless-condition").await; + register(&app, "keyless", json!(["echo"]), json!([])).await; + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_kind='condition' AND wait_reason='Waiting for condition'").await, 1); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_waits WHERE kind='condition' AND status='open' AND target_name IS NULL").await, 1); + + // A signal wakes the wait; the next one keeps a trimmed key and any + // fingerprint, and drops a blank one. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ready ", + "condition_definition_fingerprint":"any form"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let payload = opened(&run_history(&app, &started["workflow_id"], &started["run_id"]).await); + assert_eq!((payload["condition_key"].clone(), payload["condition_definition_fingerprint"].clone()), + (json!("ready"), json!("any form")), "{payload}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition ready'").await, 1); + + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/again", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":" ", + "condition_definition_fingerprint":"","timeout_seconds":60}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + let payload = opened(&history); + assert!(payload.get("condition_key").is_none() && payload.get("condition_definition_fingerprint").is_none(), "{payload}"); + let scheduled = history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == "TimerScheduled").unwrap(); + assert!(scheduled["payload"].get("condition_key").is_none(), "{scheduled}"); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_summaries WHERE wait_reason='Waiting for condition or timeout'").await, 1); + + // A key that is not a string is still refused. + let (status, body) = request(&app, "POST", "/api/workflows/keyless-condition/signal/third", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let task = poll(&app, "keyless", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":7}])).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); +} diff --git a/rust/tests/execution/message_streams.rs b/rust/tests/execution/message_streams.rs new file mode 100644 index 000000000..254cfb6f8 --- /dev/null +++ b/rust/tests/execution/message_streams.rs @@ -0,0 +1,178 @@ +use super::*; + +// Server 2.11.1 message streams: ordered, deduplicated appends delivered to +// the current run as the runtime-reserved signal; worker acknowledgements and +// waits on workflow task completions; the hand-over to a continued run. + +const STREAM_SIGNAL: &str = "__durable_workflow_message_stream"; + +fn stream_worker() -> Value { + json!({"worker_id":"streams","task_queue":"test","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":[],"capabilities":["message_streams"]}) +} + +/// An SDK receive: a condition wait keyed by the stream cursor. +fn receive(cursor: i64) -> Value { + json!([{"type":"open_condition_wait","condition_key":format!("message-stream:orders:{cursor}"), + "condition_definition_fingerprint":format!("sha256:{}", "a".repeat(64))}]) +} + +async fn append(app: &Router, workflow: &str, stream: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", &format!("/api/workflows/{workflow}/message-streams/{stream}/messages"), body).await +} + +async fn complete(app: &Router, task: &Value, commands: Value, extra: Value, protocol: &str) -> (StatusCode, Value) { + let mut body = completion(task, commands); + body.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + request_protocol(app, "POST", &format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()), body, protocol).await +} + +async fn streams(app: &Router, workflow: &str) -> Value { + let (status, body) = request(app, "GET", &format!("/api/workflows/{workflow}/message-streams"), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body +} + +/// The stream deliveries a history holds: (schema tail, stream, message id or cursor). +fn deliveries(history: &Value) -> Vec<(String, String, String)> { + history.as_array().unwrap().iter() + .filter(|event| event["event_type"] == "SignalReceived" && event["payload"]["signal_name"] == STREAM_SIGNAL) + .map(|event| { + let blob = event["payload"]["arguments"]["blob"].as_str().unwrap(); + let Payload::Array(values) = durable_workflow_server::codec::ValueCodec::new().unwrap().decode(blob).unwrap() else { + panic!("one argument list"); + }; + let Payload::Map(delivery) = &values[0] else { panic!("one delivery map") }; + let text = |key: &str| match delivery.get(key) { + Some(Payload::String(value)) => value.clone(), + Some(Payload::Long(value)) => value.to_string(), + other => panic!("{key}: {other:?}"), + }; + let schema = text("schema"); + let marker = if schema.ends_with(".cursor") { text("through_position") } else { text("message_id") }; + (schema.rsplit('.').next().unwrap().to_owned(), text("stream_name"), marker) + }) + .collect() +} + +fn types(history: &Value) -> Vec { + history.as_array().unwrap().iter().map(|event| event["event_type"].as_str().unwrap().to_owned()).collect() +} + +#[tokio::test] +async fn appended_messages_reach_the_run_and_follow_it_when_it_continues() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + let started = start(&app, "streams-flow").await; + let task = poll(&app, "streams", "workflow").await; + let waits = json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}); + let (status, body) = complete(&app, &task, receive(0), waits, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let listed = streams(&app, "streams-flow").await; + assert_eq!(listed["streams"][0]["pending_wait"]["after_position"], 0, "{listed}"); + assert_eq!(listed["streams"][0]["pending_wait"]["run_id"], started["run_id"]); + + let (status, first) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &first["position"], &first["run_id"]), (StatusCode::ACCEPTED, &json!(1), &started["run_id"]), "{first}"); + let (status, again) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &again["duplicate"], &again["position"]), (StatusCode::OK, &json!(true), &json!(1)), "{again}"); + assert!(again.get("run_id").is_none(), "a duplicate names no run: {again}"); + let (status, conflict) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":9}]})).await; + assert_eq!((status, &conflict["reason"]), (StatusCode::CONFLICT, &json!("message_identity_conflict")), "{conflict}"); + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m2","input":[{"n":2}]})).await.1["position"], 2); + let listed = streams(&app, "streams-flow").await; + let orders = &listed["streams"][0]; + assert_eq!((&orders["last_position"], &orders["pending_count"], &orders["duplicate_count"]), (&json!(2), &json!(2), &json!(2)), "{listed}"); + assert!(orders["pending_wait"].is_null(), "an append clears the wait: {listed}"); + + // The condition wait wakes for the first message; the worker reads both. + let task = poll(&app, "streams", "workflow").await; + assert_eq!(deliveries(&task["history_events"]), [("message".into(), "orders".into(), "m1".into()), ("message".into(), "orders".into(), "m2".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}); + let (status, body) = complete(&app, &task, receive(1), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["created_task_ids"].as_array().unwrap().len(), 1, "the next message task is reported: {body}"); + let history = run_history(&app, &started["workflow_id"], &started["run_id"]).await; + assert!(!types(&history).contains(&"SignalApplied".to_owned()), "the condition wait consumes the message: {history}"); + let cursors: Vec<_> = history.as_array().unwrap().iter().filter(|event| event["event_type"] == "MessageCursorAdvanced").collect(); + assert_eq!(cursors.len(), 2, "{history}"); + assert_eq!(cursors[0]["payload"]["stream_key"], "instance:streams-flow"); + assert_eq!(cursors[0]["payload"]["task"]["status"], "leased"); + assert_eq!((&cursors[1]["payload"]["stream_key"], &cursors[1]["payload"]["new_position"]), (&json!("orders"), &json!(1))); + assert_eq!(cursors[1]["payload"]["task"]["status"], "completed"); + assert_eq!(streams(&app, "streams-flow").await["streams"][0]["cursor_position"], 1); + + // Continue as new acknowledging the second message: the next run gets the + // cursor and the third message, which was still unconsumed. + assert_eq!(append(&app, "streams-flow", "orders", json!({"message_id":"m3","input":[{"n":3}]})).await.1["position"], 3); + let task = poll(&app, "streams", "workflow").await; + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":2}]}); + let (status, body) = complete(&app, &task, json!([{"type":"continue_as_new","arguments":envelope(Payload::Array(vec![]))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "streams", "workflow").await; + assert_ne!(task["run_id"], started["run_id"], "{task}"); + assert_eq!(deliveries(&task["history_events"]), [("cursor".into(), "orders".into(), "2".into()), ("message".into(), "orders".into(), "m3".into())]); + let cursor = json!({"message_stream_cursors":[{"stream_name":"orders","through_position":3}]}); + let (status, body) = complete(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::String("done".into()))}]), cursor, "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let orders = &streams(&app, "streams-flow").await["streams"][0]; + assert_eq!((&orders["cursor_position"], &orders["pending_count"]), (&json!(3), &json!(0)), "{orders}"); + let (status, closed) = append(&app, "streams-flow", "orders", json!({"message_id":"m1","input":[{"n":1}]})).await; + assert_eq!((status, &closed["reason"]), (StatusCode::CONFLICT, &json!("run_not_active")), "{closed}"); +} + +#[tokio::test] +async fn refused_appends_and_acknowledgements_change_nothing_but_the_counts() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + assert_eq!(request(&app, "POST", "/api/worker/register", stream_worker()).await.0, StatusCode::CREATED); + start(&app, "streams-refused").await; + let task = poll(&app, "streams", "workflow").await; + for (extra, field) in [ + (json!({"message_stream_waits":[{"stream_name":"orders","after_position":1}]}), "message_stream_waits.0.after_position"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":1}]}), "message_stream_cursors.0.through_position"), + (json!({"message_stream_waits":[{"stream_name":"bad name","after_position":0}]}), "message_stream_waits.0.stream_name"), + (json!({"message_stream_cursors":[{"stream_name":"orders","through_position":-1}]}), "message_stream_cursors.0.through_position"), + ] { + let (status, body) = complete(&app, &task, receive(0), extra, "1.20").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert!(body["errors"].get(field).is_some(), "{field}: {body}"); + } + let (status, body) = complete(&app, &task, receive(0), json!({"message_stream_waits":[{"stream_name":"orders","after_position":0}]}), "1.14").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("message_streams_unavailable")), "{body}"); + assert_eq!(streams(&app, "streams-refused").await["count"], 0, "a refused completion records no stream"); + + for (body, field) in [(json!({"input":[1]}), "message_id"), (json!({"message_id":"bad id!"}), "message_id"), + (json!({"message_id":"x".repeat(192)}), "message_id"), (json!({"message_id":"m1","input":"text"}), "input")] { + let (status, refused) = append(&app, "streams-refused", "orders", body).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert!(refused["errors"].get(field).is_some(), "{refused}"); + } + let (status, refused) = append(&app, "streams-refused", "bad%20name", json!({"message_id":"m1"})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + let (status, refused) = append(&app, "streams-missing", "orders", json!({"message_id":"m1"})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::NOT_FOUND, &json!("instance_not_found")), "{refused}"); + let orders = &streams(&app, "streams-refused").await["streams"][0]; + assert_eq!((&orders["malformed_count"], &orders["last_position"]), (&json!(4), &json!(0)), "{orders}"); + // The last refused append kept a valid id; the refused stream name and + // the unknown workflow count nowhere. + assert_eq!((&orders["last_input_outcome"], &orders["last_input_message_id"]), (&json!("malformed"), &json!("m1")), "{orders}"); + let (status, refused) = request(&app, "POST", &format!("/api/workflows/streams-refused/signal/{STREAM_SIGNAL}"), json!([1])).await; + assert_eq!((status, &refused["reason"]), (StatusCode::UNPROCESSABLE_ENTITY, &json!("runtime_reserved_signal")), "{refused}"); + let (status, missing) = request(&app, "GET", "/api/workflows/streams-refused/message-streams/other", Value::Null).await; + assert_eq!((status, &missing["reason"]), (StatusCode::NOT_FOUND, &json!("message_stream_not_found")), "{missing}"); +} + +#[tokio::test] +async fn cluster_info_and_workers_see_the_contract() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, info) = request(&app, "GET", "/api/cluster/info", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{info}"); + assert_eq!(info["message_streams_contract"]["schema"], "durable-workflow.v2.message-streams.contract"); + assert_eq!(info["message_streams_contract"]["transport"]["internal_signal"], STREAM_SIGNAL); + let capability = &info["worker_protocol"]["server_capabilities"]["message_streams"]; + assert_eq!(capability["supported"], true, "{capability}"); + assert_eq!(capability["minimum_worker_protocol_version"], "1.15"); +} diff --git a/rust/tests/execution/worker_history_presentation.rs b/rust/tests/execution/worker_history_presentation.rs index c1f0de28d..b96250e2b 100644 --- a/rust/tests/execution/worker_history_presentation.rs +++ b/rust/tests/execution/worker_history_presentation.rs @@ -63,6 +63,65 @@ async fn a_worker_reads_the_sequence_that_consumed_a_signal() { assert!(completed["payload"]["task"].get("last_claim_error").is_none(), "unset fields stay out: {completed}"); } +// `WorkflowHistoryEvent::record` with the task model: each event PHP records +// while handling a task carries that task's snapshot, and a fired timer +// carries its timer task, leased to itself. +#[tokio::test] +async fn events_recorded_with_a_task_carry_its_snapshot() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let started = start(&app, "snapshot-events").await; + register(&app, "snapshotter", json!(["echo", "child"]), json!([])).await; + let events = || run_history(&app, &started["workflow_id"], &started["run_id"]); + let snapshot_of = |history: &Value, kind: &str| -> Value { + history.as_array().unwrap().iter().rev().find(|event| event["event_type"] == kind) + .map(|event| event["payload"]["task"].clone()).unwrap_or(Value::Null) + }; + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([{"type":"start_timer","delay_seconds":1}])).await.0, StatusCode::OK); + let scheduled = snapshot_of(&events().await, "TimerScheduled"); + assert_eq!((scheduled["id"].clone(), scheduled["type"].clone()), (task["task_id"].clone(), json!("workflow")), "{scheduled}"); + for field in ["queue", "connection", "available_at", "leased_at", "repair_count"] { + assert!(scheduled.get(field).is_some(), "{field}: {scheduled}"); + } + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(15); + let fired = loop { + let fired = snapshot_of(&events().await, "TimerFired"); + if !fired.is_null() { + break fired; + } + assert!(tokio::time::Instant::now() < deadline, "the timer never fired"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + }; + assert_eq!((fired["type"].clone(), fired["status"].clone(), fired["attempt_count"].clone()), + (json!("timer"), json!("leased"), json!(1)), "{fired}"); + assert_eq!(fired["lease_owner"], fired["id"], "{fired}"); + assert!(fired["lease_expires_at"].is_string() && fired["leased_at"].is_string(), "{fired}"); + + let task = poll(&app, "snapshotter", "workflow").await; + assert_eq!(finish_task(&app, &task, json!([ + {"type":"start_child_workflow","workflow_type":"child","arguments":envelope(Payload::Array(vec![]))}, + {"type":"open_signal_wait","signal_name":"go"}])).await.0, StatusCode::OK); + let history = events().await; + for kind in ["ChildWorkflowScheduled", "ChildRunStarted", "SignalWaitOpened"] { + assert_eq!(snapshot_of(&history, kind)["id"], task["task_id"], "{kind}: {history}"); + } + let (status, body) = request(&app, "POST", "/api/workflows/snapshot-events/signal/go", json!([])).await; + assert_eq!(status, StatusCode::ACCEPTED, "{body}"); + let mut task = poll(&app, "snapshotter", "workflow").await; + // The child's first task may come first on the shared queue. + if task["workflow_type"] == "child" { + assert_eq!(finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Null)}])).await.0, StatusCode::OK); + task = poll(&app, "snapshotter", "workflow").await; + } + assert_eq!(finish_task(&app, &task, json!([{"type":"open_condition_wait","condition_key":"k", + "condition_definition_fingerprint":format!("sha256:{}", "c".repeat(64))}])).await.0, StatusCode::OK); + let history = events().await; + assert_eq!(snapshot_of(&history, "SignalApplied")["id"], task["task_id"], "{history}"); + assert_eq!(snapshot_of(&history, "ConditionWaitOpened")["id"], task["task_id"], "{history}"); +} + // Workflow 2.10.2 `DefaultWorkflowTaskBridge::applySignal` takes a workflow // sequence only from the signal's own wait. A signal the run's contract // declares is applied without one even while a condition wait is open, so diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index 886219a91..5bd1bdd32 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -43,9 +43,6 @@ POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome POST /api/workflows/import/embedded-v2 POST /api/workflows/import/waterline-v1 -GET /api/workflows/{workflowId}/message-streams -GET /api/workflows/{workflowId}/message-streams/{streamName} -POST /api/workflows/{workflowId}/message-streams/{streamName}/messages GET /api/workflows/{workflowId}/nexus-operations GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 69b947698..487639bfa 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -21,6 +21,7 @@ import {checkPayloadCleanup} from './payload-cleanup-contract.mjs'; import {checkClusterInfo} from './cluster-info-contract.mjs'; import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; +import {checkMessageStreams} from './message-streams-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -159,6 +160,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const clusterInfo = fixture.cluster_info ? checkClusterInfo(fixture, observation, workflowId) : null; const repairPass = fixture.repair_pass ? checkRepairPass(fixture, observation, workflowId) : null; const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; + const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -714,6 +716,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(clusterInfo ? {cluster_info: clusterInfo} : {}), ...(repairPass ? {repair_pass: repairPass} : {}), ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), + ...(messageStreams ? {message_streams: messageStreams} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/message-streams-contract.mjs b/scripts/conformance/server-parity/message-streams-contract.mjs new file mode 100644 index 000000000..829508c2a --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-contract.mjs @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; + +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/; +const MESSAGE = 'durable-workflow.v2.message-stream.message'; +const CURSOR = 'durable-workflow.v2.message-stream.cursor'; + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + assert.equal(read.status, 422, `${label} status`); + assert.deepStrictEqual(read.response, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, `${label} body`); +} + +// One stream's diagnostics: exact counts, the pending wait by run and a +// microsecond UTC time, and the last input. +function checkStream(entry, expected, runId, label) { + const {pending_wait: wait, last_input_at: inputAt, ...rest} = entry; + const {wait: expectedWait = null, input = null, ...counts} = expected; + assert.deepStrictEqual(rest, {...counts, pending_count: counts.last_position - counts.cursor_position, + duplicate_count: counts.duplicate_count ?? 0, malformed_count: counts.malformed_count ?? 0, + last_input_outcome: input?.outcome ?? null, last_input_message_id: input?.message_id ?? null}, `${label} counts`); + if (expectedWait === null) { + assert.equal(wait, null, `${label} has no pending wait`); + } else { + assert.equal(wait?.run_id, runId, `${label} pending wait run`); + assert.equal(wait?.after_position, expectedWait.after_position, `${label} pending wait position`); + assert.match(wait?.since ?? '', TIME, `${label} pending wait time`); + } + if (input === null) { + assert.equal(inputAt, null, `${label} has no input yet`); + } else { + assert.match(inputAt ?? '', TIME, `${label} last input time`); + } +} + +function checkList(read, workflowId, streams, runId, label) { + assert.equal(read.status, 200, `${label} status`); + assert.equal(read.response.workflow_id, workflowId, `${label} workflow`); + assert.equal(read.response.count, Object.keys(streams).length, `${label} count`); + assert.deepStrictEqual(read.response.streams.map(entry => entry.stream_name), Object.keys(streams), `${label} streams by name`); + for (const entry of read.response.streams) { + checkStream(entry, {stream_name: entry.stream_name, ...streams[entry.stream_name]}, runId, `${label} ${entry.stream_name}`); + } +} + +function accepted(read, workflowId, runId, stream, messageId, position, cursor, label) { + assert.equal(read.status, 202, `${label} status`); + assert.deepStrictEqual(read.response, {accepted: true, duplicate: false, workflow_id: workflowId, run_id: runId, + stream_name: stream, message_id: messageId, position, cursor_position: cursor, outcome: 'accepted', reason: null}, `${label} body`); +} + +function completed(read, runId, status, created, label) { + assert.equal(read.status, 200, `${label} status`); + const {task_id: taskId, created_task_ids: createdIds, ...rest} = read.response; + assert.equal(typeof taskId, 'string', `${label} task`); + assert.deepStrictEqual(rest, {workflow_task_attempt: 1, outcome: 'completed', recorded: true, run_id: runId, + run_status: status, reason: null}, `${label} outcome`); + assert.equal(createdIds.length, created, `${label} created tasks, the next message task included`); +} + +// A message delivery or cursor checkpoint as the worker reads it. +function delivery(stream, messageId, position, payload) { + return [{schema: MESSAGE, stream_name: stream, message_id: messageId, position, payload_envelope: ['blob', 'codec'], payload}]; +} + +// Reviewed message streams (published Server 2.11.1). +export function checkMessageStreams(fixture, observation, workflowId) { + const definition = fixture.message_streams; + const state = observation.message_streams; + // The same for every target: embedded applications have no HTTP message streams. + const projection = {scope: 'message_streams', first_run_events: definition.first_run_events.length, + continued_run_events: definition.continued_run_events.length}; + assert.ok(state && typeof state === 'object', 'actual message stream observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_message_streams'}, + 'embedded message streams explicitly inapplicable'); + return projection; + } + const id = `${workflowId}-ms`; + const {stream, other_stream: other, refusals: text, messages} = definition; + const run = state.run_id; + assert.equal(state.workflow_id, id, 'stream workflow'); + assert.deepStrictEqual(state.cluster_contract, definition.manifest, 'cluster info publishes the contract'); + assert.deepStrictEqual(state.worker_capability, {...definition.manifest, supported: true}, 'workers see the supported contract'); + + assert.deepStrictEqual(state.list_before, {status: 200, response: {workflow_id: id, count: 0, streams: []}}, 'no streams yet'); + assert.deepStrictEqual(state.show_before, {status: 404, response: {workflow_id: id, stream_name: stream, reason: 'message_stream_not_found'}}, + 'an unknown stream'); + refused(state.show_bad_name, 'stream_name', text.stream_name, 'a stream name outside the pattern'); + assert.deepStrictEqual(state.list_unknown, {status: 404, response: {message: 'Workflow not found.', reason: 'instance_not_found'}}, + 'an unknown workflow'); + assert.deepStrictEqual(state.poll1.history, [], 'nothing delivered before the first append'); + + const first = state.complete1_refusals; + refused(first.wait_ahead, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait ahead of the cursor'); + refused(first.cursor_ahead, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(first.bad_name, 'message_stream_waits.0.stream_name', text.metadata_stream_name, 'a wait stream name outside the pattern'); + refused(first.repeated_wait, 'message_stream_waits.1.stream_name', text.duplicate_wait, 'two waits on one stream'); + assert.equal(first.old_protocol.status, 409, 'stream metadata before protocol 1.15'); + const {task_id: oldTask, ...oldProtocol} = first.old_protocol.response; + assert.equal(typeof oldTask, 'string', 'the refused task'); + assert.deepStrictEqual(oldProtocol, {workflow_task_attempt: 1, outcome: 'rejected', reason: 'message_streams_unavailable', + requested_version: '1.14', minimum_protocol_version: '1.15', remediation: text.old_protocol_remediation}, 'the protocol refusal'); + completed(state.complete1, run, 'waiting', 0, 'the first completion with a wait'); + checkList(state.list_waiting, id, {[stream]: {last_position: 0, cursor_position: 0, wait: {after_position: 0}}}, run, 'the waiting stream'); + + const appends = state.appends; + accepted(appends.m1, id, run, stream, 'm1', 1, 0, 'the first message'); + assert.deepStrictEqual(appends.m1_duplicate, {status: 200, response: {accepted: true, duplicate: true, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'duplicate', reason: null}}, 'the same message again'); + assert.deepStrictEqual(appends.m1_conflict, {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: 'm1', position: 1, cursor_position: 0, outcome: 'rejected', reason: 'message_identity_conflict'}}, 'the id with another payload'); + assert.deepStrictEqual(appends.m2_sdk.response, {accepted: true, duplicate: false, workflow_id: id, run_id: run, stream_name: stream, + message_id: 'm2', position: 2, cursor_position: 0, outcome: 'accepted', reason: null}, 'the published SDK append'); + refused(appends.no_id, 'message_id', text.message_id_required, 'no message id'); + refused(appends.bad_id, 'message_id', text.message_id_format, 'a message id outside the pattern'); + refused(appends.long_id, 'message_id', text.message_id_max, 'a message id too long'); + refused(appends.bad_input, 'input', text.input_array, 'input that is not a list'); + refused(appends.bad_stream, 'stream_name', text.stream_name, 'an append to a stream name outside the pattern'); + assert.deepStrictEqual(appends.unknown_workflow, {status: 404, response: {accepted: false, workflow_id: `${id}-missing`, + stream_name: stream, outcome: 'rejected', reason: 'instance_not_found'}}, 'an append to an unknown workflow'); + accepted(appends.other_stream, id, run, other, 'a1', 1, 0, 'a message without input on another stream'); + assert.equal(appends.reserved_signal.status, 422, 'the runtime signal is not a public signal'); + assert.equal(appends.reserved_signal.response.reason, 'runtime_reserved_signal', 'the reserved signal refusal'); + // Four refused appends to the stream are malformed; the last kept the + // valid id m3. The refused stream name counts nowhere. + const appended = { + [other]: {last_position: 1, cursor_position: 0, input: {outcome: 'accepted', message_id: 'a1'}}, + [stream]: {last_position: 2, cursor_position: 0, duplicate_count: 2, malformed_count: 4, input: {outcome: 'malformed', message_id: 'm3'}}, + }; + checkList(state.list_after_appends, id, appended, run, 'after the appends'); + assert.equal(state.show_after_appends.status, 200, 'one stream'); + assert.deepStrictEqual(state.show_after_appends.response, {workflow_id: id, + stream: state.list_after_appends.response.streams.find(entry => entry.stream_name === stream)}, 'one stream as listed'); + + assert.equal(state.poll2.run_id, run, 'the woken run'); + assert.deepStrictEqual(state.poll2.history, [ + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm1', 1, messages.m1)}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm2', 2, messages.m2)}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + ], 'each message is one runtime signal, in append order'); + const second = state.complete2_refusals; + refused(second.beyond, 'message_stream_cursors.0.through_position', text.cursor_beyond, 'a cursor past the last message'); + refused(second.negative, 'message_stream_cursors.0.through_position', text.negative_cursor, 'a negative cursor'); + refused(second.wait_behind, 'message_stream_waits.0.after_position', text.wait_after_cursor, 'a wait behind the new cursor'); + // Two messages are still unconsumed, so the next message task follows. + completed(state.complete2, run, 'waiting', 1, 'the first acknowledgement'); + assert.deepStrictEqual(state.history_after2.filter(event => event.event_type === 'MessageCursorAdvanced'), [ + {event_type: 'MessageCursorAdvanced', stream_key: 'instance', previous_position: 0, new_position: 1, task_status: 'leased'}, + {event_type: 'MessageCursorAdvanced', stream_key: stream, previous_position: 0, new_position: 1, task_status: 'completed'}, + ], 'the instance cursor while the task runs, then the stream cursor once it completed'); + checkList(state.list_after2, id, {...appended, [stream]: {...appended[stream], cursor_position: 1}}, run, 'after the acknowledgement'); + + accepted(appends.m3, id, run, stream, 'm3', 3, 1, 'the third message'); + assert.equal(state.poll3.run_id, run, 'the same run again'); + assert.deepStrictEqual(state.poll3.history.filter(event => event.event_type === 'SignalReceived').at(-1), + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, 'the third message delivered'); + completed(state.complete3, run, 'completed', 1, 'continue as new'); + checkList(state.list_after_continue, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 2, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'after continue as new'); + + const continued = state.continued_run_id; + assert.equal(state.poll4.run_id, continued, 'the continued run'); + assert.notEqual(continued, run, 'a new run'); + assert.deepStrictEqual(state.poll4.history, [ + {event_type: 'SignalReceived', deliveries: [{schema: CURSOR, stream_name: stream, through_position: 2}]}, + {event_type: 'SignalReceived', deliveries: delivery(other, 'a1', 1, [])}, + {event_type: 'SignalReceived', deliveries: delivery(stream, 'm3', 3, messages.m3)}, + ], 'the continued run receives the cursor, then every unconsumed message by append order'); + completed(state.complete4, continued, 'completed', 0, 'the continued run completes'); + checkList(state.list_final, id, {...appended, [stream]: {...appended[stream], last_position: 3, cursor_position: 3, + input: {outcome: 'accepted', message_id: 'm3'}}}, run, 'the final streams'); + for (const [name, messageId] of [['closed', 'm4'], ['closed_duplicate', 'm1']]) { + assert.deepStrictEqual(appends[name], {status: 409, response: {accepted: false, duplicate: false, workflow_id: id, stream_name: stream, + message_id: messageId, position: null, outcome: 'rejected', reason: 'run_not_active'}}, `${name}: the instance has no open run`); + } + + assert.deepStrictEqual(state.first_run_events, definition.first_run_events, + 'the first run: deliveries consumed by the condition waits without SignalApplied, the rest rejected at the close'); + assert.deepStrictEqual(state.continued_run_events, definition.continued_run_events, 'the continued run'); + assert.deepStrictEqual(state.first_run_streams.filter(event => event.event_type === 'MessageCursorAdvanced').map(event => event.stream_key), + ['instance', stream, 'instance', stream], 'first run cursors'); + assert.ok(!JSON.stringify(state.list_final).includes('"n":'), 'diagnostics never carry payloads'); + for (const event of [...state.first_run_streams, ...state.continued_run_streams]) { + if (event.event_type === 'SignalReceived') { + assert.equal(event.deliveries.length, 1, 'one delivery per runtime signal'); + } + } + + return projection; +} diff --git a/scripts/conformance/server-parity/message-streams-probe.php b/scripts/conformance/server-parity/message-streams-probe.php new file mode 100644 index 000000000..bd1869b16 --- /dev/null +++ b/scripts/conformance/server-parity/message-streams-probe.php @@ -0,0 +1,163 @@ +payloadCodec(); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new MessageStreamsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, ?string $worker = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + $worker !== null ? 'X-Durable-Workflow-Protocol-Version' : 'X-Durable-Workflow-Control-Plane-Version' => $worker ?? '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $append = static fn (string $streamName, array $body): array => $raw('POST', + "workflows/{$id}/message-streams/".rawurlencode($streamName).'/messages', $body); + $complete = static fn (array $task, array $body, string $protocol = '1.20'): array => $raw('POST', + "worker/workflow-tasks/{$task['task_id']}/complete", + ['lease_owner' => $task['lease_owner'] ?? $worker, 'workflow_task_attempt' => $task['workflow_task_attempt']] + $body, $protocol); + $wait = static fn (int $cursor): array => ['type' => 'open_condition_wait', + 'condition_key' => "message-stream:{$stream}:{$cursor}", + 'condition_definition_fingerprint' => 'sha256:'.hash('sha256', $definition['condition_fingerprint_source'])]; + // A stream's deliveries and cursor moves in history, decoded. + $streamEvents = static function (array $events) use ($codec, $id): array { + $rows = []; + foreach ($events as $event) { + $payload = $event['payload'] ?? []; + if (($event['event_type'] ?? null) === 'SignalReceived' + && ($payload['signal_name'] ?? null) === '__durable_workflow_message_stream') { + $deliveries = $codec->decodeEnvelope($payload['arguments']); + foreach ($deliveries as &$delivery) { + if (is_array($delivery['payload_envelope'] ?? null)) { + $envelope = $delivery['payload_envelope']; + $delivery['payload_envelope'] = array_keys($envelope); + sort($delivery['payload_envelope']); + $delivery['payload'] = $codec->decodeEnvelope($envelope); + } + } + unset($delivery); + $rows[] = ['event_type' => 'SignalReceived', 'deliveries' => $deliveries]; + } elseif (($event['event_type'] ?? null) === 'MessageCursorAdvanced') { + $key = $payload['stream_key'] ?? null; + $rows[] = ['event_type' => 'MessageCursorAdvanced', + 'stream_key' => $key === "instance:{$id}" ? 'instance' : $key, + 'previous_position' => $payload['previous_position'] ?? null, + 'new_position' => $payload['new_position'] ?? null, + 'task_status' => $payload['task']['status'] ?? null]; + } + } + + return $rows; + }; + $history = static function (string $runId) use ($raw, $id): array { + $read = $raw('GET', "workflows/{$id}/runs/{$runId}/history"); + + return $read['response']['events'] ?? []; + }; + $poll = static function () use ($client, $worker, $queue, $streamEvents): array { + $task = $client->pollWorkflowTask($worker, $queue, 0); + if (! is_array($task)) { + throw new RuntimeException('No workflow task was ready for the message stream run.'); + } + + return [$task, ['run_id' => $task['run_id'] ?? null, 'history' => $streamEvents($task['history_events'] ?? [])]]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $info = $raw('GET', 'cluster/info')['response']; + $state['cluster_contract'] = $info['message_streams_contract'] ?? null; + $state['worker_capability'] = $info['worker_protocol']['server_capabilities']['message_streams'] ?? null; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $queue, [$fixture['input']])->selectedRunId; + $state['list_before'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_before'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + $state['show_bad_name'] = $raw('GET', "workflows/{$id}/message-streams/".rawurlencode($definition['bad_stream'])); + $state['list_unknown'] = $raw('GET', "workflows/{$id}-missing/message-streams"); + $client->registerWorker($worker, $queue, [$definition['workflow_type']], []); + [$task, $state['poll1']] = $poll(); + $state['complete1_refusals'] = [ + 'wait_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 1]]]), + 'cursor_ahead' => $complete($task, ['commands' => [$wait(0)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]), + 'bad_name' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => 'bad name', 'after_position' => 0]]]), + 'repeated_wait' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [ + ['stream_name' => $stream, 'after_position' => 0], ['stream_name' => $stream, 'after_position' => 0]]]), + 'old_protocol' => $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]], '1.14'), + ]; + $state['complete1'] = $complete($task, ['commands' => [$wait(0)], 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]); + $state['list_waiting'] = $raw('GET', "workflows/{$id}/message-streams"); + $messages = $definition['messages']; + $state['appends'] = [ + 'm1' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_duplicate' => $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]), + 'm1_conflict' => $append($stream, ['message_id' => 'm1', 'input' => $messages['conflict']]), + // The published SDK client's own append. + 'm2_sdk' => ['status' => null, 'response' => $client->appendMessageStream($id, $stream, 'm2', $messages['m2'])], + 'no_id' => $append($stream, ['input' => $messages['m3']]), + 'bad_id' => $append($stream, ['message_id' => 'bad id!', 'input' => $messages['m3']]), + 'long_id' => $append($stream, ['message_id' => str_repeat('a', $definition['long_message_id_length']), 'input' => $messages['m3']]), + 'bad_input' => $append($stream, ['message_id' => 'm3', 'input' => 'text']), + 'bad_stream' => $append($definition['bad_stream'], ['message_id' => 'm3', 'input' => $messages['m3']]), + 'unknown_workflow' => $raw('POST', "workflows/{$id}-missing/message-streams/{$stream}/messages", ['message_id' => 'm1', 'input' => [1]]), + 'other_stream' => $append($definition['other_stream'], ['message_id' => 'a1']), + 'reserved_signal' => $raw('POST', "workflows/{$id}/signal/__durable_workflow_message_stream", ['input' => [1]]), + ]; + $state['list_after_appends'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['show_after_appends'] = $raw('GET', "workflows/{$id}/message-streams/{$stream}"); + [$task, $state['poll2']] = $poll(); + $state['complete2_refusals'] = [ + 'beyond' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]), + 'negative' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => -1]]]), + 'wait_behind' => $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]], + 'message_stream_waits' => [['stream_name' => $stream, 'after_position' => 0]]]), + ]; + $state['complete2'] = $complete($task, ['commands' => [$wait(1)], 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 1]]]); + $state['history_after2'] = $streamEvents($history($state['run_id'])); + $state['list_after2'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['m3'] = $append($stream, ['message_id' => 'm3', 'input' => $messages['m3']]); + [$task, $state['poll3']] = $poll(); + $state['complete3'] = $complete($task, ['commands' => [['type' => 'continue_as_new', 'arguments' => $codec->envelope([$fixture['input']])]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 2]]]); + $state['list_after_continue'] = $raw('GET', "workflows/{$id}/message-streams"); + [$task, $state['poll4']] = $poll(); + $state['complete4'] = $complete($task, ['commands' => [['type' => 'complete_workflow', 'result' => $codec->envelope('done')]], + 'message_stream_cursors' => [['stream_name' => $stream, 'through_position' => 3]]]); + $state['continued_run_id'] = $state['poll4']['run_id']; + $state['list_final'] = $raw('GET', "workflows/{$id}/message-streams"); + $state['appends']['closed'] = $append($stream, ['message_id' => 'm4', 'input' => [1]]); + $state['appends']['closed_duplicate'] = $append($stream, ['message_id' => 'm1', 'input' => $messages['m1']]); + $eventTypes = static fn (array $events): array => array_map(static fn (array $event): string => $event['event_type'], $events); + $firstRun = $history($state['run_id']); + $continuedRun = $history($state['continued_run_id']); + $state['first_run_events'] = $eventTypes($firstRun); + $state['first_run_streams'] = $streamEvents($firstRun); + $state['continued_run_events'] = $eventTypes($continuedRun); + $state['continued_run_streams'] = $streamEvents($continuedRun); + + return $state; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 4df73a65f..e1458ed29 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -439,6 +439,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $clusterInfo = isset($fixture['cluster_info']) ? finishHttpClusterInfo($client, $fixture, $workflowId, $url) : null; $repairPass = isset($fixture['repair_pass']) ? finishHttpRepairPass($client, $fixture, $workflowId, $url) : null; $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; + $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -547,6 +548,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'cluster_info' => $clusterInfo, 'repair_pass' => $repairPass, 'namespace_delete' => $namespaceDelete, + 'message_streams' => $messageStreams, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -859,6 +861,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/cluster-info-probe.php'; require __DIR__.'/repair-pass-probe.php'; require __DIR__.'/namespace-delete-probe.php'; + require __DIR__.'/message-streams-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1044,6 +1047,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['namespace_delete'])) { $observation['namespace_delete'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_namespaces']; } + if ($mode === 'embedded' && isset($fixture['message_streams'])) { + $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/published-sdk-patch-profile.sh b/scripts/conformance/server-parity/published-sdk-patch-profile.sh index 5e012bd34..738c21d1d 100644 --- a/scripts/conformance/server-parity/published-sdk-patch-profile.sh +++ b/scripts/conformance/server-parity/published-sdk-patch-profile.sh @@ -25,14 +25,29 @@ shared=(-v "$PWD:/source:ro" -v "$PWD/published-sdk-adapters:/sdk-adapters:ro" -e DW_PARITY_RUST_PATCH_WORKER=/sdk-adapters/rust-worker -e PYTHONDONTWRITEBYTECODE=1 -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) -for target in php rust; do - if test "$target" = php; then url="$php_url"; else url="$native_url"; fi +# Server #721. On SQLite the published PHP API server and its maintenance loop +# share one database file. SQLite refuses a deferred transaction's upgrade to a +# writer at once, without a busy wait, so PHP answers a contended poll with its +# typed backend_lock_pressure refusal. The published Python run_until helper +# then ends its Worker (sdk-python #218). This cohort tests patch replacement, +# not writer contention, so the PHP record runs with the maintenance loop +# stopped, and the loop restarts right after it. The fixtures use no timers or +# repair. The observer still never retries a poll refusal. +. scripts/conformance/server-parity/published-sdk-patch/maintenance.sh +record() { docker run --rm --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" --entrypoint node "${shared[@]}" "$image" \ - scripts/conformance/server-parity.mjs record --mode http --url "$url" --target "$target" \ + scripts/conformance/server-parity.mjs record --mode http --url "$1" --target "$2" \ --prefix published-sdk-v1 --artifacts "$profile/artifacts.json" "${fixtures[@]}" \ - --output "/evidence/published-sdk-patch/$target/record.json" -done + --output "/evidence/published-sdk-patch/$2/record.json" +} +if test "$backend" = sqlite; then + run_without_maintenance parity-evidence/published-sdk-patch/maintenance-schedule.txt -- \ + record "$php_url" php +else + record "$php_url" php +fi +record "$native_url" rust case "$backend" in mysql) embedded=(-e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=embedded_ref -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password) ;; diff --git a/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh new file mode 100644 index 000000000..33528001f --- /dev/null +++ b/scripts/conformance/server-parity/published-sdk-patch/maintenance.sh @@ -0,0 +1,62 @@ +# Sourced by published-sdk-patch-profile.sh (Server #721). +# +# run_without_maintenance SCHEDULE -- COMMAND... +# +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + test "$(maintenance_state)" = running || return 1 + sleep "$maintenance_poll_seconds" + done + test "$(maintenance_state)" = running +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then + return 0 + fi + done + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 + return 1 +} + +run_without_maintenance() { + local schedule="$1" + shift + test "$1" = -- && shift + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + test "$maintenance_container" = "$RESOURCE_SCOPE-php-maintenance" + echo "before: $(maintenance_state)" > "$schedule" + grep -qx 'before: running' "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/tests/Fixtures/ServerParity/message-streams.json b/tests/Fixtures/ServerParity/message-streams.json new file mode 100644 index 000000000..49b0ea7ef --- /dev/null +++ b/tests/Fixtures/ServerParity/message-streams.json @@ -0,0 +1,182 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "message-streams", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "message_streams": { + "workflow_type": "parity.v1.message_streams", + "stream": "orders", + "other_stream": "audit", + "bad_stream": "bad name!", + "condition_fingerprint_source": "message-stream:orders", + "manifest": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "messages": { + "m1": [ + { + "n": 1 + } + ], + "m2": [ + { + "n": 2 + } + ], + "m3": [ + { + "n": 3 + } + ], + "conflict": [ + { + "n": 9 + } + ] + }, + "long_message_id_length": 192, + "refusals": { + "stream_name": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "metadata_stream_name": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "message_id_required": "The message id field is required.", + "message_id_format": "The message id field format is invalid.", + "message_id_max": "The message id field must not be greater than 191 characters.", + "input_array": "The input field must be an array.", + "wait_after_cursor": "The pending wait must start after the durable stream cursor.", + "cursor_beyond": "The cursor cannot advance beyond the last accepted stream position.", + "duplicate_wait": "A stream may have only one pending wait per workflow task completion.", + "negative_cursor": "The message_stream_cursors.0.through_position field must be at least 0.", + "old_protocol_remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ] + }, + "contract": "Message streams (published Server 2.11.1, MessageStreamController and MessageStreamService over the Workflow package's runtime-reserved signal). Cluster info publishes the message streams contract and the worker capability the same contract with supported true. A client appends ordered messages to a named stream of a workflow instance: the first message is position 1, a repeated message id with the same payload is a duplicate at its original position, and with another payload a 409 message_identity_conflict. Each accepted message reaches the current run as a __durable_workflow_message_stream SignalReceived whose single argument holds the message schema, stream name, message id, position and the inline payload envelope. Refused requests are counted as malformed without keeping their payload, by a diagnostic message id. Diagnostics list every stream of the instance by name with positions, pending counts, the pending wait, duplicate and malformed counts and the last input, without payloads. A worker on protocol 1.15 or newer reports open receives with message_stream_waits and consumed messages with message_stream_cursors on a workflow task completion; the server checks both before the completion changes anything, records MessageCursorAdvanced for each moved cursor and clears the wait. An open condition wait is woken by a stream message, which it consumes without SignalApplied. When the run continues as new, the next run receives a payload-free cursor checkpoint for each acknowledged stream and every unconsumed message again, in append order; signals the old run had not consumed are rejected. Appends to a closed run are refused with run_not_active, duplicates included. Embedded applications have no HTTP message streams." +} diff --git a/tests/Fixtures/ServerParityExpected/message-streams-observation.json b/tests/Fixtures/ServerParityExpected/message-streams-observation.json new file mode 100644 index 000000000..215042c4c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/message-streams-observation.json @@ -0,0 +1,1323 @@ +{ + "recorded_from": "published Server 2.11.1 (ghcr.io/durable-workflow/server@sha256:bf8bed13f5f809ee3425233180a581353eac4683adbee5c4ecbf63bb18bbd102) on SQLite, workflow id probe-5", + "message_streams": { + "applicable": true, + "workflow_id": "probe-5-ms", + "cluster_contract": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + } + }, + "worker_capability": { + "schema": "durable-workflow.v2.message-streams.contract", + "version": 1, + "capability_flag": "message_streams", + "minimum_worker_protocol_version": "1.15", + "scope": "workflow_instance", + "transport": { + "append_path": "/api/workflows/{workflow_id}/message-streams/{stream_name}/messages", + "diagnostics_path": "/api/workflows/{workflow_id}/message-streams", + "internal_signal": "__durable_workflow_message_stream", + "message_schema": "durable-workflow.v2.message-stream.message", + "cursor_schema": "durable-workflow.v2.message-stream.cursor", + "payload_envelope": "inline_avro_at_the_server_to_worker_delivery_boundary", + "durable_payload": "exact_inline_avro_bytes_or_the_verified_external_storage_reference", + "payload_visibility": "payload_is_present_only_in_durable_signal_history_and_is_omitted_from_stream_diagnostics" + }, + "identity": { + "producer_key": "message_id", + "order_key": "position", + "first_position": 1, + "duplicate_same_payload": "return_existing_position_without_redelivery", + "duplicate_different_payload": "reject_message_identity_conflict" + }, + "cursor": { + "ownership": "server", + "worker_completion_field": "message_stream_cursors", + "monotonic": true, + "advancement": "highest_contiguous_consumed_position", + "history_event": "MessageCursorAdvanced", + "continue_as_new": "a_payload_free_cursor_checkpoint_and_unconsumed_messages_are_delivered_to_the_current_run" + }, + "waiting": { + "worker_completion_field": "message_stream_waits", + "durable_wakeup": "runtime_reserved_signal_task_resume", + "pending_wait_is_diagnostic": true + }, + "batch": { + "minimum": 1, + "maximum": 100, + "waits_for": "first_message", + "returns": "up_to_maximum_currently_available_messages" + }, + "replay": { + "ordering_source": "server_assigned_position", + "deduplication_source": "message_id_and_instance_cursor", + "worker_replacement": "history_replay_reconstructs_pending_messages_and_cursor_acknowledgement_is_idempotent", + "server_restart": "stream_items_cursor_and_wait_state_are_database_durable", + "external_payloads": "references_remain_reference_backed_at_rest_and_are_resolved_for_worker_delivery_then_removed_with_namespace_retention" + }, + "sdk_handoffs": { + "php": "WorkflowContext::messageStream(string)->receive(int)", + "python": "WorkflowContext.message_stream(str).receive(max_items)", + "rust": "WorkflowContext::message_stream(name).receive(max_items).await" + }, + "conformance": { + "runtimes": [ + "php", + "python", + "rust" + ], + "targets": [ + "server", + "cloud" + ], + "required_scenarios": [ + "repeated_messages_are_observed_in_position_order_exactly_once", + "duplicate_delivery_preserves_one_logical_message", + "worker_is_replaced_mid_stream", + "continue_as_new_retains_cursor_and_pending_messages", + "server_restart_retains_cursor_wait_and_order" + ], + "artifacts": "published" + }, + "supported": true + }, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "list_before": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 0, + "streams": [] + } + }, + "show_before": { + "status": 404, + "response": { + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "reason": "message_stream_not_found" + } + }, + "show_bad_name": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "list_unknown": { + "status": 404, + "response": { + "message": "Workflow not found.", + "reason": "instance_not_found" + } + }, + "poll1": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [] + }, + "complete1_refusals": { + "wait_ahead": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + }, + "cursor_ahead": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "bad_name": { + "status": 422, + "response": { + "message": "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "message_stream_waits.0.stream_name": [ + "The stream_name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "repeated_wait": { + "status": 422, + "response": { + "message": "A stream may have only one pending wait per workflow task completion.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + }, + "validation_errors": { + "message_stream_waits.1.stream_name": [ + "A stream may have only one pending wait per workflow task completion." + ] + } + } + }, + "old_protocol": { + "status": 409, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "rejected", + "reason": "message_streams_unavailable", + "requested_version": "1.14", + "minimum_protocol_version": "1.15", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.15 or newer." + } + } + }, + "complete1": { + "status": 200, + "response": { + "task_id": "01m5n2d2h1y9p7kk8qea67rxxj", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "list_waiting": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 1, + "streams": [ + { + "stream_name": "orders", + "last_position": 0, + "cursor_position": 0, + "pending_count": 0, + "pending_wait": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "after_position": 0, + "since": "2026-10-23T19:04:33.000000Z" + }, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": null, + "last_input_message_id": null, + "last_input_at": null + } + ] + } + }, + "appends": { + "m1": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "m1_duplicate": { + "status": 200, + "response": { + "accepted": true, + "duplicate": true, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "duplicate", + "reason": null + } + }, + "m1_conflict": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "cursor_position": 0, + "outcome": "rejected", + "reason": "message_identity_conflict" + } + }, + "m2_sdk": { + "status": null, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "no_id": { + "status": 422, + "response": { + "message": "The message id field is required.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field is required." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field is required." + ] + } + } + }, + "bad_id": { + "status": 422, + "response": { + "message": "The message id field format is invalid.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field format is invalid." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field format is invalid." + ] + } + } + }, + "long_id": { + "status": 422, + "response": { + "message": "The message id field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "message_id": [ + "The message id field must not be greater than 191 characters." + ] + } + } + }, + "bad_input": { + "status": 422, + "response": { + "message": "The input field must be an array.", + "reason": "validation_failed", + "errors": { + "input": [ + "The input field must be an array." + ] + }, + "validation_errors": { + "input": [ + "The input field must be an array." + ] + } + } + }, + "bad_stream": { + "status": 422, + "response": { + "message": "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens.", + "reason": "validation_failed", + "errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + }, + "validation_errors": { + "stream_name": [ + "The stream name must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens." + ] + } + } + }, + "unknown_workflow": { + "status": 404, + "response": { + "accepted": false, + "workflow_id": "probe-5-ms-missing", + "stream_name": "orders", + "outcome": "rejected", + "reason": "instance_not_found" + } + }, + "other_stream": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "cursor_position": 0, + "outcome": "accepted", + "reason": null + } + }, + "reserved_signal": { + "status": 422, + "response": { + "workflow_id": "probe-5-ms", + "signal_name": "__durable_workflow_message_stream", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport.", + "accepted": false, + "control_plane": { + "schema": "durable-workflow.v2.control-plane-response", + "version": 1, + "operation": "signal", + "workflow_id": "probe-5-ms", + "contract": { + "schema": "durable-workflow.v2.control-plane-response.contract", + "version": 1, + "legacy_field_policy": "reject_non_canonical", + "legacy_fields": { + "query": "query_name", + "signal": "signal_name", + "update": "update_name", + "wait_policy": "wait_for" + }, + "required_fields": [ + "workflow_id", + "operation_name", + "operation_name_field" + ], + "success_fields": [ + "outcome" + ], + "rejection_fields": [ + "workflow_id", + "run_id", + "signal_name", + "command_status", + "command_source", + "target_scope", + "outcome", + "reason", + "rejection_reason", + "rejection_category", + "message", + "command_contract_source", + "command_contract_backfill_needed", + "command_contract_backfill_available", + "declared_signals", + "signal_admission", + "retryable", + "error_id" + ], + "rejection_reasons": [ + "instance_not_found", + "historical_run_command_rejected", + "unknown_signal", + "invalid_signal_arguments", + "run_not_active", + "instance_not_started", + "selected_run_not_current", + "configured_workflow_type_invalid", + "v1_projection_read_only", + "backend_lock_pressure", + "control_plane_internal_error" + ] + }, + "operation_name": "__durable_workflow_message_stream", + "operation_name_field": "signal_name", + "outcome": "rejected", + "reason": "runtime_reserved_signal", + "message": "This signal name is reserved for the durable runtime transport." + } + } + }, + "m3": { + "status": 202, + "response": { + "accepted": true, + "duplicate": false, + "workflow_id": "probe-5-ms", + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "cursor_position": 1, + "outcome": "accepted", + "reason": null + } + }, + "closed": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m4", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + }, + "closed_duplicate": { + "status": 409, + "response": { + "accepted": false, + "duplicate": false, + "workflow_id": "probe-5-ms", + "stream_name": "orders", + "message_id": "m1", + "position": null, + "outcome": "rejected", + "reason": "run_not_active" + } + } + }, + "list_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "stream": { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 0, + "pending_count": 2, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + } + }, + "poll2": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + } + ] + }, + "complete2_refusals": { + "beyond": { + "status": 422, + "response": { + "message": "The cursor cannot advance beyond the last accepted stream position.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The cursor cannot advance beyond the last accepted stream position." + ] + } + } + }, + "negative": { + "status": 422, + "response": { + "message": "The message_stream_cursors.0.through_position field must be at least 0.", + "reason": "validation_failed", + "errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + }, + "validation_errors": { + "message_stream_cursors.0.through_position": [ + "The message_stream_cursors.0.through_position field must be at least 0." + ] + } + } + }, + "wait_behind": { + "status": 422, + "response": { + "message": "The pending wait must start after the durable stream cursor.", + "reason": "validation_failed", + "errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + }, + "validation_errors": { + "message_stream_waits.0.after_position": [ + "The pending wait must start after the durable stream cursor." + ] + } + } + } + }, + "complete2": { + "status": 200, + "response": { + "task_id": "01m5n2d3eb25p3d5t00y35nhhk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "waiting", + "created_task_ids": [ + "01m5n2d516v62fqj8j4dzkb82t" + ], + "reason": null + } + }, + "history_after2": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + } + ], + "list_after2": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 2, + "cursor_position": 1, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "malformed", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:34.000000Z" + } + ] + } + }, + "poll3": { + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete3": { + "status": 200, + "response": { + "task_id": "01m5n2d516v62fqj8j4dzkb82t", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d2gpbkqp605f4w699nfy", + "run_status": "completed", + "created_task_ids": [ + "01m5n2d5gnm3yatnb4jaqtvfnk" + ], + "reason": null + } + }, + "list_after_continue": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 2, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "poll4": { + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "history": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + } + ] + }, + "complete4": { + "status": 200, + "response": { + "task_id": "01m5n2d5gnm3yatnb4jaqtvfnk", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "continued_run_id": "01m5n2d5g530fnzmcjg1mtmffs", + "list_final": { + "status": 200, + "response": { + "workflow_id": "probe-5-ms", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "last_position": 1, + "cursor_position": 0, + "pending_count": 1, + "pending_wait": null, + "duplicate_count": 0, + "malformed_count": 0, + "last_input_outcome": "accepted", + "last_input_message_id": "a1", + "last_input_at": "2026-10-23T19:04:34.000000Z" + }, + { + "stream_name": "orders", + "last_position": 3, + "cursor_position": 3, + "pending_count": 0, + "pending_wait": null, + "duplicate_count": 2, + "malformed_count": 4, + "last_input_outcome": "accepted", + "last_input_message_id": "m3", + "last_input_at": "2026-10-23T19:04:35.000000Z" + } + ] + } + }, + "first_run_events": [ + "StartAccepted", + "WorkflowStarted", + "ConditionWaitOpened", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "ConditionWaitOpened", + "MessageCursorAdvanced", + "SignalReceived", + "MessageCursorAdvanced", + "ConditionWaitSatisfied", + "WorkflowContinuedAsNew", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "first_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 1 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m2", + "position": 2, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 2 + } + ] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 0, + "new_position": 1, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 0, + "new_position": 1, + "task_status": "completed" + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "instance", + "previous_position": 1, + "new_position": 2, + "task_status": "leased" + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 1, + "new_position": 2, + "task_status": "completed" + } + ], + "continued_run_events": [ + "WorkflowStarted", + "SignalReceived", + "SignalReceived", + "SignalReceived", + "WorkflowCompleted", + "SignalRejected", + "SignalRejected", + "SignalRejected", + "MessageCursorAdvanced" + ], + "continued_run_streams": [ + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.cursor", + "stream_name": "orders", + "through_position": 2 + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "audit", + "message_id": "a1", + "position": 1, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [] + } + ] + }, + { + "event_type": "SignalReceived", + "deliveries": [ + { + "schema": "durable-workflow.v2.message-stream.message", + "stream_name": "orders", + "message_id": "m3", + "position": 3, + "payload_envelope": [ + "blob", + "codec" + ], + "payload": [ + { + "n": 3 + } + ] + } + ] + }, + { + "event_type": "MessageCursorAdvanced", + "stream_key": "orders", + "previous_position": 2, + "new_position": 3, + "task_status": "completed" + } + ] + } +} diff --git a/tests/Unit/PatchCohortMaintenanceTest.mjs b/tests/Unit/PatchCohortMaintenanceTest.mjs new file mode 100644 index 000000000..24398e2cc --- /dev/null +++ b/tests/Unit/PatchCohortMaintenanceTest.mjs @@ -0,0 +1,100 @@ +// Server #721: the SQLite published-SDK patch cohort stops PHP's maintenance +// loop for its PHP record. Every failure path must leave the loop restored or +// fail loudly, and never record a restart that did not stay running. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const helper = resolve('scripts/conformance/server-parity/published-sdk-patch/maintenance.sh'); + +// One container's state in a file; every call is logged. +// SCENARIO=stop-nonzero: the stop takes effect but exits 1. +// SCENARIO=immediate-exit: start exits 0, the container runs for one +// inspection and then exits. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +test "\${!#}" = dw-parity-7-2-sqlite-php-maintenance || exit 3 +case "$1" in + inspect) + state=$(cat "$dir/state") + if test "$state" = running-once; then echo exited > "$dir/state"; state=running; fi + echo "$state" ;; + stop) echo exited > "$dir/state"; test "$SCENARIO" != stop-nonzero ;; + start) + if test "$SCENARIO" = immediate-exit; then echo running-once > "$dir/state"; else echo running > "$dir/state"; fi ;; + *) exit 2 ;; +esac +`; + +function run(scenario, command, scope = 'dw-parity-7-2-sqlite') { + const dir = mkdtempSync(join(tmpdir(), 'patch-maintenance-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', ['-c', + `set -euo pipefail; . "$HELPER"; run_without_maintenance "$SCHEDULE" -- ${command}`], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05', + GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', RESOURCE_SCOPE: scope}, + }); + const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); + const calls = read('calls').trim().split('\n'); + return { + status: result.status, + stderr: result.stderr, + schedule: read('schedule.txt').trim().split('\n'), + starts: calls.filter(call => call.startsWith('start ')).length, + stopIndex: calls.findIndex(call => call.startsWith('stop ')), + lastStart: calls.findLastIndex(call => call.startsWith('start ')), + state: read('state').trim(), + }; +} + +test('a normal window stops, runs, and restores a loop that stays running', () => { + const result = run('normal', 'true'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.schedule, ['before: running', 'during php record: exited', 'after php record: running for 0.05s x3']); + assert.equal(result.starts, 1, 'restoration is disarmed after the sustained check'); + assert.equal(result.state, 'running'); +}); + +test('a stop that takes effect but exits nonzero still restores', () => { + const result = run('stop-nonzero', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.lastStart > result.stopIndex, 'a start follows the stop'); + assert.equal(result.state, 'running'); + assert.deepEqual(result.schedule, ['before: running']); +}); + +test('a failed PHP record restores the loop', () => { + const result = run('normal', 'false'); + assert.notEqual(result.status, 0); + assert.equal(result.state, 'running'); + assert.ok(result.starts >= 1); + assert.ok(!result.schedule.some(line => line.startsWith('after php record'))); +}); + +test('a restart that exits at once stays armed and fails without recording running', () => { + const result = run('immediate-exit', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.schedule.includes('after php record: exited'), result.schedule.join('\n')); + assert.ok(!result.schedule.some(line => line.startsWith('after php record: running'))); + // Three attempts after the record, then three more from the still-armed trap. + assert.equal(result.starts, 6); + assert.match(result.stderr, /did not stay running/); +}); + +test('a container outside the job scope is never touched', () => { + const result = run('normal', 'true', 'dw-parity-other-job'); + assert.notEqual(result.status, 0); + assert.equal(result.starts, 0); + assert.equal(result.stopIndex, -1); + assert.equal(result.state, 'running'); +}); diff --git a/tests/Unit/ServerParityMessageStreamsContractTest.mjs b/tests/Unit/ServerParityMessageStreamsContractTest.mjs new file mode 100644 index 000000000..f98be161b --- /dev/null +++ b/tests/Unit/ServerParityMessageStreamsContractTest.mjs @@ -0,0 +1,81 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkMessageStreams} from '../../scripts/conformance/server-parity/message-streams-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/message-streams.json', import.meta.url))); +// What published Server 2.11.1 answered for this fixture's probe. +const recorded = JSON.parse(readFileSync(new URL('../Fixtures/ServerParityExpected/message-streams-observation.json', import.meta.url))); +const model = () => structuredClone({mode: 'http', message_streams: recorded.message_streams}); +const projection = {scope: 'message_streams', first_run_events: 17, continued_run_events: 9}; +const check = observation => checkMessageStreams(fixture, observation, 'probe-5'); +const stream = (state, step, name) => state[step].response.streams.find(entry => entry.stream_name === name); + +test('published PHP message streams pass', () => assert.deepStrictEqual(check(model()), projection)); +test('embedded explicitly declares HTTP message streams inapplicable', () => assert.deepStrictEqual(check( + {mode: 'embedded', message_streams: {applicable: false, reason: 'embedded_has_no_http_message_streams'}}), projection)); +test('the recorded observation keeps the published reference', () => { + assert.match(recorded.recorded_from, /^published Server 2\.11\.1 /); + assert.equal(recorded.message_streams.workflow_id, 'probe-5-ms'); +}); + +for (const [label, mutate] of [ + ['a missing observation', s => { for (const key of Object.keys(s)) delete s[key]; }], + ['no contract in cluster info', s => s.cluster_contract = null], + ['another contract version', s => s.cluster_contract.version = 2], + ['the capability as a plain flag', s => s.worker_capability = false], + ['an unsupported capability', s => s.worker_capability.supported = false], + ['a stream before any append', s => s.list_before.response.count = 1], + ['an unknown stream answered', s => s.show_before = {status: 200, response: {workflow_id: s.workflow_id, stream: null}}], + ['a bad stream name served', s => s.show_bad_name = {status: 404, response: {reason: 'message_stream_not_found'}}], + ['an unknown workflow listed', s => s.list_unknown = {status: 200, response: {workflow_id: 'x', count: 0, streams: []}}], + ['a wait ahead of the cursor accepted', s => s.complete1_refusals.wait_ahead = s.complete1], + ['a cursor past the stream accepted', s => s.complete1_refusals.cursor_ahead.status = 200], + ['two waits on one stream accepted', s => s.complete1_refusals.repeated_wait.response.errors = {}], + ['stream metadata before protocol 1.15 accepted', s => s.complete1_refusals.old_protocol.status = 200], + ['the wait not recorded', s => stream(s, 'list_waiting', 'orders').pending_wait = null], + ['a wait time without microseconds', s => stream(s, 'list_waiting', 'orders').pending_wait.since = '2026-10-23T18:00:00Z'], + ['the first message at position 0', s => s.appends.m1.response.position = 0], + ['an accepted append without its run', s => delete s.appends.m1.response.run_id], + ['a duplicate appended again', s => s.appends.m1_duplicate = {status: 202, response: {...s.appends.m2_sdk.response, message_id: 'm1', position: 3}}], + ['a conflicting payload accepted', s => Object.assign(s.appends.m1_conflict, {status: 200})], + ['a missing message id accepted', s => s.appends.no_id.status = 202], + ['a long message id accepted', s => s.appends.long_id.status = 202], + ['input that is not a list accepted', s => s.appends.bad_input.status = 202], + ['an append to an unknown workflow with the service shape', s => s.appends.unknown_workflow.response.duplicate = false], + ['the runtime signal accepted from a client', s => s.appends.reserved_signal = {status: 202, response: {outcome: 'signal_received'}}], + ['a refused stream name counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 5], + ['a malformed append not counted', s => stream(s, 'list_after_appends', 'orders').malformed_count = 3], + ['a raw refused message id kept', s => stream(s, 'list_after_appends', 'orders').last_input_message_id = 'bad id!'], + ['a wait kept after an append', s => { stream(s, 'list_after_appends', 'orders').pending_wait = stream(s, 'list_waiting', 'orders').pending_wait; }], + ['streams listed out of name order', s => s.list_after_appends.response.streams.reverse()], + ['a show unlike the list', s => s.show_after_appends.response.stream.duplicate_count = 0], + ['messages delivered out of order', s => s.poll2.history.reverse()], + ['two messages in one signal', s => s.poll2.history[0].deliveries.push(s.poll2.history[1].deliveries[0])], + ['a payload delivered by reference', s => s.poll2.history[0].deliveries[0].payload_envelope = ['codec', 'external_storage']], + ['a negative cursor accepted', s => s.complete2_refusals.negative.status = 200], + ['a wait behind the new cursor accepted', s => s.complete2_refusals.wait_behind.status = 200], + ['the next message task not reported', s => s.complete2.response.created_task_ids = []], + ['no stream cursor event', s => s.history_after2 = s.history_after2.filter(event => event.stream_key !== 'orders')], + ['the stream cursor before the task completed', s => { + s.history_after2.find(event => event.stream_key === 'orders').task_status = 'leased'; + }], + ['the cursor not moved', s => stream(s, 'list_after2', 'orders').cursor_position = 0], + ['the third message without the cursor', s => s.appends.m3.response.cursor_position = 0], + ['the cursor not carried to the continued run', s => s.poll4.history.shift()], + ['a consumed message delivered again', s => s.poll4.history.splice(1, 0, structuredClone(s.poll2.history[0]))], + ['unconsumed messages out of append order', s => s.poll4.history.push(s.poll4.history.splice(1, 1)[0])], + ['the continuation as the same run', s => s.continued_run_id = s.run_id], + ['an append to a closed instance accepted', s => s.appends.closed.status = 202], + ['a closed duplicate answered as a duplicate', s => Object.assign(s.appends.closed_duplicate.response, {reason: null, outcome: 'duplicate'})], + ['SignalApplied for a consumed stream message', s => s.first_run_events.splice(7, 0, 'SignalApplied')], + ['unconsumed signals not rejected', s => s.first_run_events = s.first_run_events.filter(type => type !== 'SignalRejected')], + ['a continued run without its final cursor', s => s.continued_run_events.pop()], + ['diagnostics with payloads', s => { stream(s, 'list_final', 'orders').payload = [{n: 3}]; }], +]) { + test(`refuses ${label}`, () => { + const observation = model(); + mutate(observation.message_streams); + assert.throws(() => check(observation)); + }); +}